1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define TIME_OUT_POLL 1e9 30 #define WAIT_OFFLOAD_US 1000 31 32 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 33 #include <fpga_lte_fec.h> 34 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_4G_QUEUE_VALUE 4 37 #define VF_DL_4G_QUEUE_VALUE 4 38 #define UL_4G_BANDWIDTH 3 39 #define DL_4G_BANDWIDTH 3 40 #define UL_4G_LOAD_BALANCE 128 41 #define DL_4G_LOAD_BALANCE 128 42 #define FLR_4G_TIMEOUT 610 43 #endif 44 45 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 46 #include <rte_pmd_fpga_5gnr_fec.h> 47 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 48 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 49 #define VF_UL_5G_QUEUE_VALUE 4 50 #define VF_DL_5G_QUEUE_VALUE 4 51 #define UL_5G_BANDWIDTH 3 52 #define DL_5G_BANDWIDTH 3 53 #define UL_5G_LOAD_BALANCE 128 54 #define DL_5G_LOAD_BALANCE 128 55 #endif 56 57 #ifdef RTE_BASEBAND_ACC 58 #include <rte_acc_cfg.h> 59 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 60 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 61 #define ACC100_QMGR_NUM_AQS 16 62 #define ACC100_QMGR_NUM_QGS 2 63 #define ACC100_QMGR_AQ_DEPTH 5 64 #define ACC100_QMGR_INVALID_IDX -1 65 #define ACC100_QMGR_RR 1 66 #define ACC100_QOS_GBR 0 67 #define VRBPF_DRIVER_NAME ("intel_vran_boost_pf") 68 #define VRBVF_DRIVER_NAME ("intel_vran_boost_vf") 69 #define VRB_QMGR_NUM_AQS 16 70 #define VRB_QMGR_NUM_QGS 2 71 #define VRB_QMGR_AQ_DEPTH 5 72 #define VRB_QMGR_INVALID_IDX -1 73 #define VRB_QMGR_RR 1 74 #define VRB_QOS_GBR 0 75 #endif 76 77 #define OPS_CACHE_SIZE 256U 78 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 79 80 #define SYNC_WAIT 0 81 #define SYNC_START 1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 2048 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 98 #define HARQ_MEM_TOLERANCE 256 99 static struct test_bbdev_vector test_vector; 100 101 /* Switch between PMD and Interrupt for throughput TC */ 102 static bool intr_enabled; 103 104 /* LLR arithmetic representation for numerical conversion */ 105 static int ldpc_llr_decimals; 106 static int ldpc_llr_size; 107 /* Keep track of the LDPC decoder device capability flag */ 108 static uint32_t ldpc_cap_flags; 109 110 /* Represents tested active devices */ 111 static struct active_device { 112 const char *driver_name; 113 uint8_t dev_id; 114 uint16_t supported_ops; 115 uint16_t queue_ids[MAX_QUEUES]; 116 uint16_t nb_queues; 117 struct rte_mempool *ops_mempool; 118 struct rte_mempool *in_mbuf_pool; 119 struct rte_mempool *hard_out_mbuf_pool; 120 struct rte_mempool *soft_out_mbuf_pool; 121 struct rte_mempool *harq_in_mbuf_pool; 122 struct rte_mempool *harq_out_mbuf_pool; 123 } active_devs[RTE_BBDEV_MAX_DEVS]; 124 125 static uint8_t nb_active_devs; 126 127 /* Data buffers used by BBDEV ops */ 128 struct test_buffers { 129 struct rte_bbdev_op_data *inputs; 130 struct rte_bbdev_op_data *hard_outputs; 131 struct rte_bbdev_op_data *soft_outputs; 132 struct rte_bbdev_op_data *harq_inputs; 133 struct rte_bbdev_op_data *harq_outputs; 134 }; 135 136 /* Operation parameters specific for given test case */ 137 struct test_op_params { 138 struct rte_mempool *mp; 139 struct rte_bbdev_dec_op *ref_dec_op; 140 struct rte_bbdev_enc_op *ref_enc_op; 141 struct rte_bbdev_fft_op *ref_fft_op; 142 struct rte_bbdev_mldts_op *ref_mldts_op; 143 uint16_t burst_sz; 144 uint16_t num_to_process; 145 uint16_t num_lcores; 146 int vector_mask; 147 RTE_ATOMIC(uint16_t) sync; 148 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 149 }; 150 151 /* Contains per lcore params */ 152 struct thread_params { 153 uint8_t dev_id; 154 uint16_t queue_id; 155 uint32_t lcore_id; 156 uint64_t start_time; 157 double ops_per_sec; 158 double mbps; 159 uint8_t iter_count; 160 double iter_average; 161 double bler; 162 RTE_ATOMIC(uint16_t) nb_dequeued; 163 RTE_ATOMIC(int16_t) processing_status; 164 RTE_ATOMIC(uint16_t) burst_sz; 165 struct test_op_params *op_params; 166 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 167 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 168 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 169 struct rte_bbdev_mldts_op *mldts_ops[MAX_BURST]; 170 }; 171 172 /* Stores time statistics */ 173 struct test_time_stats { 174 /* Stores software enqueue total working time */ 175 uint64_t enq_sw_total_time; 176 /* Stores minimum value of software enqueue working time */ 177 uint64_t enq_sw_min_time; 178 /* Stores maximum value of software enqueue working time */ 179 uint64_t enq_sw_max_time; 180 /* Stores turbo enqueue total working time */ 181 uint64_t enq_acc_total_time; 182 /* Stores minimum value of accelerator enqueue working time */ 183 uint64_t enq_acc_min_time; 184 /* Stores maximum value of accelerator enqueue working time */ 185 uint64_t enq_acc_max_time; 186 /* Stores dequeue total working time */ 187 uint64_t deq_total_time; 188 /* Stores minimum value of dequeue working time */ 189 uint64_t deq_min_time; 190 /* Stores maximum value of dequeue working time */ 191 uint64_t deq_max_time; 192 }; 193 194 typedef int (test_case_function)(struct active_device *ad, 195 struct test_op_params *op_params); 196 197 /* Get device status before timeout exit */ 198 static inline void 199 timeout_exit(uint8_t dev_id) 200 { 201 struct rte_bbdev_info info; 202 rte_bbdev_info_get(dev_id, &info); 203 printf("Device Status %s\n", rte_bbdev_device_status_str(info.drv.device_status)); 204 } 205 206 static inline void 207 mbuf_reset(struct rte_mbuf *m) 208 { 209 m->pkt_len = 0; 210 211 do { 212 m->data_len = 0; 213 m = m->next; 214 } while (m != NULL); 215 } 216 217 /* Read flag value 0/1 from bitmap */ 218 static inline bool 219 check_bit(uint32_t bitmap, uint32_t bitmask) 220 { 221 return bitmap & bitmask; 222 } 223 224 static inline void 225 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 226 { 227 ad->supported_ops |= (1 << op_type); 228 } 229 230 static inline bool 231 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 232 { 233 return ad->supported_ops & (1 << op_type); 234 } 235 236 static inline bool 237 flags_match(uint32_t flags_req, uint32_t flags_present) 238 { 239 return (flags_req & flags_present) == flags_req; 240 } 241 242 static void 243 clear_soft_out_cap(uint32_t *op_flags) 244 { 245 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 246 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 247 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 248 } 249 250 /* This API is to convert all the test vector op data entries 251 * to big endian format. It is used when the device supports 252 * the input in the big endian format. 253 */ 254 static inline void 255 convert_op_data_to_be(void) 256 { 257 struct op_data_entries *op; 258 enum op_data_type type; 259 uint8_t nb_segs, *rem_data, temp; 260 uint32_t *data, len; 261 int complete, rem, i, j; 262 263 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 264 nb_segs = test_vector.entries[type].nb_segments; 265 op = &test_vector.entries[type]; 266 267 /* Invert byte endianness for all the segments */ 268 for (i = 0; i < nb_segs; ++i) { 269 len = op->segments[i].length; 270 data = op->segments[i].addr; 271 272 /* Swap complete u32 bytes */ 273 complete = len / 4; 274 for (j = 0; j < complete; j++) 275 data[j] = rte_bswap32(data[j]); 276 277 /* Swap any remaining bytes */ 278 rem = len % 4; 279 rem_data = (uint8_t *)&data[j]; 280 for (j = 0; j < rem/2; j++) { 281 temp = rem_data[j]; 282 rem_data[j] = rem_data[rem - j - 1]; 283 rem_data[rem - j - 1] = temp; 284 } 285 } 286 } 287 } 288 289 static int 290 check_dev_cap(const struct rte_bbdev_info *dev_info) 291 { 292 unsigned int i; 293 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 294 nb_harq_inputs, nb_harq_outputs; 295 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 296 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 297 298 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 299 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 300 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 301 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 302 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 303 304 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 305 if (op_cap->type != test_vector.op_type) 306 continue; 307 308 if (dev_data_endianness == RTE_BIG_ENDIAN) 309 convert_op_data_to_be(); 310 311 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 312 const struct rte_bbdev_op_cap_turbo_dec *cap = 313 &op_cap->cap.turbo_dec; 314 /* Ignore lack of soft output capability, just skip 315 * checking if soft output is valid. 316 */ 317 if ((test_vector.turbo_dec.op_flags & 318 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 319 !(cap->capability_flags & 320 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 321 printf( 322 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 323 dev_info->dev_name); 324 clear_soft_out_cap( 325 &test_vector.turbo_dec.op_flags); 326 } 327 328 if (!flags_match(test_vector.turbo_dec.op_flags, 329 cap->capability_flags)) 330 return TEST_FAILED; 331 if (nb_inputs > cap->num_buffers_src) { 332 printf("Too many inputs defined: %u, max: %u\n", 333 nb_inputs, cap->num_buffers_src); 334 return TEST_FAILED; 335 } 336 if (nb_soft_outputs > cap->num_buffers_soft_out && 337 (test_vector.turbo_dec.op_flags & 338 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 339 printf( 340 "Too many soft outputs defined: %u, max: %u\n", 341 nb_soft_outputs, 342 cap->num_buffers_soft_out); 343 return TEST_FAILED; 344 } 345 if (nb_hard_outputs > cap->num_buffers_hard_out) { 346 printf( 347 "Too many hard outputs defined: %u, max: %u\n", 348 nb_hard_outputs, 349 cap->num_buffers_hard_out); 350 return TEST_FAILED; 351 } 352 if (intr_enabled && !(cap->capability_flags & 353 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 354 printf( 355 "Dequeue interrupts are not supported!\n"); 356 return TEST_FAILED; 357 } 358 359 return TEST_SUCCESS; 360 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 361 const struct rte_bbdev_op_cap_turbo_enc *cap = 362 &op_cap->cap.turbo_enc; 363 364 if (!flags_match(test_vector.turbo_enc.op_flags, 365 cap->capability_flags)) 366 return TEST_FAILED; 367 if (nb_inputs > cap->num_buffers_src) { 368 printf("Too many inputs defined: %u, max: %u\n", 369 nb_inputs, cap->num_buffers_src); 370 return TEST_FAILED; 371 } 372 if (nb_hard_outputs > cap->num_buffers_dst) { 373 printf( 374 "Too many hard outputs defined: %u, max: %u\n", 375 nb_hard_outputs, cap->num_buffers_dst); 376 return TEST_FAILED; 377 } 378 if (intr_enabled && !(cap->capability_flags & 379 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 380 printf( 381 "Dequeue interrupts are not supported!\n"); 382 return TEST_FAILED; 383 } 384 385 return TEST_SUCCESS; 386 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 387 const struct rte_bbdev_op_cap_ldpc_enc *cap = 388 &op_cap->cap.ldpc_enc; 389 390 if (!flags_match(test_vector.ldpc_enc.op_flags, 391 cap->capability_flags)){ 392 printf("Flag Mismatch\n"); 393 return TEST_FAILED; 394 } 395 if (nb_inputs > cap->num_buffers_src) { 396 printf("Too many inputs defined: %u, max: %u\n", 397 nb_inputs, cap->num_buffers_src); 398 return TEST_FAILED; 399 } 400 if (nb_hard_outputs > cap->num_buffers_dst) { 401 printf( 402 "Too many hard outputs defined: %u, max: %u\n", 403 nb_hard_outputs, cap->num_buffers_dst); 404 return TEST_FAILED; 405 } 406 if (intr_enabled && !(cap->capability_flags & 407 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 408 printf( 409 "Dequeue interrupts are not supported!\n"); 410 return TEST_FAILED; 411 } 412 413 return TEST_SUCCESS; 414 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 415 const struct rte_bbdev_op_cap_ldpc_dec *cap = 416 &op_cap->cap.ldpc_dec; 417 418 if (!flags_match(test_vector.ldpc_dec.op_flags, 419 cap->capability_flags)){ 420 printf("Flag Mismatch\n"); 421 return TEST_FAILED; 422 } 423 if (nb_inputs > cap->num_buffers_src) { 424 printf("Too many inputs defined: %u, max: %u\n", 425 nb_inputs, cap->num_buffers_src); 426 return TEST_FAILED; 427 } 428 if (nb_hard_outputs > cap->num_buffers_hard_out) { 429 printf( 430 "Too many hard outputs defined: %u, max: %u\n", 431 nb_hard_outputs, 432 cap->num_buffers_hard_out); 433 return TEST_FAILED; 434 } 435 if (nb_harq_inputs > cap->num_buffers_hard_out) { 436 printf( 437 "Too many HARQ inputs defined: %u, max: %u\n", 438 nb_harq_inputs, 439 cap->num_buffers_hard_out); 440 return TEST_FAILED; 441 } 442 if (nb_harq_outputs > cap->num_buffers_hard_out) { 443 printf( 444 "Too many HARQ outputs defined: %u, max: %u\n", 445 nb_harq_outputs, 446 cap->num_buffers_hard_out); 447 return TEST_FAILED; 448 } 449 if (intr_enabled && !(cap->capability_flags & 450 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 451 printf( 452 "Dequeue interrupts are not supported!\n"); 453 return TEST_FAILED; 454 } 455 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 456 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 457 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 458 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 459 ))) { 460 printf("Skip loop-back with interrupt\n"); 461 return TEST_FAILED; 462 } 463 return TEST_SUCCESS; 464 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 465 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 466 467 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 468 printf("Flag Mismatch\n"); 469 return TEST_FAILED; 470 } 471 if (nb_inputs > cap->num_buffers_src) { 472 printf("Too many inputs defined: %u, max: %u\n", 473 nb_inputs, cap->num_buffers_src); 474 return TEST_FAILED; 475 } 476 return TEST_SUCCESS; 477 } else if (op_cap->type == RTE_BBDEV_OP_MLDTS) { 478 const struct rte_bbdev_op_cap_mld *cap = &op_cap->cap.mld; 479 if (!flags_match(test_vector.mldts.op_flags, cap->capability_flags)) { 480 printf("Flag Mismatch\n"); 481 return TEST_FAILED; 482 } 483 if (nb_inputs > cap->num_buffers_src) { 484 printf("Too many inputs defined: %u, max: %u\n", 485 nb_inputs, cap->num_buffers_src); 486 return TEST_FAILED; 487 } 488 return TEST_SUCCESS; 489 } 490 } 491 492 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 493 return TEST_SUCCESS; /* Special case for NULL device */ 494 495 return TEST_FAILED; 496 } 497 498 /* calculates optimal mempool size not smaller than the val */ 499 static unsigned int 500 optimal_mempool_size(unsigned int val) 501 { 502 return rte_align32pow2(val + 1) - 1; 503 } 504 505 /* allocates mbuf mempool for inputs and outputs */ 506 static struct rte_mempool * 507 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 508 int socket_id, unsigned int mbuf_pool_size, 509 const char *op_type_str) 510 { 511 unsigned int i; 512 uint32_t max_seg_sz = 0; 513 char pool_name[RTE_MEMPOOL_NAMESIZE]; 514 515 /* find max input segment size */ 516 for (i = 0; i < entries->nb_segments; ++i) 517 if (entries->segments[i].length > max_seg_sz) 518 max_seg_sz = entries->segments[i].length; 519 520 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 521 dev_id); 522 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 523 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 524 + FILLER_HEADROOM, 525 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 526 } 527 528 static int 529 create_mempools(struct active_device *ad, int socket_id, 530 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 531 { 532 struct rte_mempool *mp; 533 unsigned int ops_pool_size, mbuf_pool_size = 0; 534 char pool_name[RTE_MEMPOOL_NAMESIZE]; 535 const char *op_type_str; 536 enum rte_bbdev_op_type op_type = org_op_type; 537 538 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 539 struct op_data_entries *hard_out = 540 &test_vector.entries[DATA_HARD_OUTPUT]; 541 struct op_data_entries *soft_out = 542 &test_vector.entries[DATA_SOFT_OUTPUT]; 543 struct op_data_entries *harq_in = 544 &test_vector.entries[DATA_HARQ_INPUT]; 545 struct op_data_entries *harq_out = 546 &test_vector.entries[DATA_HARQ_OUTPUT]; 547 548 /* allocate ops mempool */ 549 ops_pool_size = optimal_mempool_size(RTE_MAX( 550 /* Ops used plus 1 reference op */ 551 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 552 /* Minimal cache size plus 1 reference op */ 553 (unsigned int)(1.5 * rte_lcore_count() * 554 OPS_CACHE_SIZE + 1)), 555 OPS_POOL_SIZE_MIN)); 556 557 if (org_op_type == RTE_BBDEV_OP_NONE) 558 op_type = RTE_BBDEV_OP_TURBO_ENC; 559 560 op_type_str = rte_bbdev_op_type_str(op_type); 561 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 562 563 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 564 ad->dev_id); 565 mp = rte_bbdev_op_pool_create(pool_name, op_type, 566 ops_pool_size, OPS_CACHE_SIZE, socket_id); 567 TEST_ASSERT_NOT_NULL(mp, 568 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 569 ops_pool_size, 570 ad->dev_id, 571 socket_id); 572 ad->ops_mempool = mp; 573 574 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 575 if (org_op_type == RTE_BBDEV_OP_NONE) 576 return TEST_SUCCESS; 577 578 /* Inputs */ 579 if (in->nb_segments > 0) { 580 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 581 in->nb_segments); 582 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 583 mbuf_pool_size, "in"); 584 TEST_ASSERT_NOT_NULL(mp, 585 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 586 mbuf_pool_size, 587 ad->dev_id, 588 socket_id); 589 ad->in_mbuf_pool = mp; 590 } 591 592 /* Hard outputs */ 593 if (hard_out->nb_segments > 0) { 594 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 595 hard_out->nb_segments); 596 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 597 mbuf_pool_size, 598 "hard_out"); 599 TEST_ASSERT_NOT_NULL(mp, 600 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 601 mbuf_pool_size, 602 ad->dev_id, 603 socket_id); 604 ad->hard_out_mbuf_pool = mp; 605 } 606 607 /* Soft outputs */ 608 if (soft_out->nb_segments > 0) { 609 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 610 soft_out->nb_segments); 611 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 612 mbuf_pool_size, 613 "soft_out"); 614 TEST_ASSERT_NOT_NULL(mp, 615 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 616 mbuf_pool_size, 617 ad->dev_id, 618 socket_id); 619 ad->soft_out_mbuf_pool = mp; 620 } 621 622 /* HARQ inputs */ 623 if (harq_in->nb_segments > 0) { 624 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 625 harq_in->nb_segments); 626 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 627 mbuf_pool_size, 628 "harq_in"); 629 TEST_ASSERT_NOT_NULL(mp, 630 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 631 mbuf_pool_size, 632 ad->dev_id, 633 socket_id); 634 ad->harq_in_mbuf_pool = mp; 635 } 636 637 /* HARQ outputs */ 638 if (harq_out->nb_segments > 0) { 639 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 640 harq_out->nb_segments); 641 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 642 mbuf_pool_size, 643 "harq_out"); 644 TEST_ASSERT_NOT_NULL(mp, 645 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 646 mbuf_pool_size, 647 ad->dev_id, 648 socket_id); 649 ad->harq_out_mbuf_pool = mp; 650 } 651 652 return TEST_SUCCESS; 653 } 654 655 static int 656 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 657 struct test_bbdev_vector *vector) 658 { 659 int ret; 660 unsigned int queue_id; 661 struct rte_bbdev_queue_conf qconf; 662 struct active_device *ad = &active_devs[nb_active_devs]; 663 unsigned int nb_queues; 664 enum rte_bbdev_op_type op_type = vector->op_type; 665 666 /* Configure fpga lte fec with PF & VF values 667 * if '-i' flag is set and using fpga device 668 */ 669 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 670 if ((get_init_device() == true) && 671 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 672 struct rte_fpga_lte_fec_conf conf; 673 unsigned int i; 674 675 printf("Configure FPGA LTE FEC Driver %s with default values\n", 676 info->drv.driver_name); 677 678 /* clear default configuration before initialization */ 679 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 680 681 /* Set PF mode : 682 * true if PF is used for data plane 683 * false for VFs 684 */ 685 conf.pf_mode_en = true; 686 687 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 688 /* Number of UL queues per VF (fpga supports 8 VFs) */ 689 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 690 /* Number of DL queues per VF (fpga supports 8 VFs) */ 691 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 692 } 693 694 /* UL bandwidth. Needed for schedule algorithm */ 695 conf.ul_bandwidth = UL_4G_BANDWIDTH; 696 /* DL bandwidth */ 697 conf.dl_bandwidth = DL_4G_BANDWIDTH; 698 699 /* UL & DL load Balance Factor to 64 */ 700 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 701 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 702 703 /**< FLR timeout value */ 704 conf.flr_time_out = FLR_4G_TIMEOUT; 705 706 /* setup FPGA PF with configuration information */ 707 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 708 TEST_ASSERT_SUCCESS(ret, 709 "Failed to configure 4G FPGA PF for bbdev %s", 710 info->dev_name); 711 } 712 #endif 713 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 714 if ((get_init_device() == true) && 715 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 716 struct rte_fpga_5gnr_fec_conf conf; 717 unsigned int i; 718 719 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 720 info->drv.driver_name); 721 722 /* clear default configuration before initialization */ 723 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 724 725 /* Set PF mode : 726 * true if PF is used for data plane 727 * false for VFs 728 */ 729 conf.pf_mode_en = true; 730 731 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 732 /* Number of UL queues per VF (fpga supports 8 VFs) */ 733 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 734 /* Number of DL queues per VF (fpga supports 8 VFs) */ 735 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 736 } 737 738 /* UL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 739 conf.ul_bandwidth = UL_5G_BANDWIDTH; 740 /* DL bandwidth. Needed only for Vista Creek 5GNR schedule algorithm */ 741 conf.dl_bandwidth = DL_5G_BANDWIDTH; 742 743 /* UL & DL load Balance Factor to 64 */ 744 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 745 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 746 747 /* setup FPGA PF with configuration information */ 748 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 749 TEST_ASSERT_SUCCESS(ret, 750 "Failed to configure 5G FPGA PF for bbdev %s", 751 info->dev_name); 752 } 753 #endif 754 #ifdef RTE_BASEBAND_ACC 755 if ((get_init_device() == true) && 756 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 757 struct rte_acc_conf conf; 758 unsigned int i; 759 760 printf("Configure ACC100 FEC device %s with default values\n", 761 info->drv.driver_name); 762 763 /* clear default configuration before initialization */ 764 memset(&conf, 0, sizeof(struct rte_acc_conf)); 765 766 /* Always set in PF mode for built-in configuration */ 767 conf.pf_mode_en = true; 768 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 769 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 770 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 771 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 772 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 773 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 774 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 775 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 776 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 777 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 778 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 779 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 780 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 781 } 782 783 conf.input_pos_llr_1_bit = true; 784 conf.output_pos_llr_1_bit = true; 785 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 786 787 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 788 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 789 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 790 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 791 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 792 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 793 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 794 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 795 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 796 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 797 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 798 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 799 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 800 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 801 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 802 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 803 804 /* setup PF with configuration information */ 805 ret = rte_acc_configure(info->dev_name, &conf); 806 TEST_ASSERT_SUCCESS(ret, 807 "Failed to configure ACC100 PF for bbdev %s", 808 info->dev_name); 809 } 810 if ((get_init_device() == true) && 811 (!strcmp(info->drv.driver_name, VRBPF_DRIVER_NAME))) { 812 struct rte_acc_conf conf; 813 unsigned int i; 814 815 printf("Configure Driver %s with default values\n", 816 info->drv.driver_name); 817 818 /* clear default configuration before initialization */ 819 memset(&conf, 0, sizeof(struct rte_acc_conf)); 820 821 /* Always set in PF mode for built-in configuration */ 822 conf.pf_mode_en = true; 823 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 824 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 825 conf.arb_dl_4g[i].gbr_threshold1 = VRB_QOS_GBR; 826 conf.arb_dl_4g[i].round_robin_weight = VRB_QMGR_RR; 827 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 828 conf.arb_ul_4g[i].gbr_threshold1 = VRB_QOS_GBR; 829 conf.arb_ul_4g[i].round_robin_weight = VRB_QMGR_RR; 830 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 831 conf.arb_dl_5g[i].gbr_threshold1 = VRB_QOS_GBR; 832 conf.arb_dl_5g[i].round_robin_weight = VRB_QMGR_RR; 833 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 834 conf.arb_ul_5g[i].gbr_threshold1 = VRB_QOS_GBR; 835 conf.arb_ul_5g[i].round_robin_weight = VRB_QMGR_RR; 836 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 837 conf.arb_fft[i].gbr_threshold1 = VRB_QOS_GBR; 838 conf.arb_fft[i].round_robin_weight = VRB_QMGR_RR; 839 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 840 conf.arb_mld[i].gbr_threshold1 = VRB_QOS_GBR; 841 conf.arb_mld[i].round_robin_weight = VRB_QMGR_RR; 842 } 843 844 conf.input_pos_llr_1_bit = true; 845 conf.output_pos_llr_1_bit = true; 846 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 847 848 conf.q_ul_4g.num_qgroups = VRB_QMGR_NUM_QGS; 849 conf.q_ul_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 850 conf.q_ul_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 851 conf.q_ul_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 852 conf.q_dl_4g.num_qgroups = VRB_QMGR_NUM_QGS; 853 conf.q_dl_4g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 854 conf.q_dl_4g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 855 conf.q_dl_4g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 856 conf.q_ul_5g.num_qgroups = VRB_QMGR_NUM_QGS; 857 conf.q_ul_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 858 conf.q_ul_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 859 conf.q_ul_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 860 conf.q_dl_5g.num_qgroups = VRB_QMGR_NUM_QGS; 861 conf.q_dl_5g.first_qgroup_index = VRB_QMGR_INVALID_IDX; 862 conf.q_dl_5g.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 863 conf.q_dl_5g.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 864 conf.q_fft.num_qgroups = VRB_QMGR_NUM_QGS; 865 conf.q_fft.first_qgroup_index = VRB_QMGR_INVALID_IDX; 866 conf.q_fft.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 867 conf.q_fft.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 868 conf.q_mld.num_qgroups = VRB_QMGR_NUM_QGS; 869 conf.q_mld.first_qgroup_index = VRB_QMGR_INVALID_IDX; 870 conf.q_mld.num_aqs_per_groups = VRB_QMGR_NUM_AQS; 871 conf.q_mld.aq_depth_log2 = VRB_QMGR_AQ_DEPTH; 872 873 /* setup PF with configuration information */ 874 ret = rte_acc_configure(info->dev_name, &conf); 875 TEST_ASSERT_SUCCESS(ret, 876 "Failed to configure PF for bbdev %s", 877 info->dev_name); 878 } 879 #endif 880 /* Let's refresh this now this is configured */ 881 rte_bbdev_info_get(dev_id, info); 882 if (info->drv.device_status == RTE_BBDEV_DEV_FATAL_ERR) 883 printf("Device Status %s\n", rte_bbdev_device_status_str(info->drv.device_status)); 884 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 885 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 886 887 /* setup device */ 888 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 889 if (ret < 0) { 890 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 891 dev_id, nb_queues, info->socket_id, ret); 892 return TEST_FAILED; 893 } 894 895 /* configure interrupts if needed */ 896 if (intr_enabled) { 897 ret = rte_bbdev_intr_enable(dev_id); 898 if (ret < 0) { 899 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 900 ret); 901 return TEST_FAILED; 902 } 903 } 904 905 /* setup device queues */ 906 qconf.socket = info->socket_id; 907 qconf.queue_size = info->drv.default_queue_conf.queue_size; 908 qconf.priority = 0; 909 qconf.deferred_start = 0; 910 qconf.op_type = op_type; 911 912 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 913 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 914 if (ret != 0) { 915 printf( 916 "Allocated all queues (id=%u) at prio%u on dev%u\n", 917 queue_id, qconf.priority, dev_id); 918 qconf.priority++; 919 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, &qconf); 920 } 921 if (ret != 0) { 922 printf("All queues on dev %u allocated: %u\n", dev_id, queue_id); 923 break; 924 } 925 ret = rte_bbdev_queue_start(ad->dev_id, queue_id); 926 if (ret != 0) { 927 printf("Failed to start queue on dev %u q_id: %u\n", dev_id, queue_id); 928 break; 929 } 930 ad->queue_ids[queue_id] = queue_id; 931 } 932 TEST_ASSERT(queue_id != 0, 933 "ERROR Failed to configure any queues on dev %u\n" 934 "\tthe device may not support the related operation capability\n" 935 "\tor the device may not have been configured yet", dev_id); 936 ad->nb_queues = queue_id; 937 938 set_avail_op(ad, op_type); 939 940 return TEST_SUCCESS; 941 } 942 943 static int 944 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 945 struct test_bbdev_vector *vector) 946 { 947 int ret; 948 949 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 950 active_devs[nb_active_devs].dev_id = dev_id; 951 952 ret = add_bbdev_dev(dev_id, info, vector); 953 if (ret == TEST_SUCCESS) 954 ++nb_active_devs; 955 return ret; 956 } 957 958 static uint8_t 959 populate_active_devices(void) 960 { 961 int ret; 962 uint8_t dev_id; 963 uint8_t nb_devs_added = 0; 964 struct rte_bbdev_info info; 965 966 RTE_BBDEV_FOREACH(dev_id) { 967 rte_bbdev_info_get(dev_id, &info); 968 969 if (check_dev_cap(&info)) { 970 printf( 971 "Device %d (%s) does not support specified capabilities\n", 972 dev_id, info.dev_name); 973 continue; 974 } 975 976 ret = add_active_device(dev_id, &info, &test_vector); 977 if (ret != 0) { 978 printf("Adding active bbdev %s skipped\n", 979 info.dev_name); 980 continue; 981 } 982 nb_devs_added++; 983 } 984 985 return nb_devs_added; 986 } 987 988 static int 989 read_test_vector(void) 990 { 991 int ret; 992 993 memset(&test_vector, 0, sizeof(test_vector)); 994 printf("Test vector file = %s\n", get_vector_filename()); 995 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 996 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 997 get_vector_filename()); 998 999 return TEST_SUCCESS; 1000 } 1001 1002 static int 1003 testsuite_setup(void) 1004 { 1005 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1006 1007 if (populate_active_devices() == 0) { 1008 printf("No suitable devices found!\n"); 1009 return TEST_SKIPPED; 1010 } 1011 1012 return TEST_SUCCESS; 1013 } 1014 1015 static int 1016 interrupt_testsuite_setup(void) 1017 { 1018 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 1019 1020 /* Enable interrupts */ 1021 intr_enabled = true; 1022 1023 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 1024 if (populate_active_devices() == 0 || 1025 test_vector.op_type == RTE_BBDEV_OP_NONE) { 1026 intr_enabled = false; 1027 printf("No suitable devices found!\n"); 1028 return TEST_SKIPPED; 1029 } 1030 1031 return TEST_SUCCESS; 1032 } 1033 1034 static void 1035 testsuite_teardown(void) 1036 { 1037 uint8_t dev_id; 1038 1039 /* Unconfigure devices */ 1040 RTE_BBDEV_FOREACH(dev_id) 1041 rte_bbdev_close(dev_id); 1042 1043 /* Clear active devices structs. */ 1044 memset(active_devs, 0, sizeof(active_devs)); 1045 nb_active_devs = 0; 1046 1047 /* Disable interrupts */ 1048 intr_enabled = false; 1049 } 1050 1051 static int 1052 ut_setup(void) 1053 { 1054 uint8_t i, dev_id; 1055 1056 for (i = 0; i < nb_active_devs; i++) { 1057 dev_id = active_devs[i].dev_id; 1058 /* reset bbdev stats */ 1059 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1060 "Failed to reset stats of bbdev %u", dev_id); 1061 /* start the device */ 1062 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1063 "Failed to start bbdev %u", dev_id); 1064 } 1065 1066 return TEST_SUCCESS; 1067 } 1068 1069 static void 1070 ut_teardown(void) 1071 { 1072 uint8_t i, dev_id, ret; 1073 struct rte_bbdev_stats stats; 1074 1075 for (i = 0; i < nb_active_devs; i++) { 1076 dev_id = active_devs[i].dev_id; 1077 /* read stats and print */ 1078 ret = rte_bbdev_stats_get(dev_id, &stats); 1079 if (ret != 0) 1080 printf("Failed to get stats on bbdev %u\n", dev_id); 1081 /* Stop the device */ 1082 rte_bbdev_stop(dev_id); 1083 } 1084 } 1085 1086 static int 1087 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1088 struct op_data_entries *ref_entries, 1089 struct rte_mempool *mbuf_pool, const uint16_t n, 1090 enum op_data_type op_type, uint16_t min_alignment) 1091 { 1092 int ret; 1093 unsigned int i, j; 1094 bool large_input = false; 1095 1096 for (i = 0; i < n; ++i) { 1097 char *data; 1098 struct op_data_buf *seg = &ref_entries->segments[0]; 1099 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1100 TEST_ASSERT_NOT_NULL(m_head, 1101 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1102 op_type, n * ref_entries->nb_segments, 1103 mbuf_pool->size); 1104 1105 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1106 /* 1107 * Special case when DPDK mbuf cannot handle 1108 * the required input size 1109 */ 1110 large_input = true; 1111 } 1112 bufs[i].data = m_head; 1113 bufs[i].offset = 0; 1114 bufs[i].length = 0; 1115 1116 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1117 if (large_input) { 1118 /* Allocate a fake overused mbuf */ 1119 data = rte_malloc(NULL, seg->length, 0); 1120 TEST_ASSERT_NOT_NULL(data, 1121 "rte malloc failed with %u bytes", 1122 seg->length); 1123 memcpy(data, seg->addr, seg->length); 1124 m_head->buf_addr = data; 1125 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1126 m_head->data_off = 0; 1127 m_head->data_len = seg->length; 1128 } else { 1129 data = rte_pktmbuf_append(m_head, seg->length); 1130 TEST_ASSERT_NOT_NULL(data, 1131 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1132 seg->length, op_type); 1133 1134 TEST_ASSERT(data == RTE_PTR_ALIGN( 1135 data, min_alignment), 1136 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1137 data, min_alignment); 1138 rte_memcpy(data, seg->addr, seg->length); 1139 } 1140 1141 bufs[i].length += seg->length; 1142 1143 for (j = 1; j < ref_entries->nb_segments; ++j) { 1144 struct rte_mbuf *m_tail = 1145 rte_pktmbuf_alloc(mbuf_pool); 1146 TEST_ASSERT_NOT_NULL(m_tail, 1147 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1148 op_type, 1149 n * ref_entries->nb_segments, 1150 mbuf_pool->size); 1151 seg += 1; 1152 1153 data = rte_pktmbuf_append(m_tail, seg->length); 1154 TEST_ASSERT_NOT_NULL(data, 1155 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1156 seg->length, op_type); 1157 1158 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1159 min_alignment), 1160 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1161 data, min_alignment); 1162 rte_memcpy(data, seg->addr, seg->length); 1163 bufs[i].length += seg->length; 1164 1165 ret = rte_pktmbuf_chain(m_head, m_tail); 1166 TEST_ASSERT_SUCCESS(ret, 1167 "Couldn't chain mbufs from %d data type mbuf pool", 1168 op_type); 1169 } 1170 } else { 1171 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1172 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1173 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1174 /* Allocate a fake overused mbuf + margin */ 1175 data = rte_malloc(NULL, seg->length + 1024, 0); 1176 TEST_ASSERT_NOT_NULL(data, 1177 "rte malloc failed with %u bytes", 1178 seg->length + 1024); 1179 m_head->buf_addr = data; 1180 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1181 m_head->data_off = 0; 1182 m_head->data_len = seg->length; 1183 } else { 1184 /* allocate chained-mbuf for output buffer */ 1185 for (j = 1; j < ref_entries->nb_segments; ++j) { 1186 struct rte_mbuf *m_tail = 1187 rte_pktmbuf_alloc(mbuf_pool); 1188 TEST_ASSERT_NOT_NULL(m_tail, 1189 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1190 op_type, 1191 n * ref_entries->nb_segments, 1192 mbuf_pool->size); 1193 1194 ret = rte_pktmbuf_chain(m_head, m_tail); 1195 TEST_ASSERT_SUCCESS(ret, 1196 "Couldn't chain mbufs from %d data type mbuf pool", 1197 op_type); 1198 } 1199 } 1200 bufs[i].length += seg->length; 1201 } 1202 } 1203 1204 return 0; 1205 } 1206 1207 static int 1208 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1209 const int socket) 1210 { 1211 int i; 1212 1213 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1214 if (*buffers == NULL) { 1215 printf("WARNING: Failed to allocate op_data on socket %d\n", 1216 socket); 1217 /* try to allocate memory on other detected sockets */ 1218 for (i = 0; i < socket; i++) { 1219 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1220 if (*buffers != NULL) 1221 break; 1222 } 1223 } 1224 1225 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1226 } 1227 1228 static void 1229 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1230 const uint16_t n, const int8_t max_llr_modulus) 1231 { 1232 uint16_t i, byte_idx; 1233 1234 for (i = 0; i < n; ++i) { 1235 struct rte_mbuf *m = input_ops[i].data; 1236 while (m != NULL) { 1237 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1238 input_ops[i].offset); 1239 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1240 ++byte_idx) 1241 llr[byte_idx] = round((double)max_llr_modulus * 1242 llr[byte_idx] / INT8_MAX); 1243 1244 m = m->next; 1245 } 1246 } 1247 } 1248 1249 /* 1250 * We may have to insert filler bits 1251 * when they are required by the HARQ assumption 1252 */ 1253 static void 1254 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1255 const uint16_t n, struct test_op_params *op_params) 1256 { 1257 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1258 1259 if (input_ops == NULL) 1260 return; 1261 /* No need to add filler if not required by device */ 1262 if (!(ldpc_cap_flags & 1263 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1264 return; 1265 /* No need to add filler for loopback operation */ 1266 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1267 return; 1268 1269 uint16_t i, j, parity_offset; 1270 for (i = 0; i < n; ++i) { 1271 struct rte_mbuf *m = input_ops[i].data; 1272 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1273 input_ops[i].offset); 1274 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1275 * dec.z_c - dec.n_filler; 1276 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1277 m->data_len = new_hin_size; 1278 input_ops[i].length = new_hin_size; 1279 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1280 j--) 1281 llr[j] = llr[j - dec.n_filler]; 1282 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1283 for (j = 0; j < dec.n_filler; j++) 1284 llr[parity_offset + j] = llr_max_pre_scaling; 1285 } 1286 } 1287 1288 static void 1289 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1290 const uint16_t n, const int8_t llr_size, 1291 const int8_t llr_decimals) 1292 { 1293 if (input_ops == NULL) 1294 return; 1295 1296 uint16_t i, byte_idx; 1297 1298 int16_t llr_max, llr_min, llr_tmp; 1299 llr_max = (1 << (llr_size - 1)) - 1; 1300 llr_min = -llr_max; 1301 for (i = 0; i < n; ++i) { 1302 struct rte_mbuf *m = input_ops[i].data; 1303 while (m != NULL) { 1304 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1305 input_ops[i].offset); 1306 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1307 ++byte_idx) { 1308 1309 llr_tmp = llr[byte_idx]; 1310 if (llr_decimals == 4) 1311 llr_tmp *= 8; 1312 else if (llr_decimals == 2) 1313 llr_tmp *= 2; 1314 else if (llr_decimals == 0) 1315 llr_tmp /= 2; 1316 llr_tmp = RTE_MIN(llr_max, 1317 RTE_MAX(llr_min, llr_tmp)); 1318 llr[byte_idx] = (int8_t) llr_tmp; 1319 } 1320 1321 m = m->next; 1322 } 1323 } 1324 } 1325 1326 1327 1328 static int 1329 fill_queue_buffers(struct test_op_params *op_params, 1330 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1331 struct rte_mempool *soft_out_mp, 1332 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1333 uint16_t queue_id, 1334 const struct rte_bbdev_op_cap *capabilities, 1335 uint16_t min_alignment, const int socket_id) 1336 { 1337 int ret; 1338 enum op_data_type type; 1339 const uint16_t n = op_params->num_to_process; 1340 1341 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1342 in_mp, 1343 soft_out_mp, 1344 hard_out_mp, 1345 harq_in_mp, 1346 harq_out_mp, 1347 }; 1348 1349 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1350 &op_params->q_bufs[socket_id][queue_id].inputs, 1351 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1352 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1353 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1354 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1355 }; 1356 1357 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1358 struct op_data_entries *ref_entries = 1359 &test_vector.entries[type]; 1360 if (ref_entries->nb_segments == 0) 1361 continue; 1362 1363 ret = allocate_buffers_on_socket(queue_ops[type], 1364 n * sizeof(struct rte_bbdev_op_data), 1365 socket_id); 1366 TEST_ASSERT_SUCCESS(ret, 1367 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1368 1369 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1370 mbuf_pools[type], n, type, min_alignment); 1371 TEST_ASSERT_SUCCESS(ret, 1372 "Couldn't init rte_bbdev_op_data structs"); 1373 } 1374 1375 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1376 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1377 capabilities->cap.turbo_dec.max_llr_modulus); 1378 1379 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1380 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1381 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1382 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1383 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1384 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1385 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1386 1387 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1388 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1389 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1390 if (!loopback && !llr_comp) 1391 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1392 ldpc_llr_size, ldpc_llr_decimals); 1393 if (!loopback && !harq_comp) 1394 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1395 ldpc_llr_size, ldpc_llr_decimals); 1396 if (!loopback) 1397 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1398 op_params); 1399 } 1400 1401 return 0; 1402 } 1403 1404 static void 1405 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1406 { 1407 unsigned int i, j; 1408 1409 rte_mempool_free(ad->ops_mempool); 1410 rte_mempool_free(ad->in_mbuf_pool); 1411 rte_mempool_free(ad->hard_out_mbuf_pool); 1412 rte_mempool_free(ad->soft_out_mbuf_pool); 1413 rte_mempool_free(ad->harq_in_mbuf_pool); 1414 rte_mempool_free(ad->harq_out_mbuf_pool); 1415 1416 for (i = 0; i < rte_lcore_count(); ++i) { 1417 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1418 rte_free(op_params->q_bufs[j][i].inputs); 1419 rte_free(op_params->q_bufs[j][i].hard_outputs); 1420 rte_free(op_params->q_bufs[j][i].soft_outputs); 1421 rte_free(op_params->q_bufs[j][i].harq_inputs); 1422 rte_free(op_params->q_bufs[j][i].harq_outputs); 1423 } 1424 } 1425 } 1426 1427 static void 1428 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1429 unsigned int start_idx, 1430 struct rte_bbdev_op_data *inputs, 1431 struct rte_bbdev_op_data *hard_outputs, 1432 struct rte_bbdev_op_data *soft_outputs, 1433 struct rte_bbdev_dec_op *ref_op) 1434 { 1435 unsigned int i; 1436 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1437 1438 for (i = 0; i < n; ++i) { 1439 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1440 ops[i]->turbo_dec.tb_params.ea = 1441 turbo_dec->tb_params.ea; 1442 ops[i]->turbo_dec.tb_params.eb = 1443 turbo_dec->tb_params.eb; 1444 ops[i]->turbo_dec.tb_params.k_pos = 1445 turbo_dec->tb_params.k_pos; 1446 ops[i]->turbo_dec.tb_params.k_neg = 1447 turbo_dec->tb_params.k_neg; 1448 ops[i]->turbo_dec.tb_params.c = 1449 turbo_dec->tb_params.c; 1450 ops[i]->turbo_dec.tb_params.c_neg = 1451 turbo_dec->tb_params.c_neg; 1452 ops[i]->turbo_dec.tb_params.cab = 1453 turbo_dec->tb_params.cab; 1454 ops[i]->turbo_dec.tb_params.r = 1455 turbo_dec->tb_params.r; 1456 } else { 1457 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1458 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1459 } 1460 1461 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1462 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1463 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1464 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1465 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1466 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1467 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1468 1469 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1470 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1471 if (soft_outputs != NULL) 1472 ops[i]->turbo_dec.soft_output = 1473 soft_outputs[start_idx + i]; 1474 } 1475 } 1476 1477 static void 1478 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1479 unsigned int start_idx, 1480 struct rte_bbdev_op_data *inputs, 1481 struct rte_bbdev_op_data *outputs, 1482 struct rte_bbdev_enc_op *ref_op) 1483 { 1484 unsigned int i; 1485 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1486 for (i = 0; i < n; ++i) { 1487 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1488 ops[i]->turbo_enc.tb_params.ea = 1489 turbo_enc->tb_params.ea; 1490 ops[i]->turbo_enc.tb_params.eb = 1491 turbo_enc->tb_params.eb; 1492 ops[i]->turbo_enc.tb_params.k_pos = 1493 turbo_enc->tb_params.k_pos; 1494 ops[i]->turbo_enc.tb_params.k_neg = 1495 turbo_enc->tb_params.k_neg; 1496 ops[i]->turbo_enc.tb_params.c = 1497 turbo_enc->tb_params.c; 1498 ops[i]->turbo_enc.tb_params.c_neg = 1499 turbo_enc->tb_params.c_neg; 1500 ops[i]->turbo_enc.tb_params.cab = 1501 turbo_enc->tb_params.cab; 1502 ops[i]->turbo_enc.tb_params.ncb_pos = 1503 turbo_enc->tb_params.ncb_pos; 1504 ops[i]->turbo_enc.tb_params.ncb_neg = 1505 turbo_enc->tb_params.ncb_neg; 1506 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1507 } else { 1508 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1509 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1510 ops[i]->turbo_enc.cb_params.ncb = 1511 turbo_enc->cb_params.ncb; 1512 } 1513 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1514 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1515 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1516 1517 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1518 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1519 } 1520 } 1521 1522 1523 /* Returns a random number drawn from a normal distribution 1524 * with mean of 0 and variance of 1 1525 * Marsaglia algorithm 1526 */ 1527 static double 1528 randn(int n) 1529 { 1530 double S, Z, U1, U2, u, v, fac; 1531 1532 do { 1533 U1 = (double)rand() / RAND_MAX; 1534 U2 = (double)rand() / RAND_MAX; 1535 u = 2. * U1 - 1.; 1536 v = 2. * U2 - 1.; 1537 S = u * u + v * v; 1538 } while (S >= 1 || S == 0); 1539 fac = sqrt(-2. * log(S) / S); 1540 Z = (n % 2) ? u * fac : v * fac; 1541 return Z; 1542 } 1543 1544 static inline double 1545 maxstar(double A, double B) 1546 { 1547 if (fabs(A - B) > 5) 1548 return RTE_MAX(A, B); 1549 else 1550 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1551 } 1552 1553 /* 1554 * Generate Qm LLRS for Qm==8 1555 * Modulation, AWGN and LLR estimation from max log development 1556 */ 1557 static void 1558 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1559 { 1560 int qm = 8; 1561 int qam = 256; 1562 int m, k; 1563 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1564 /* 5.1.4 of TS38.211 */ 1565 const double symbols_I[256] = { 1566 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1567 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1568 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1569 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1570 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1571 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1572 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1573 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1574 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1575 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1576 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1577 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1578 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1579 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1580 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1581 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1582 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1583 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1584 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1585 -13, -13, -15, -15, -13, -13, -15, -15}; 1586 const double symbols_Q[256] = { 1587 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1588 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1589 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1590 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1591 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1592 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1593 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1594 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1595 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1596 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1597 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1598 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1599 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1600 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1601 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1602 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1603 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1604 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1605 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1606 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1607 /* Average constellation point energy */ 1608 N0 *= 170.0; 1609 for (k = 0; k < qm; k++) 1610 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1611 /* 5.1.4 of TS38.211 */ 1612 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1613 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1614 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1615 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1616 /* AWGN channel */ 1617 I += sqrt(N0 / 2) * randn(0); 1618 Q += sqrt(N0 / 2) * randn(1); 1619 /* 1620 * Calculate the log of the probability that each of 1621 * the constellation points was transmitted 1622 */ 1623 for (m = 0; m < qam; m++) 1624 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1625 + pow(Q - symbols_Q[m], 2.0)) / N0; 1626 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1627 for (k = 0; k < qm; k++) { 1628 p0 = -999999; 1629 p1 = -999999; 1630 /* For each constellation point */ 1631 for (m = 0; m < qam; m++) { 1632 if ((m >> (qm - k - 1)) & 1) 1633 p1 = maxstar(p1, log_syml_prob[m]); 1634 else 1635 p0 = maxstar(p0, log_syml_prob[m]); 1636 } 1637 /* Calculate the LLR */ 1638 llr_ = p0 - p1; 1639 llr_ *= (1 << ldpc_llr_decimals); 1640 llr_ = round(llr_); 1641 if (llr_ > llr_max) 1642 llr_ = llr_max; 1643 if (llr_ < -llr_max) 1644 llr_ = -llr_max; 1645 llrs[qm * i + k] = (int8_t) llr_; 1646 } 1647 } 1648 1649 1650 /* 1651 * Generate Qm LLRS for Qm==6 1652 * Modulation, AWGN and LLR estimation from max log development 1653 */ 1654 static void 1655 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1656 { 1657 int qm = 6; 1658 int qam = 64; 1659 int m, k; 1660 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1661 /* 5.1.4 of TS38.211 */ 1662 const double symbols_I[64] = { 1663 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1664 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1665 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1666 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1667 -5, -5, -7, -7, -5, -5, -7, -7}; 1668 const double symbols_Q[64] = { 1669 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1670 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1671 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1672 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1673 -3, -1, -3, -1, -5, -7, -5, -7}; 1674 /* Average constellation point energy */ 1675 N0 *= 42.0; 1676 for (k = 0; k < qm; k++) 1677 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1678 /* 5.1.4 of TS38.211 */ 1679 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1680 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1681 /* AWGN channel */ 1682 I += sqrt(N0 / 2) * randn(0); 1683 Q += sqrt(N0 / 2) * randn(1); 1684 /* 1685 * Calculate the log of the probability that each of 1686 * the constellation points was transmitted 1687 */ 1688 for (m = 0; m < qam; m++) 1689 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1690 + pow(Q - symbols_Q[m], 2.0)) / N0; 1691 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1692 for (k = 0; k < qm; k++) { 1693 p0 = -999999; 1694 p1 = -999999; 1695 /* For each constellation point */ 1696 for (m = 0; m < qam; m++) { 1697 if ((m >> (qm - k - 1)) & 1) 1698 p1 = maxstar(p1, log_syml_prob[m]); 1699 else 1700 p0 = maxstar(p0, log_syml_prob[m]); 1701 } 1702 /* Calculate the LLR */ 1703 llr_ = p0 - p1; 1704 llr_ *= (1 << ldpc_llr_decimals); 1705 llr_ = round(llr_); 1706 if (llr_ > llr_max) 1707 llr_ = llr_max; 1708 if (llr_ < -llr_max) 1709 llr_ = -llr_max; 1710 llrs[qm * i + k] = (int8_t) llr_; 1711 } 1712 } 1713 1714 /* 1715 * Generate Qm LLRS for Qm==4 1716 * Modulation, AWGN and LLR estimation from max log development 1717 */ 1718 static void 1719 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1720 { 1721 int qm = 4; 1722 int qam = 16; 1723 int m, k; 1724 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1725 /* 5.1.4 of TS38.211 */ 1726 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1727 -1, -1, -3, -3, -1, -1, -3, -3}; 1728 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1729 1, 3, 1, 3, -1, -3, -1, -3}; 1730 /* Average constellation point energy */ 1731 N0 *= 10.0; 1732 for (k = 0; k < qm; k++) 1733 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1734 /* 5.1.4 of TS38.211 */ 1735 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1736 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1737 /* AWGN channel */ 1738 I += sqrt(N0 / 2) * randn(0); 1739 Q += sqrt(N0 / 2) * randn(1); 1740 /* 1741 * Calculate the log of the probability that each of 1742 * the constellation points was transmitted 1743 */ 1744 for (m = 0; m < qam; m++) 1745 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1746 + pow(Q - symbols_Q[m], 2.0)) / N0; 1747 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1748 for (k = 0; k < qm; k++) { 1749 p0 = -999999; 1750 p1 = -999999; 1751 /* For each constellation point */ 1752 for (m = 0; m < qam; m++) { 1753 if ((m >> (qm - k - 1)) & 1) 1754 p1 = maxstar(p1, log_syml_prob[m]); 1755 else 1756 p0 = maxstar(p0, log_syml_prob[m]); 1757 } 1758 /* Calculate the LLR */ 1759 llr_ = p0 - p1; 1760 llr_ *= (1 << ldpc_llr_decimals); 1761 llr_ = round(llr_); 1762 if (llr_ > llr_max) 1763 llr_ = llr_max; 1764 if (llr_ < -llr_max) 1765 llr_ = -llr_max; 1766 llrs[qm * i + k] = (int8_t) llr_; 1767 } 1768 } 1769 1770 static void 1771 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1772 { 1773 double b, b1, n; 1774 double coeff = 2.0 * sqrt(N0); 1775 1776 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1777 if (llrs[j] < 8 && llrs[j] > -8) 1778 return; 1779 1780 /* Note don't change sign here */ 1781 n = randn(j % 2); 1782 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1783 + coeff * n) / N0; 1784 b = b1 * (1 << ldpc_llr_decimals); 1785 b = round(b); 1786 if (b > llr_max) 1787 b = llr_max; 1788 if (b < -llr_max) 1789 b = -llr_max; 1790 llrs[j] = (int8_t) b; 1791 } 1792 1793 /* Simple LLR generation assuming AWGN and QPSK */ 1794 static void 1795 gen_turbo_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1796 { 1797 double b, b1, n; 1798 double coeff = 2.0 * sqrt(N0); 1799 1800 /* Ignore in vectors null LLRs not to be saturated */ 1801 if (llrs[j] == 0) 1802 return; 1803 1804 /* Note don't change sign here */ 1805 n = randn(j % 2); 1806 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1807 + coeff * n) / N0; 1808 b = b1 * (1 << 4); 1809 b = round(b); 1810 if (b > llr_max) 1811 b = llr_max; 1812 if (b < -llr_max) 1813 b = -llr_max; 1814 llrs[j] = (int8_t) b; 1815 } 1816 1817 /* Generate LLR for a given SNR */ 1818 static void 1819 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1820 struct rte_bbdev_dec_op *ref_op) 1821 { 1822 struct rte_mbuf *m; 1823 uint16_t qm; 1824 uint32_t i, j, e, range; 1825 double N0, llr_max; 1826 1827 e = ref_op->ldpc_dec.cb_params.e; 1828 qm = ref_op->ldpc_dec.q_m; 1829 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1830 range = e / qm; 1831 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1832 1833 for (i = 0; i < n; ++i) { 1834 m = inputs[i].data; 1835 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1836 if (qm == 8) { 1837 for (j = 0; j < range; ++j) 1838 gen_qm8_llr(llrs, j, N0, llr_max); 1839 } else if (qm == 6) { 1840 for (j = 0; j < range; ++j) 1841 gen_qm6_llr(llrs, j, N0, llr_max); 1842 } else if (qm == 4) { 1843 for (j = 0; j < range; ++j) 1844 gen_qm4_llr(llrs, j, N0, llr_max); 1845 } else { 1846 for (j = 0; j < e; ++j) 1847 gen_qm2_llr(llrs, j, N0, llr_max); 1848 } 1849 } 1850 } 1851 1852 /* Generate LLR for turbo decoder for a given SNR */ 1853 static void 1854 generate_turbo_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1855 struct rte_bbdev_dec_op *ref_op) 1856 { 1857 struct rte_mbuf *m; 1858 uint32_t i, j, range; 1859 double N0, llr_max; 1860 1861 llr_max = 127; 1862 range = ref_op->turbo_dec.input.length; 1863 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1864 1865 if (range > inputs[0].data->data_len) { 1866 printf("Warning: Limiting LLR generation to first segment (%d from %d)\n", 1867 inputs[0].data->data_len, range); 1868 range = inputs[0].data->data_len; 1869 } 1870 1871 for (i = 0; i < n; ++i) { 1872 m = inputs[i].data; 1873 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1874 for (j = 0; j < range; ++j) 1875 gen_turbo_llr(llrs, j, N0, llr_max); 1876 } 1877 } 1878 1879 static void 1880 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1881 unsigned int start_idx, 1882 struct rte_bbdev_op_data *inputs, 1883 struct rte_bbdev_op_data *hard_outputs, 1884 struct rte_bbdev_op_data *soft_outputs, 1885 struct rte_bbdev_op_data *harq_inputs, 1886 struct rte_bbdev_op_data *harq_outputs, 1887 struct rte_bbdev_dec_op *ref_op) 1888 { 1889 unsigned int i; 1890 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1891 1892 for (i = 0; i < n; ++i) { 1893 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1894 ops[i]->ldpc_dec.tb_params.ea = 1895 ldpc_dec->tb_params.ea; 1896 ops[i]->ldpc_dec.tb_params.eb = 1897 ldpc_dec->tb_params.eb; 1898 ops[i]->ldpc_dec.tb_params.c = 1899 ldpc_dec->tb_params.c; 1900 ops[i]->ldpc_dec.tb_params.cab = 1901 ldpc_dec->tb_params.cab; 1902 ops[i]->ldpc_dec.tb_params.r = 1903 ldpc_dec->tb_params.r; 1904 } else { 1905 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1906 } 1907 1908 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1909 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1910 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1911 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1912 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1913 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1914 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1915 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1916 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1917 1918 if (hard_outputs != NULL) 1919 ops[i]->ldpc_dec.hard_output = 1920 hard_outputs[start_idx + i]; 1921 if (inputs != NULL) 1922 ops[i]->ldpc_dec.input = 1923 inputs[start_idx + i]; 1924 if (soft_outputs != NULL) 1925 ops[i]->ldpc_dec.soft_output = 1926 soft_outputs[start_idx + i]; 1927 if (harq_inputs != NULL) 1928 ops[i]->ldpc_dec.harq_combined_input = 1929 harq_inputs[start_idx + i]; 1930 if (harq_outputs != NULL) 1931 ops[i]->ldpc_dec.harq_combined_output = 1932 harq_outputs[start_idx + i]; 1933 } 1934 } 1935 1936 1937 static void 1938 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1939 unsigned int start_idx, 1940 struct rte_bbdev_op_data *inputs, 1941 struct rte_bbdev_op_data *outputs, 1942 struct rte_bbdev_enc_op *ref_op) 1943 { 1944 unsigned int i; 1945 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1946 for (i = 0; i < n; ++i) { 1947 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1948 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1949 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1950 ops[i]->ldpc_enc.tb_params.cab = 1951 ldpc_enc->tb_params.cab; 1952 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1953 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1954 } else { 1955 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1956 } 1957 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1958 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1959 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1960 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1961 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1962 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1963 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1964 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1965 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1966 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1967 } 1968 } 1969 1970 static void 1971 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1972 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1973 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1974 struct rte_bbdev_op_data *win_inputs, struct rte_bbdev_fft_op *ref_op) 1975 { 1976 unsigned int i, j; 1977 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1978 for (i = 0; i < n; i++) { 1979 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1980 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1981 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1982 ops[i]->fft.output_leading_depadding = 1983 fft->output_leading_depadding; 1984 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1985 ops[i]->fft.window_index[j] = fft->window_index[j]; 1986 for (j = 0; j < RTE_BBDEV_MAX_CS; j++) { 1987 ops[i]->fft.cs_theta_0[j] = fft->cs_theta_0[j]; 1988 ops[i]->fft.cs_theta_d[j] = fft->cs_theta_d[j]; 1989 ops[i]->fft.time_offset[j] = fft->time_offset[j]; 1990 } 1991 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 1992 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 1993 ops[i]->fft.idft_log2 = fft->idft_log2; 1994 ops[i]->fft.dft_log2 = fft->dft_log2; 1995 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 1996 ops[i]->fft.idft_shift = fft->idft_shift; 1997 ops[i]->fft.dft_shift = fft->dft_shift; 1998 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 1999 ops[i]->fft.power_shift = fft->power_shift; 2000 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 2001 ops[i]->fft.output_depadded_size = fft->output_depadded_size; 2002 ops[i]->fft.freq_resample_mode = fft->freq_resample_mode; 2003 ops[i]->fft.base_output = outputs[start_idx + i]; 2004 ops[i]->fft.base_input = inputs[start_idx + i]; 2005 if (win_inputs != NULL) 2006 ops[i]->fft.dewindowing_input = win_inputs[start_idx + i]; 2007 if (pwrouts != NULL) 2008 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 2009 ops[i]->fft.op_flags = fft->op_flags; 2010 } 2011 } 2012 2013 static void 2014 copy_reference_mldts_op(struct rte_bbdev_mldts_op **ops, unsigned int n, 2015 unsigned int start_idx, 2016 struct rte_bbdev_op_data *q_inputs, 2017 struct rte_bbdev_op_data *r_inputs, 2018 struct rte_bbdev_op_data *outputs, 2019 struct rte_bbdev_mldts_op *ref_op) 2020 { 2021 unsigned int i, j; 2022 struct rte_bbdev_op_mldts *mldts = &ref_op->mldts; 2023 for (i = 0; i < n; i++) { 2024 ops[i]->mldts.c_rep = mldts->c_rep; 2025 ops[i]->mldts.num_layers = mldts->num_layers; 2026 ops[i]->mldts.num_rbs = mldts->num_rbs; 2027 ops[i]->mldts.op_flags = mldts->op_flags; 2028 for (j = 0; j < RTE_BBDEV_MAX_MLD_LAYERS; j++) 2029 ops[i]->mldts.q_m[j] = mldts->q_m[j]; 2030 ops[i]->mldts.r_rep = mldts->r_rep; 2031 ops[i]->mldts.c_rep = mldts->c_rep; 2032 ops[i]->mldts.r_input = r_inputs[start_idx + i]; 2033 ops[i]->mldts.qhy_input = q_inputs[start_idx + i]; 2034 ops[i]->mldts.output = outputs[start_idx + i]; 2035 } 2036 } 2037 2038 static int 2039 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 2040 unsigned int order_idx, const int expected_status) 2041 { 2042 int status = op->status; 2043 /* ignore parity mismatch false alarms for long iterations */ 2044 if (get_iter_max() >= 10) { 2045 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2046 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2047 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2048 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 2049 } 2050 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 2051 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 2052 printf("WARNING: Ignore Syndrome Check mismatch\n"); 2053 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 2054 } 2055 } 2056 2057 TEST_ASSERT(status == expected_status, 2058 "op_status (%d) != expected_status (%d)", 2059 op->status, expected_status); 2060 2061 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2062 "Ordering error, expected %p, got %p", 2063 (void *)(uintptr_t)order_idx, op->opaque_data); 2064 2065 return TEST_SUCCESS; 2066 } 2067 2068 static int 2069 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 2070 unsigned int order_idx, const int expected_status) 2071 { 2072 TEST_ASSERT(op->status == expected_status, 2073 "op_status (%d) != expected_status (%d)", 2074 op->status, expected_status); 2075 2076 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2077 "Ordering error, expected %p, got %p", 2078 (void *)(uintptr_t)order_idx, op->opaque_data); 2079 2080 return TEST_SUCCESS; 2081 } 2082 2083 static int 2084 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 2085 unsigned int order_idx, const int expected_status) 2086 { 2087 TEST_ASSERT(op->status == expected_status, 2088 "op_status (%d) != expected_status (%d)", 2089 op->status, expected_status); 2090 2091 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2092 "Ordering error, expected %p, got %p", 2093 (void *)(uintptr_t)order_idx, op->opaque_data); 2094 2095 return TEST_SUCCESS; 2096 } 2097 2098 static int 2099 check_mldts_status_and_ordering(struct rte_bbdev_mldts_op *op, 2100 unsigned int order_idx, const int expected_status) 2101 { 2102 TEST_ASSERT(op->status == expected_status, 2103 "op_status (%d) != expected_status (%d)", 2104 op->status, expected_status); 2105 2106 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 2107 "Ordering error, expected %p, got %p", 2108 (void *)(uintptr_t)order_idx, op->opaque_data); 2109 2110 return TEST_SUCCESS; 2111 } 2112 2113 static inline int 2114 validate_op_chain(struct rte_bbdev_op_data *op, 2115 struct op_data_entries *orig_op) 2116 { 2117 uint8_t i; 2118 struct rte_mbuf *m = op->data; 2119 uint8_t nb_dst_segments = orig_op->nb_segments; 2120 uint32_t total_data_size = 0; 2121 bool ignore_mbuf = false; /* ignore mbuf limitations */ 2122 2123 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2124 "Number of segments differ in original (%u) and filled (%u) op", 2125 nb_dst_segments, m->nb_segs); 2126 2127 /* Validate each mbuf segment length */ 2128 for (i = 0; i < nb_dst_segments; ++i) { 2129 /* Apply offset to the first mbuf segment */ 2130 uint16_t offset = (i == 0) ? op->offset : 0; 2131 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2132 total_data_size += orig_op->segments[i].length; 2133 2134 if (orig_op->segments[i].length > RTE_BBDEV_LDPC_E_MAX_MBUF) 2135 ignore_mbuf = true; 2136 if (!ignore_mbuf) 2137 TEST_ASSERT(orig_op->segments[i].length == data_len, 2138 "Length of segment differ in original (%u) and filled (%u) op", 2139 orig_op->segments[i].length, data_len); 2140 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2141 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2142 orig_op->segments[i].length, 2143 "Output buffers (CB=%u) are not equal", i); 2144 m = m->next; 2145 } 2146 2147 /* Validate total mbuf pkt length */ 2148 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2149 if (!ignore_mbuf) 2150 TEST_ASSERT(total_data_size == pkt_len, 2151 "Length of data differ in original (%u) and filled (%u) op", 2152 total_data_size, pkt_len); 2153 2154 return TEST_SUCCESS; 2155 } 2156 2157 /* 2158 * Compute K0 for a given configuration for HARQ output length computation 2159 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2160 */ 2161 static inline uint16_t 2162 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 2163 { 2164 if (rv_index == 0) 2165 return 0; 2166 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2167 if (n_cb == n) { 2168 if (rv_index == 1) 2169 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2170 else if (rv_index == 2) 2171 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2172 else 2173 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2174 } 2175 /* LBRM case - includes a division by N */ 2176 if (rv_index == 1) 2177 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2178 / n) * z_c; 2179 else if (rv_index == 2) 2180 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2181 / n) * z_c; 2182 else 2183 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2184 / n) * z_c; 2185 } 2186 2187 /* HARQ output length including the Filler bits */ 2188 static inline uint16_t 2189 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2190 { 2191 uint16_t k0 = 0; 2192 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2193 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 2194 /* Compute RM out size and number of rows */ 2195 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2196 * ops_ld->z_c - ops_ld->n_filler; 2197 uint16_t deRmOutSize = RTE_MIN( 2198 k0 + ops_ld->cb_params.e + 2199 ((k0 > parity_offset) ? 2200 0 : ops_ld->n_filler), 2201 ops_ld->n_cb); 2202 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2203 / ops_ld->z_c); 2204 uint16_t harq_output_len = numRows * ops_ld->z_c; 2205 return harq_output_len; 2206 } 2207 2208 static inline int 2209 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2210 struct op_data_entries *orig_op, 2211 struct rte_bbdev_op_ldpc_dec *ops_ld) 2212 { 2213 uint8_t i; 2214 uint32_t j, jj, k; 2215 struct rte_mbuf *m = op->data; 2216 uint8_t nb_dst_segments = orig_op->nb_segments; 2217 uint32_t total_data_size = 0; 2218 int8_t *harq_orig, *harq_out, abs_harq_origin; 2219 uint32_t byte_error = 0, cum_error = 0, error; 2220 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2221 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2222 uint16_t parity_offset; 2223 2224 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2225 "Number of segments differ in original (%u) and filled (%u) op", 2226 nb_dst_segments, m->nb_segs); 2227 2228 /* Validate each mbuf segment length */ 2229 for (i = 0; i < nb_dst_segments; ++i) { 2230 /* Apply offset to the first mbuf segment */ 2231 uint16_t offset = (i == 0) ? op->offset : 0; 2232 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2233 total_data_size += orig_op->segments[i].length; 2234 2235 TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), 2236 "Length of segment differ in original (%u) and filled (%u) op", 2237 orig_op->segments[i].length, data_len); 2238 harq_orig = (int8_t *) orig_op->segments[i].addr; 2239 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2240 2241 /* Cannot compare HARQ output data for such cases */ 2242 if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) 2243 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION) 2244 || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_4BIT_COMPRESSION))) 2245 break; 2246 2247 if (!(ldpc_cap_flags & 2248 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2249 ) || (ops_ld->op_flags & 2250 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2251 data_len -= ops_ld->z_c; 2252 parity_offset = data_len; 2253 } else { 2254 /* Compute RM out size and number of rows */ 2255 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2256 * ops_ld->z_c - ops_ld->n_filler; 2257 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2258 ops_ld->n_filler; 2259 if (data_len > deRmOutSize) 2260 data_len = deRmOutSize; 2261 } 2262 if (data_len > orig_op->segments[i].length) 2263 data_len = orig_op->segments[i].length; 2264 /* 2265 * HARQ output can have minor differences 2266 * due to integer representation and related scaling 2267 */ 2268 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2269 if (j == parity_offset) { 2270 /* Special Handling of the filler bits */ 2271 for (k = 0; k < ops_ld->n_filler; k++) { 2272 if (harq_out[jj] != 2273 llr_max_pre_scaling) { 2274 printf("HARQ Filler issue %d: %d %d\n", 2275 jj, harq_out[jj], 2276 llr_max); 2277 byte_error++; 2278 } 2279 jj++; 2280 } 2281 } 2282 if (!(ops_ld->op_flags & 2283 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2284 if (ldpc_llr_decimals > 1) 2285 harq_out[jj] = (harq_out[jj] + 1) 2286 >> (ldpc_llr_decimals - 1); 2287 /* Saturated to S7 */ 2288 if (harq_orig[j] > llr_max) 2289 harq_orig[j] = llr_max; 2290 if (harq_orig[j] < -llr_max) 2291 harq_orig[j] = -llr_max; 2292 } 2293 if (harq_orig[j] != harq_out[jj]) { 2294 error = (harq_orig[j] > harq_out[jj]) ? 2295 harq_orig[j] - harq_out[jj] : 2296 harq_out[jj] - harq_orig[j]; 2297 abs_harq_origin = harq_orig[j] > 0 ? 2298 harq_orig[j] : 2299 -harq_orig[j]; 2300 /* Residual quantization error */ 2301 if ((error > 8 && (abs_harq_origin < 2302 (llr_max - 16))) || 2303 (error > 16)) { 2304 /* 2305 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2306 j, harq_orig[j], 2307 harq_out[jj], error); 2308 */ 2309 byte_error++; 2310 cum_error += error; 2311 } 2312 } 2313 } 2314 m = m->next; 2315 } 2316 2317 if (byte_error) 2318 TEST_ASSERT(byte_error <= 1, 2319 "HARQ output mismatch (%d) %d", 2320 byte_error, cum_error); 2321 2322 /* Validate total mbuf pkt length */ 2323 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2324 TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, 2325 "Length of data differ in original (%u) and filled (%u) op", 2326 total_data_size, pkt_len); 2327 2328 return TEST_SUCCESS; 2329 } 2330 2331 2332 static inline int 2333 validate_op_so_chain(struct rte_bbdev_op_data *op, 2334 struct op_data_entries *orig_op) 2335 { 2336 struct rte_mbuf *m = op->data; 2337 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2338 uint32_t j, jj; 2339 int8_t *so_orig, *so_out; 2340 uint32_t byte_error = 0, error, margin_error = 0; 2341 2342 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2343 "Number of segments differ in original (%u) and filled (%u) op", 2344 nb_dst_segments, m->nb_segs); 2345 2346 /* Validate each mbuf segment length. */ 2347 for (i = 0; i < nb_dst_segments; ++i) { 2348 /* Apply offset to the first mbuf segment. */ 2349 uint16_t offset = (i == 0) ? op->offset : 0; 2350 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2351 2352 TEST_ASSERT(orig_op->segments[i].length == data_len, 2353 "Length of segment differ in original (%u) and filled (%u) op", 2354 orig_op->segments[i].length, data_len); 2355 so_orig = (int8_t *) orig_op->segments[i].addr; 2356 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2357 margin_error += data_len / 8; /* Allow for few % errors. */ 2358 2359 /* SO output can have minor differences due to algorithm variations. */ 2360 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2361 if (so_orig[j] != so_out[jj]) { 2362 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2363 so_out[jj] - so_orig[j]; 2364 /* Residual quantization error. */ 2365 if (error > 32) { 2366 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2367 j, so_orig[j], so_out[jj], error); 2368 byte_error++; 2369 } 2370 } 2371 } 2372 m = m->next; 2373 } 2374 2375 if (byte_error > margin_error) 2376 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2377 byte_error, margin_error); 2378 2379 return TEST_SUCCESS; 2380 } 2381 2382 static int 2383 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2384 struct rte_bbdev_dec_op *ref_op) 2385 { 2386 unsigned int i; 2387 int ret; 2388 struct op_data_entries *hard_data_orig = 2389 &test_vector.entries[DATA_HARD_OUTPUT]; 2390 struct op_data_entries *soft_data_orig = 2391 &test_vector.entries[DATA_SOFT_OUTPUT]; 2392 struct rte_bbdev_op_turbo_dec *ops_td; 2393 struct rte_bbdev_op_data *hard_output; 2394 struct rte_bbdev_op_data *soft_output; 2395 2396 for (i = 0; i < n; ++i) { 2397 ops_td = &ops[i]->turbo_dec; 2398 hard_output = &ops_td->hard_output; 2399 soft_output = &ops_td->soft_output; 2400 2401 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2402 TEST_ASSERT_SUCCESS(ret, 2403 "Checking status and ordering for decoder failed"); 2404 2405 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2406 hard_data_orig), 2407 "Hard output buffers (CB=%u) are not equal", 2408 i); 2409 2410 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2411 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2412 soft_data_orig), 2413 "Soft output buffers (CB=%u) are not equal", 2414 i); 2415 } 2416 2417 return TEST_SUCCESS; 2418 } 2419 2420 /* Check Number of code blocks errors */ 2421 static int 2422 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2423 { 2424 unsigned int i; 2425 struct op_data_entries *hard_data_orig = 2426 &test_vector.entries[DATA_HARD_OUTPUT]; 2427 struct rte_bbdev_op_ldpc_dec *ops_td; 2428 struct rte_bbdev_op_data *hard_output; 2429 int errors = 0; 2430 struct rte_mbuf *m; 2431 2432 for (i = 0; i < n; ++i) { 2433 ops_td = &ops[i]->ldpc_dec; 2434 hard_output = &ops_td->hard_output; 2435 m = hard_output->data; 2436 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2437 hard_data_orig->segments[0].addr, 2438 hard_data_orig->segments[0].length)) 2439 errors++; 2440 } 2441 return errors; 2442 } 2443 2444 /* Check Number of code blocks errors */ 2445 static int 2446 validate_turbo_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2447 { 2448 unsigned int i; 2449 struct op_data_entries *hard_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2450 struct rte_bbdev_op_turbo_dec *ops_td; 2451 struct rte_bbdev_op_data *hard_output; 2452 int errors = 0; 2453 struct rte_mbuf *m; 2454 2455 for (i = 0; i < n; ++i) { 2456 ops_td = &ops[i]->turbo_dec; 2457 hard_output = &ops_td->hard_output; 2458 m = hard_output->data; 2459 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2460 hard_data_orig->segments[0].addr, 2461 hard_data_orig->segments[0].length)) 2462 errors++; 2463 } 2464 return errors; 2465 } 2466 2467 2468 static int 2469 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2470 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2471 { 2472 unsigned int i; 2473 int ret; 2474 struct op_data_entries *hard_data_orig = 2475 &test_vector.entries[DATA_HARD_OUTPUT]; 2476 struct op_data_entries *soft_data_orig = 2477 &test_vector.entries[DATA_SOFT_OUTPUT]; 2478 struct op_data_entries *harq_data_orig = 2479 &test_vector.entries[DATA_HARQ_OUTPUT]; 2480 struct rte_bbdev_op_ldpc_dec *ops_td; 2481 struct rte_bbdev_op_data *hard_output; 2482 struct rte_bbdev_op_data *harq_output; 2483 struct rte_bbdev_op_data *soft_output; 2484 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2485 2486 for (i = 0; i < n; ++i) { 2487 ops_td = &ops[i]->ldpc_dec; 2488 hard_output = &ops_td->hard_output; 2489 harq_output = &ops_td->harq_combined_output; 2490 soft_output = &ops_td->soft_output; 2491 2492 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2493 TEST_ASSERT_SUCCESS(ret, 2494 "Checking status and ordering for decoder failed"); 2495 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2496 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2497 "Returned iter_count (%d) > expected iter_count (%d)", 2498 ops_td->iter_count, ref_td->iter_count); 2499 /* 2500 * We can ignore output data when the decoding failed to 2501 * converge or for loop-back cases 2502 */ 2503 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2504 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2505 ) && ( 2506 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2507 )) == 0) 2508 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2509 hard_data_orig), 2510 "Hard output buffers (CB=%u) are not equal", 2511 i); 2512 2513 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2514 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2515 soft_data_orig), 2516 "Soft output buffers (CB=%u) are not equal", 2517 i); 2518 if (ref_op->ldpc_dec.op_flags & 2519 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2520 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2521 harq_data_orig, ops_td), 2522 "HARQ output buffers (CB=%u) are not equal", 2523 i); 2524 } 2525 if (ref_op->ldpc_dec.op_flags & 2526 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2527 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2528 harq_data_orig, ops_td), 2529 "HARQ output buffers (CB=%u) are not equal", 2530 i); 2531 2532 } 2533 2534 return TEST_SUCCESS; 2535 } 2536 2537 2538 static int 2539 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2540 struct rte_bbdev_enc_op *ref_op) 2541 { 2542 unsigned int i; 2543 int ret; 2544 struct op_data_entries *hard_data_orig = 2545 &test_vector.entries[DATA_HARD_OUTPUT]; 2546 2547 for (i = 0; i < n; ++i) { 2548 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2549 TEST_ASSERT_SUCCESS(ret, 2550 "Checking status and ordering for encoder failed"); 2551 TEST_ASSERT_SUCCESS(validate_op_chain( 2552 &ops[i]->turbo_enc.output, 2553 hard_data_orig), 2554 "Output buffers (CB=%u) are not equal", 2555 i); 2556 } 2557 2558 return TEST_SUCCESS; 2559 } 2560 2561 static int 2562 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2563 struct rte_bbdev_enc_op *ref_op) 2564 { 2565 unsigned int i; 2566 int ret; 2567 struct op_data_entries *hard_data_orig = 2568 &test_vector.entries[DATA_HARD_OUTPUT]; 2569 2570 for (i = 0; i < n; ++i) { 2571 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2572 TEST_ASSERT_SUCCESS(ret, 2573 "Checking status and ordering for encoder failed"); 2574 TEST_ASSERT_SUCCESS(validate_op_chain( 2575 &ops[i]->ldpc_enc.output, 2576 hard_data_orig), 2577 "Output buffers (CB=%u) are not equal", 2578 i); 2579 } 2580 2581 return TEST_SUCCESS; 2582 } 2583 2584 static inline int 2585 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op) 2586 { 2587 struct rte_mbuf *m = op->data; 2588 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2589 int16_t delt, abs_delt, thres_hold = 4; 2590 uint32_t j, data_len_iq, error_num; 2591 int16_t *ref_out, *op_out; 2592 2593 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2594 "Number of segments differ in original (%u) and filled (%u) op fft", 2595 nb_dst_segments, m->nb_segs); 2596 2597 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2598 for (i = 0; i < nb_dst_segments; ++i) { 2599 uint16_t offset = (i == 0) ? op->offset : 0; 2600 uint32_t data_len = op->length; 2601 2602 TEST_ASSERT(orig_op->segments[i].length == data_len, 2603 "Length of segment differ in original (%u) and filled (%u) op fft", 2604 orig_op->segments[i].length, data_len); 2605 /* Divided by 2 to get the number of 16bits data. */ 2606 data_len_iq = data_len >> 1; 2607 ref_out = (int16_t *)(orig_op->segments[i].addr); 2608 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2609 error_num = 0; 2610 for (j = 0; j < data_len_iq; j++) { 2611 delt = ref_out[j] - op_out[j]; 2612 abs_delt = delt > 0 ? delt : -delt; 2613 error_num += (abs_delt > thres_hold ? 1 : 0); 2614 } 2615 if (error_num > 0) { 2616 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2617 rte_memdump(stdout, "Buffer B", op_out, data_len); 2618 TEST_ASSERT(error_num == 0, 2619 "FFT Output are not matched total (%u) errors (%u)", 2620 data_len_iq, error_num); 2621 } 2622 2623 m = m->next; 2624 } 2625 2626 return TEST_SUCCESS; 2627 } 2628 2629 static inline int 2630 validate_op_mldts_chain(struct rte_bbdev_op_data *op, 2631 struct op_data_entries *orig_op) 2632 { 2633 uint8_t i; 2634 struct rte_mbuf *m = op->data; 2635 uint8_t nb_dst_segments = orig_op->nb_segments; 2636 /*the result is not bit exact*/ 2637 int16_t thres_hold = 3; 2638 int16_t delt, abs_delt; 2639 uint32_t j, data_len_iq; 2640 uint32_t error_num; 2641 int8_t *ref_out; 2642 int8_t *op_out; 2643 2644 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2645 "Number of segments differ in original (%u) and filled (%u) op mldts", 2646 nb_dst_segments, m->nb_segs); 2647 2648 /* Due to size limitation of mbuf, MLDTS doesn't use real mbuf. */ 2649 for (i = 0; i < nb_dst_segments; ++i) { 2650 uint16_t offset = (i == 0) ? op->offset : 0; 2651 uint32_t data_len = op->length; 2652 2653 TEST_ASSERT(orig_op->segments[i].length == data_len, 2654 "Length of segment differ in original (%u) and filled (%u) op mldts", 2655 orig_op->segments[i].length, data_len); 2656 data_len_iq = data_len; 2657 ref_out = (int8_t *)(orig_op->segments[i].addr); 2658 op_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2659 error_num = 0; 2660 for (j = 0; j < data_len_iq; j++) { 2661 2662 delt = ref_out[j] - op_out[j]; 2663 abs_delt = delt > 0 ? delt : -delt; 2664 error_num += (abs_delt > thres_hold ? 1 : 0); 2665 if (error_num > 0) 2666 printf("MLD Error %d: Exp %x %d Actual %x %d Diff %d\n", 2667 j, ref_out[j], ref_out[j], op_out[j], op_out[j], 2668 delt); 2669 } 2670 TEST_ASSERT(error_num == 0, 2671 "MLDTS Output are not matched total (%u) errors (%u)", 2672 data_len_iq, error_num); 2673 2674 m = m->next; 2675 } 2676 2677 return TEST_SUCCESS; 2678 } 2679 2680 static int 2681 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2682 struct rte_bbdev_fft_op *ref_op) 2683 { 2684 unsigned int i; 2685 int ret; 2686 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2687 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2688 2689 for (i = 0; i < n; ++i) { 2690 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2691 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2692 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2693 &ops[i]->fft.base_output, fft_data_orig), 2694 "FFT Output buffers (op=%u) are not matched", i); 2695 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2696 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2697 &ops[i]->fft.power_meas_output, fft_pwr_orig), 2698 "FFT Power Output buffers (op=%u) are not matched", i); 2699 } 2700 2701 return TEST_SUCCESS; 2702 } 2703 2704 static int 2705 validate_mldts_op(struct rte_bbdev_mldts_op **ops, const uint16_t n, 2706 struct rte_bbdev_mldts_op *ref_op) 2707 { 2708 unsigned int i; 2709 int ret; 2710 struct op_data_entries *mldts_data_orig = 2711 &test_vector.entries[DATA_HARD_OUTPUT]; 2712 for (i = 0; i < n; ++i) { 2713 ret = check_mldts_status_and_ordering(ops[i], i, ref_op->status); 2714 TEST_ASSERT_SUCCESS(ret, 2715 "Checking status and ordering for MLDTS failed"); 2716 TEST_ASSERT_SUCCESS(validate_op_mldts_chain( 2717 &ops[i]->mldts.output, 2718 mldts_data_orig), 2719 "MLDTS Output buffers (op=%u) are not matched", 2720 i); 2721 } 2722 2723 return TEST_SUCCESS; 2724 } 2725 2726 static void 2727 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2728 { 2729 unsigned int i; 2730 struct op_data_entries *entry; 2731 2732 op->turbo_dec = test_vector.turbo_dec; 2733 entry = &test_vector.entries[DATA_INPUT]; 2734 for (i = 0; i < entry->nb_segments; ++i) 2735 op->turbo_dec.input.length += 2736 entry->segments[i].length; 2737 } 2738 2739 static void 2740 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2741 { 2742 unsigned int i; 2743 struct op_data_entries *entry; 2744 2745 op->ldpc_dec = test_vector.ldpc_dec; 2746 entry = &test_vector.entries[DATA_INPUT]; 2747 for (i = 0; i < entry->nb_segments; ++i) 2748 op->ldpc_dec.input.length += 2749 entry->segments[i].length; 2750 if (test_vector.ldpc_dec.op_flags & 2751 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2752 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2753 for (i = 0; i < entry->nb_segments; ++i) 2754 op->ldpc_dec.harq_combined_input.length += 2755 entry->segments[i].length; 2756 } 2757 } 2758 2759 static void 2760 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2761 { 2762 unsigned int i; 2763 struct op_data_entries *entry; 2764 op->fft = test_vector.fft; 2765 entry = &test_vector.entries[DATA_INPUT]; 2766 for (i = 0; i < entry->nb_segments; ++i) 2767 op->fft.base_input.length += entry->segments[i].length; 2768 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2769 for (i = 0; i < entry->nb_segments; ++i) 2770 op->fft.dewindowing_input.length += entry->segments[i].length; 2771 } 2772 2773 static void 2774 create_reference_mldts_op(struct rte_bbdev_mldts_op *op) 2775 { 2776 unsigned int i; 2777 struct op_data_entries *entry; 2778 op->mldts = test_vector.mldts; 2779 entry = &test_vector.entries[DATA_INPUT]; 2780 for (i = 0; i < entry->nb_segments; ++i) 2781 op->mldts.qhy_input.length += entry->segments[i].length; 2782 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2783 for (i = 0; i < entry->nb_segments; ++i) 2784 op->mldts.r_input.length += entry->segments[i].length; 2785 } 2786 2787 static void 2788 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2789 { 2790 unsigned int i; 2791 struct op_data_entries *entry; 2792 2793 op->turbo_enc = test_vector.turbo_enc; 2794 entry = &test_vector.entries[DATA_INPUT]; 2795 for (i = 0; i < entry->nb_segments; ++i) 2796 op->turbo_enc.input.length += 2797 entry->segments[i].length; 2798 } 2799 2800 static void 2801 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2802 { 2803 unsigned int i; 2804 struct op_data_entries *entry; 2805 2806 op->ldpc_enc = test_vector.ldpc_enc; 2807 entry = &test_vector.entries[DATA_INPUT]; 2808 for (i = 0; i < entry->nb_segments; ++i) 2809 op->ldpc_enc.input.length += 2810 entry->segments[i].length; 2811 } 2812 2813 static uint32_t 2814 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2815 { 2816 uint8_t i; 2817 uint32_t c, r, tb_size = 0; 2818 2819 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2820 tb_size = op->turbo_dec.tb_params.k_neg; 2821 } else { 2822 c = op->turbo_dec.tb_params.c; 2823 r = op->turbo_dec.tb_params.r; 2824 for (i = 0; i < c-r; i++) 2825 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2826 op->turbo_dec.tb_params.k_neg : 2827 op->turbo_dec.tb_params.k_pos; 2828 } 2829 return tb_size; 2830 } 2831 2832 static uint32_t 2833 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2834 { 2835 uint8_t num_cbs = 0; 2836 uint32_t tb_size = 0; 2837 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2838 2839 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2840 num_cbs = 1; 2841 else 2842 num_cbs = op->ldpc_dec.tb_params.c - op->ldpc_dec.tb_params.r; 2843 2844 tb_size = (sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler) * num_cbs; 2845 return tb_size; 2846 } 2847 2848 static uint32_t 2849 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2850 { 2851 uint8_t i; 2852 uint32_t c, r, tb_size = 0; 2853 2854 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2855 tb_size = op->turbo_enc.tb_params.k_neg; 2856 } else { 2857 c = op->turbo_enc.tb_params.c; 2858 r = op->turbo_enc.tb_params.r; 2859 for (i = 0; i < c-r; i++) 2860 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2861 op->turbo_enc.tb_params.k_neg : 2862 op->turbo_enc.tb_params.k_pos; 2863 } 2864 return tb_size; 2865 } 2866 2867 static uint32_t 2868 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2869 { 2870 uint8_t num_cbs = 0; 2871 uint32_t tb_size = 0; 2872 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2873 2874 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) 2875 num_cbs = 1; 2876 else 2877 num_cbs = op->ldpc_enc.tb_params.c - op->ldpc_enc.tb_params.r; 2878 2879 tb_size = (sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler) * num_cbs; 2880 return tb_size; 2881 } 2882 2883 static uint32_t 2884 calc_fft_size(struct rte_bbdev_fft_op *op) 2885 { 2886 uint32_t output_size; 2887 int num_cs = 0, i; 2888 for (i = 0; i < 12; i++) 2889 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2890 num_cs++; 2891 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2892 return output_size; 2893 } 2894 2895 static uint32_t 2896 calc_mldts_size(struct rte_bbdev_mldts_op *op) 2897 { 2898 uint32_t output_size; 2899 output_size = op->mldts.num_layers * op->mldts.num_rbs * op->mldts.c_rep; 2900 return output_size; 2901 } 2902 2903 static int 2904 init_test_op_params(struct test_op_params *op_params, 2905 enum rte_bbdev_op_type op_type, const int expected_status, 2906 const int vector_mask, struct rte_mempool *ops_mp, 2907 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2908 { 2909 int ret = 0; 2910 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2911 op_type == RTE_BBDEV_OP_LDPC_DEC) 2912 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2913 &op_params->ref_dec_op, 1); 2914 else if (op_type == RTE_BBDEV_OP_FFT) 2915 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2916 &op_params->ref_fft_op, 1); 2917 else if (op_type == RTE_BBDEV_OP_MLDTS) 2918 ret = rte_bbdev_mldts_op_alloc_bulk(ops_mp, 2919 &op_params->ref_mldts_op, 1); 2920 else 2921 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2922 &op_params->ref_enc_op, 1); 2923 2924 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2925 2926 op_params->mp = ops_mp; 2927 op_params->burst_sz = burst_sz; 2928 op_params->num_to_process = num_to_process; 2929 op_params->num_lcores = num_lcores; 2930 op_params->vector_mask = vector_mask; 2931 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2932 op_type == RTE_BBDEV_OP_LDPC_DEC) 2933 op_params->ref_dec_op->status = expected_status; 2934 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2935 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2936 op_params->ref_enc_op->status = expected_status; 2937 else if (op_type == RTE_BBDEV_OP_FFT) 2938 op_params->ref_fft_op->status = expected_status; 2939 else if (op_type == RTE_BBDEV_OP_MLDTS) 2940 op_params->ref_mldts_op->status = expected_status; 2941 return 0; 2942 } 2943 2944 static int 2945 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2946 struct test_op_params *op_params) 2947 { 2948 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2949 unsigned int i; 2950 struct active_device *ad; 2951 unsigned int burst_sz = get_burst_sz(); 2952 enum rte_bbdev_op_type op_type = test_vector.op_type; 2953 const struct rte_bbdev_op_cap *capabilities = NULL; 2954 2955 ad = &active_devs[dev_id]; 2956 2957 /* Check if device supports op_type */ 2958 if (!is_avail_op(ad, test_vector.op_type)) 2959 return TEST_SUCCESS; 2960 2961 struct rte_bbdev_info info; 2962 rte_bbdev_info_get(ad->dev_id, &info); 2963 socket_id = GET_SOCKET(info.socket_id); 2964 2965 f_ret = create_mempools(ad, socket_id, op_type, 2966 get_num_ops()); 2967 if (f_ret != TEST_SUCCESS) { 2968 printf("Couldn't create mempools"); 2969 goto fail; 2970 } 2971 if (op_type == RTE_BBDEV_OP_NONE) 2972 op_type = RTE_BBDEV_OP_TURBO_ENC; 2973 2974 f_ret = init_test_op_params(op_params, test_vector.op_type, 2975 test_vector.expected_status, 2976 test_vector.mask, 2977 ad->ops_mempool, 2978 burst_sz, 2979 get_num_ops(), 2980 get_num_lcores()); 2981 if (f_ret != TEST_SUCCESS) { 2982 printf("Couldn't init test op params"); 2983 goto fail; 2984 } 2985 2986 2987 /* Find capabilities */ 2988 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2989 do { 2990 if (cap->type == test_vector.op_type) { 2991 capabilities = cap; 2992 break; 2993 } 2994 cap++; 2995 } while (cap->type != RTE_BBDEV_OP_NONE); 2996 TEST_ASSERT_NOT_NULL(capabilities, 2997 "Couldn't find capabilities"); 2998 2999 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3000 create_reference_dec_op(op_params->ref_dec_op); 3001 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3002 create_reference_enc_op(op_params->ref_enc_op); 3003 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3004 create_reference_ldpc_enc_op(op_params->ref_enc_op); 3005 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3006 create_reference_ldpc_dec_op(op_params->ref_dec_op); 3007 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3008 create_reference_fft_op(op_params->ref_fft_op); 3009 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3010 create_reference_mldts_op(op_params->ref_mldts_op); 3011 3012 for (i = 0; i < ad->nb_queues; ++i) { 3013 f_ret = fill_queue_buffers(op_params, 3014 ad->in_mbuf_pool, 3015 ad->hard_out_mbuf_pool, 3016 ad->soft_out_mbuf_pool, 3017 ad->harq_in_mbuf_pool, 3018 ad->harq_out_mbuf_pool, 3019 ad->queue_ids[i], 3020 capabilities, 3021 info.drv.min_alignment, 3022 socket_id); 3023 if (f_ret != TEST_SUCCESS) { 3024 printf("Couldn't init queue buffers"); 3025 goto fail; 3026 } 3027 } 3028 3029 /* Run test case function */ 3030 t_ret = test_case_func(ad, op_params); 3031 3032 /* Free active device resources and return */ 3033 free_buffers(ad, op_params); 3034 return t_ret; 3035 3036 fail: 3037 free_buffers(ad, op_params); 3038 return TEST_FAILED; 3039 } 3040 3041 /* Run given test function per active device per supported op type 3042 * per burst size. 3043 */ 3044 static int 3045 run_test_case(test_case_function *test_case_func) 3046 { 3047 int ret = 0; 3048 uint8_t dev; 3049 3050 /* Alloc op_params */ 3051 struct test_op_params *op_params = rte_zmalloc(NULL, 3052 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 3053 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 3054 RTE_ALIGN(sizeof(struct test_op_params), 3055 RTE_CACHE_LINE_SIZE)); 3056 3057 /* For each device run test case function */ 3058 for (dev = 0; dev < nb_active_devs; ++dev) 3059 ret |= run_test_case_on_device(test_case_func, dev, op_params); 3060 3061 rte_free(op_params); 3062 3063 return ret; 3064 } 3065 3066 3067 /* Push back the HARQ output from DDR to host */ 3068 static void 3069 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3070 struct rte_bbdev_dec_op **ops, 3071 const uint16_t n) 3072 { 3073 uint16_t j; 3074 int save_status, ret; 3075 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3076 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 3077 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3078 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 3079 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3080 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3081 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3082 for (j = 0; j < n; ++j) { 3083 if ((loopback && mem_out) || hc_out) { 3084 save_status = ops[j]->status; 3085 ops[j]->ldpc_dec.op_flags = 3086 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3087 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3088 if (h_comp) 3089 ops[j]->ldpc_dec.op_flags += 3090 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3091 ops[j]->ldpc_dec.harq_combined_input.offset = 3092 harq_offset; 3093 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 3094 harq_offset += HARQ_INCR; 3095 if (!loopback) 3096 ops[j]->ldpc_dec.harq_combined_input.length = 3097 ops[j]->ldpc_dec.harq_combined_output.length; 3098 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3099 &ops[j], 1); 3100 ret = 0; 3101 while (ret == 0) 3102 ret = rte_bbdev_dequeue_ldpc_dec_ops( 3103 dev_id, queue_id, 3104 &ops_deq[j], 1); 3105 ops[j]->ldpc_dec.op_flags = flags; 3106 ops[j]->status = save_status; 3107 } 3108 } 3109 } 3110 3111 /* 3112 * Push back the HARQ output from HW DDR to Host 3113 * Preload HARQ memory input and adjust HARQ offset 3114 */ 3115 static void 3116 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 3117 struct rte_bbdev_dec_op **ops, const uint16_t n, 3118 bool preload) 3119 { 3120 uint16_t j; 3121 int deq; 3122 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3123 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 3124 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 3125 uint32_t flags = ops[0]->ldpc_dec.op_flags; 3126 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 3127 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 3128 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3129 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 3130 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3131 if ((mem_in || hc_in) && preload) { 3132 for (j = 0; j < n; ++j) { 3133 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 3134 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 3135 ops[j]->ldpc_dec.op_flags = 3136 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 3137 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 3138 if (h_comp) 3139 ops[j]->ldpc_dec.op_flags += 3140 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 3141 ops[j]->ldpc_dec.harq_combined_output.offset = 3142 harq_offset; 3143 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 3144 harq_offset += HARQ_INCR; 3145 } 3146 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 3147 deq = 0; 3148 while (deq != n) 3149 deq += rte_bbdev_dequeue_ldpc_dec_ops( 3150 dev_id, queue_id, &ops_deq[deq], 3151 n - deq); 3152 /* Restore the operations */ 3153 for (j = 0; j < n; ++j) { 3154 ops[j]->ldpc_dec.op_flags = flags; 3155 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 3156 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 3157 } 3158 } 3159 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 3160 for (j = 0; j < n; ++j) { 3161 /* Adjust HARQ offset when we reach external DDR */ 3162 if (mem_in || hc_in) 3163 ops[j]->ldpc_dec.harq_combined_input.offset 3164 = harq_offset; 3165 if (mem_out || hc_out) 3166 ops[j]->ldpc_dec.harq_combined_output.offset 3167 = harq_offset; 3168 harq_offset += HARQ_INCR; 3169 } 3170 } 3171 3172 static void 3173 dequeue_event_callback(uint16_t dev_id, 3174 enum rte_bbdev_event_type event, void *cb_arg, 3175 void *ret_param) 3176 { 3177 int ret; 3178 uint16_t i; 3179 uint64_t total_time; 3180 uint16_t deq, burst_sz, num_ops; 3181 uint16_t queue_id = *(uint16_t *) ret_param; 3182 struct rte_bbdev_info info; 3183 double tb_len_bits; 3184 struct thread_params *tp = cb_arg; 3185 3186 /* Find matching thread params using queue_id */ 3187 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 3188 if (tp->queue_id == queue_id) 3189 break; 3190 3191 if (i == MAX_QUEUES) { 3192 printf("%s: Queue_id from interrupt details was not found!\n", 3193 __func__); 3194 return; 3195 } 3196 3197 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 3198 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3199 rte_memory_order_relaxed); 3200 printf( 3201 "Dequeue interrupt handler called for incorrect event!\n"); 3202 return; 3203 } 3204 3205 burst_sz = rte_atomic_load_explicit(&tp->burst_sz, rte_memory_order_relaxed); 3206 num_ops = tp->op_params->num_to_process; 3207 3208 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3209 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3210 &tp->dec_ops[ 3211 rte_atomic_load_explicit(&tp->nb_dequeued, 3212 rte_memory_order_relaxed)], 3213 burst_sz); 3214 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3215 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3216 &tp->dec_ops[ 3217 rte_atomic_load_explicit(&tp->nb_dequeued, 3218 rte_memory_order_relaxed)], 3219 burst_sz); 3220 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3221 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3222 &tp->enc_ops[ 3223 rte_atomic_load_explicit(&tp->nb_dequeued, 3224 rte_memory_order_relaxed)], 3225 burst_sz); 3226 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 3227 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 3228 &tp->fft_ops[ 3229 rte_atomic_load_explicit(&tp->nb_dequeued, 3230 rte_memory_order_relaxed)], 3231 burst_sz); 3232 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 3233 deq = rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 3234 &tp->mldts_ops[ 3235 rte_atomic_load_explicit(&tp->nb_dequeued, 3236 rte_memory_order_relaxed)], 3237 burst_sz); 3238 else /*RTE_BBDEV_OP_TURBO_ENC*/ 3239 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3240 &tp->enc_ops[ 3241 rte_atomic_load_explicit(&tp->nb_dequeued, 3242 rte_memory_order_relaxed)], 3243 burst_sz); 3244 3245 if (deq < burst_sz) { 3246 printf( 3247 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 3248 burst_sz, deq); 3249 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3250 rte_memory_order_relaxed); 3251 return; 3252 } 3253 3254 if (rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) + deq < num_ops) { 3255 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3256 return; 3257 } 3258 3259 total_time = rte_rdtsc_precise() - tp->start_time; 3260 3261 rte_bbdev_info_get(dev_id, &info); 3262 3263 ret = TEST_SUCCESS; 3264 3265 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 3266 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3267 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op); 3268 /* get the max of iter_count for all dequeued ops */ 3269 for (i = 0; i < num_ops; ++i) 3270 tp->iter_count = RTE_MAX( 3271 tp->dec_ops[i]->turbo_dec.iter_count, 3272 tp->iter_count); 3273 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3274 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 3275 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3276 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3277 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3278 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3279 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3280 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3281 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3282 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3283 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3284 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3285 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3286 } else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) { 3287 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 3288 ret = validate_mldts_op(tp->mldts_ops, num_ops, ref_op); 3289 rte_bbdev_mldts_op_free_bulk(tp->mldts_ops, deq); 3290 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3291 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3292 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3293 tp->op_params->vector_mask); 3294 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3295 } 3296 3297 if (ret) { 3298 printf("Buffers validation failed\n"); 3299 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3300 rte_memory_order_relaxed); 3301 } 3302 3303 switch (test_vector.op_type) { 3304 case RTE_BBDEV_OP_TURBO_DEC: 3305 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3306 break; 3307 case RTE_BBDEV_OP_TURBO_ENC: 3308 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3309 break; 3310 case RTE_BBDEV_OP_LDPC_DEC: 3311 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3312 break; 3313 case RTE_BBDEV_OP_FFT: 3314 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3315 break; 3316 case RTE_BBDEV_OP_MLDTS: 3317 tb_len_bits = calc_mldts_size(tp->op_params->ref_mldts_op); 3318 break; 3319 case RTE_BBDEV_OP_LDPC_ENC: 3320 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3321 break; 3322 case RTE_BBDEV_OP_NONE: 3323 tb_len_bits = 0.0; 3324 break; 3325 default: 3326 printf("Unknown op type: %d\n", test_vector.op_type); 3327 rte_atomic_store_explicit(&tp->processing_status, TEST_FAILED, 3328 rte_memory_order_relaxed); 3329 return; 3330 } 3331 3332 tp->ops_per_sec += ((double)num_ops) / 3333 ((double)total_time / (double)rte_get_tsc_hz()); 3334 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3335 ((double)total_time / (double)rte_get_tsc_hz()); 3336 3337 rte_atomic_fetch_add_explicit(&tp->nb_dequeued, deq, rte_memory_order_relaxed); 3338 } 3339 3340 static int 3341 throughput_intr_lcore_ldpc_dec(void *arg) 3342 { 3343 struct thread_params *tp = arg; 3344 unsigned int enqueued; 3345 const uint16_t queue_id = tp->queue_id; 3346 const uint16_t burst_sz = tp->op_params->burst_sz; 3347 const uint16_t num_to_process = tp->op_params->num_to_process; 3348 struct rte_bbdev_dec_op *ops[num_to_process]; 3349 struct test_buffers *bufs = NULL; 3350 struct rte_bbdev_info info; 3351 int ret, i, j; 3352 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3353 uint16_t num_to_enq, enq; 3354 3355 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3356 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3357 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3358 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3359 3360 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3361 "BURST_SIZE should be <= %u", MAX_BURST); 3362 3363 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3364 "Failed to enable interrupts for dev: %u, queue_id: %u", 3365 tp->dev_id, queue_id); 3366 3367 rte_bbdev_info_get(tp->dev_id, &info); 3368 3369 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3370 "NUM_OPS cannot exceed %u for this device", 3371 info.drv.queue_size_lim); 3372 3373 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3374 3375 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3376 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3377 3378 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3379 rte_memory_order_relaxed); 3380 3381 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3382 num_to_process); 3383 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3384 num_to_process); 3385 ref_op->ldpc_dec.iter_max = get_iter_max(); 3386 3387 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3388 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3389 bufs->hard_outputs, bufs->soft_outputs, 3390 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3391 3392 /* Set counter to validate the ordering */ 3393 for (j = 0; j < num_to_process; ++j) 3394 ops[j]->opaque_data = (void *)(uintptr_t)j; 3395 3396 for (j = 0; j < TEST_REPETITIONS; ++j) { 3397 for (i = 0; i < num_to_process; ++i) { 3398 if (!loopback) 3399 mbuf_reset(ops[i]->ldpc_dec.hard_output.data); 3400 if (hc_out || loopback) 3401 mbuf_reset(ops[i]->ldpc_dec.harq_combined_output.data); 3402 if (ops[i]->ldpc_dec.soft_output.data != NULL) 3403 mbuf_reset(ops[i]->ldpc_dec.soft_output.data); 3404 } 3405 3406 tp->start_time = rte_rdtsc_precise(); 3407 for (enqueued = 0; enqueued < num_to_process;) { 3408 num_to_enq = burst_sz; 3409 3410 if (unlikely(num_to_process - enqueued < num_to_enq)) 3411 num_to_enq = num_to_process - enqueued; 3412 3413 enq = 0; 3414 do { 3415 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3416 tp->dev_id, 3417 queue_id, &ops[enqueued], 3418 num_to_enq); 3419 } while (unlikely(num_to_enq != enq)); 3420 enqueued += enq; 3421 3422 /* Write to thread burst_sz current number of enqueued 3423 * descriptors. It ensures that proper number of 3424 * descriptors will be dequeued in callback 3425 * function - needed for last batch in case where 3426 * the number of operations is not a multiple of 3427 * burst size. 3428 */ 3429 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3430 rte_memory_order_relaxed); 3431 3432 /* Wait until processing of previous batch is 3433 * completed 3434 */ 3435 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3436 rte_memory_order_relaxed); 3437 } 3438 if (j != TEST_REPETITIONS - 1) 3439 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3440 } 3441 3442 return TEST_SUCCESS; 3443 } 3444 3445 static int 3446 throughput_intr_lcore_dec(void *arg) 3447 { 3448 struct thread_params *tp = arg; 3449 unsigned int enqueued; 3450 const uint16_t queue_id = tp->queue_id; 3451 const uint16_t burst_sz = tp->op_params->burst_sz; 3452 const uint16_t num_to_process = tp->op_params->num_to_process; 3453 struct rte_bbdev_dec_op *ops[num_to_process]; 3454 struct test_buffers *bufs = NULL; 3455 struct rte_bbdev_info info; 3456 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3457 int ret, i, j; 3458 uint16_t num_to_enq, enq; 3459 3460 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3461 "BURST_SIZE should be <= %u", MAX_BURST); 3462 3463 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3464 "Failed to enable interrupts for dev: %u, queue_id: %u", 3465 tp->dev_id, queue_id); 3466 3467 rte_bbdev_info_get(tp->dev_id, &info); 3468 3469 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3470 "NUM_OPS cannot exceed %u for this device", 3471 info.drv.queue_size_lim); 3472 3473 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3474 3475 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3476 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3477 3478 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3479 rte_memory_order_relaxed); 3480 3481 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3482 num_to_process); 3483 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3484 ref_op->turbo_dec.iter_max = get_iter_max(); 3485 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3486 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3487 bufs->hard_outputs, bufs->soft_outputs, 3488 tp->op_params->ref_dec_op); 3489 3490 /* Set counter to validate the ordering. */ 3491 for (j = 0; j < num_to_process; ++j) 3492 ops[j]->opaque_data = (void *)(uintptr_t)j; 3493 3494 for (j = 0; j < TEST_REPETITIONS; ++j) { 3495 for (i = 0; i < num_to_process; ++i) { 3496 mbuf_reset(ops[i]->turbo_dec.hard_output.data); 3497 if (ops[i]->turbo_dec.soft_output.data != NULL) 3498 mbuf_reset(ops[i]->turbo_dec.soft_output.data); 3499 } 3500 3501 tp->start_time = rte_rdtsc_precise(); 3502 for (enqueued = 0; enqueued < num_to_process;) { 3503 num_to_enq = burst_sz; 3504 3505 if (unlikely(num_to_process - enqueued < num_to_enq)) 3506 num_to_enq = num_to_process - enqueued; 3507 3508 enq = 0; 3509 do { 3510 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3511 queue_id, &ops[enqueued], 3512 num_to_enq); 3513 } while (unlikely(num_to_enq != enq)); 3514 enqueued += enq; 3515 3516 /* Write to thread burst_sz current number of enqueued 3517 * descriptors. It ensures that proper number of 3518 * descriptors will be dequeued in callback 3519 * function - needed for last batch in case where 3520 * the number of operations is not a multiple of 3521 * burst size. 3522 */ 3523 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3524 rte_memory_order_relaxed); 3525 3526 /* Wait until processing of previous batch is 3527 * completed 3528 */ 3529 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3530 rte_memory_order_relaxed); 3531 } 3532 if (j != TEST_REPETITIONS - 1) 3533 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3534 } 3535 3536 return TEST_SUCCESS; 3537 } 3538 3539 static int 3540 throughput_intr_lcore_enc(void *arg) 3541 { 3542 struct thread_params *tp = arg; 3543 unsigned int enqueued; 3544 const uint16_t queue_id = tp->queue_id; 3545 const uint16_t burst_sz = tp->op_params->burst_sz; 3546 const uint16_t num_to_process = tp->op_params->num_to_process; 3547 struct rte_bbdev_enc_op *ops[num_to_process]; 3548 struct test_buffers *bufs = NULL; 3549 struct rte_bbdev_info info; 3550 int ret, i, j; 3551 uint16_t num_to_enq, enq; 3552 3553 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3554 "BURST_SIZE should be <= %u", MAX_BURST); 3555 3556 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3557 "Failed to enable interrupts for dev: %u, queue_id: %u", 3558 tp->dev_id, queue_id); 3559 3560 rte_bbdev_info_get(tp->dev_id, &info); 3561 3562 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3563 "NUM_OPS cannot exceed %u for this device", 3564 info.drv.queue_size_lim); 3565 3566 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3567 3568 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3569 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3570 3571 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3572 rte_memory_order_relaxed); 3573 3574 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3575 num_to_process); 3576 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3577 num_to_process); 3578 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3579 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3580 bufs->hard_outputs, tp->op_params->ref_enc_op); 3581 3582 /* Set counter to validate the ordering */ 3583 for (j = 0; j < num_to_process; ++j) 3584 ops[j]->opaque_data = (void *)(uintptr_t)j; 3585 3586 for (j = 0; j < TEST_REPETITIONS; ++j) { 3587 for (i = 0; i < num_to_process; ++i) 3588 mbuf_reset(ops[i]->turbo_enc.output.data); 3589 3590 tp->start_time = rte_rdtsc_precise(); 3591 for (enqueued = 0; enqueued < num_to_process;) { 3592 num_to_enq = burst_sz; 3593 3594 if (unlikely(num_to_process - enqueued < num_to_enq)) 3595 num_to_enq = num_to_process - enqueued; 3596 3597 enq = 0; 3598 do { 3599 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3600 queue_id, &ops[enqueued], 3601 num_to_enq); 3602 } while (unlikely(enq != num_to_enq)); 3603 enqueued += enq; 3604 3605 /* Write to thread burst_sz current number of enqueued 3606 * descriptors. It ensures that proper number of 3607 * descriptors will be dequeued in callback 3608 * function - needed for last batch in case where 3609 * the number of operations is not a multiple of 3610 * burst size. 3611 */ 3612 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3613 rte_memory_order_relaxed); 3614 3615 /* Wait until processing of previous batch is 3616 * completed 3617 */ 3618 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3619 rte_memory_order_relaxed); 3620 } 3621 if (j != TEST_REPETITIONS - 1) 3622 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3623 } 3624 3625 return TEST_SUCCESS; 3626 } 3627 3628 3629 static int 3630 throughput_intr_lcore_ldpc_enc(void *arg) 3631 { 3632 struct thread_params *tp = arg; 3633 unsigned int enqueued; 3634 const uint16_t queue_id = tp->queue_id; 3635 const uint16_t burst_sz = tp->op_params->burst_sz; 3636 const uint16_t num_to_process = tp->op_params->num_to_process; 3637 struct rte_bbdev_enc_op *ops[num_to_process]; 3638 struct test_buffers *bufs = NULL; 3639 struct rte_bbdev_info info; 3640 int ret, i, j; 3641 uint16_t num_to_enq, enq; 3642 3643 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3644 "BURST_SIZE should be <= %u", MAX_BURST); 3645 3646 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3647 "Failed to enable interrupts for dev: %u, queue_id: %u", 3648 tp->dev_id, queue_id); 3649 3650 rte_bbdev_info_get(tp->dev_id, &info); 3651 3652 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3653 "NUM_OPS cannot exceed %u for this device", 3654 info.drv.queue_size_lim); 3655 3656 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3657 3658 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3659 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3660 3661 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3662 rte_memory_order_relaxed); 3663 3664 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3665 num_to_process); 3666 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3667 num_to_process); 3668 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3669 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3670 bufs->inputs, bufs->hard_outputs, 3671 tp->op_params->ref_enc_op); 3672 3673 /* Set counter to validate the ordering */ 3674 for (j = 0; j < num_to_process; ++j) 3675 ops[j]->opaque_data = (void *)(uintptr_t)j; 3676 3677 for (j = 0; j < TEST_REPETITIONS; ++j) { 3678 for (i = 0; i < num_to_process; ++i) 3679 mbuf_reset(ops[i]->turbo_enc.output.data); 3680 3681 tp->start_time = rte_rdtsc_precise(); 3682 for (enqueued = 0; enqueued < num_to_process;) { 3683 num_to_enq = burst_sz; 3684 3685 if (unlikely(num_to_process - enqueued < num_to_enq)) 3686 num_to_enq = num_to_process - enqueued; 3687 3688 enq = 0; 3689 do { 3690 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3691 tp->dev_id, 3692 queue_id, &ops[enqueued], 3693 num_to_enq); 3694 } while (unlikely(enq != num_to_enq)); 3695 enqueued += enq; 3696 3697 /* Write to thread burst_sz current number of enqueued 3698 * descriptors. It ensures that proper number of 3699 * descriptors will be dequeued in callback 3700 * function - needed for last batch in case where 3701 * the number of operations is not a multiple of 3702 * burst size. 3703 */ 3704 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3705 rte_memory_order_relaxed); 3706 3707 /* Wait until processing of previous batch is 3708 * completed 3709 */ 3710 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3711 rte_memory_order_relaxed); 3712 } 3713 if (j != TEST_REPETITIONS - 1) 3714 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3715 } 3716 3717 return TEST_SUCCESS; 3718 } 3719 3720 3721 static int 3722 throughput_intr_lcore_fft(void *arg) 3723 { 3724 struct thread_params *tp = arg; 3725 unsigned int enqueued; 3726 const uint16_t queue_id = tp->queue_id; 3727 const uint16_t burst_sz = tp->op_params->burst_sz; 3728 const uint16_t num_to_process = tp->op_params->num_to_process; 3729 struct rte_bbdev_fft_op *ops[num_to_process]; 3730 struct test_buffers *bufs = NULL; 3731 struct rte_bbdev_info info; 3732 int ret, i, j; 3733 uint16_t num_to_enq, enq; 3734 3735 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3736 "BURST_SIZE should be <= %u", MAX_BURST); 3737 3738 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3739 "Failed to enable interrupts for dev: %u, queue_id: %u", 3740 tp->dev_id, queue_id); 3741 3742 rte_bbdev_info_get(tp->dev_id, &info); 3743 3744 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3745 "NUM_OPS cannot exceed %u for this device", 3746 info.drv.queue_size_lim); 3747 3748 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3749 3750 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3751 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3752 3753 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3754 rte_memory_order_relaxed); 3755 3756 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3757 num_to_process); 3758 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3759 num_to_process); 3760 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3761 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3762 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 3763 tp->op_params->ref_fft_op); 3764 3765 /* Set counter to validate the ordering */ 3766 for (j = 0; j < num_to_process; ++j) 3767 ops[j]->opaque_data = (void *)(uintptr_t)j; 3768 3769 for (j = 0; j < TEST_REPETITIONS; ++j) { 3770 for (i = 0; i < num_to_process; ++i) 3771 mbuf_reset(ops[i]->fft.base_output.data); 3772 3773 tp->start_time = rte_rdtsc_precise(); 3774 for (enqueued = 0; enqueued < num_to_process;) { 3775 num_to_enq = burst_sz; 3776 3777 if (unlikely(num_to_process - enqueued < num_to_enq)) 3778 num_to_enq = num_to_process - enqueued; 3779 3780 enq = 0; 3781 do { 3782 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3783 queue_id, &ops[enqueued], 3784 num_to_enq); 3785 } while (unlikely(enq != num_to_enq)); 3786 enqueued += enq; 3787 3788 /* Write to thread burst_sz current number of enqueued 3789 * descriptors. It ensures that proper number of 3790 * descriptors will be dequeued in callback 3791 * function - needed for last batch in case where 3792 * the number of operations is not a multiple of 3793 * burst size. 3794 */ 3795 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3796 rte_memory_order_relaxed); 3797 3798 /* Wait until processing of previous batch is 3799 * completed 3800 */ 3801 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3802 rte_memory_order_relaxed); 3803 } 3804 if (j != TEST_REPETITIONS - 1) 3805 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3806 } 3807 3808 return TEST_SUCCESS; 3809 } 3810 3811 static int 3812 throughput_intr_lcore_mldts(void *arg) 3813 { 3814 struct thread_params *tp = arg; 3815 unsigned int enqueued; 3816 const uint16_t queue_id = tp->queue_id; 3817 const uint16_t burst_sz = tp->op_params->burst_sz; 3818 const uint16_t num_to_process = tp->op_params->num_to_process; 3819 struct rte_bbdev_mldts_op *ops[num_to_process]; 3820 struct test_buffers *bufs = NULL; 3821 struct rte_bbdev_info info; 3822 int ret, i, j; 3823 uint16_t num_to_enq, enq; 3824 3825 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 3826 3827 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3828 "Failed to enable interrupts for dev: %u, queue_id: %u", 3829 tp->dev_id, queue_id); 3830 3831 rte_bbdev_info_get(tp->dev_id, &info); 3832 3833 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3834 "NUM_OPS cannot exceed %u for this device", 3835 info.drv.queue_size_lim); 3836 3837 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3838 3839 rte_atomic_store_explicit(&tp->processing_status, 0, rte_memory_order_relaxed); 3840 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3841 3842 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3843 rte_memory_order_relaxed); 3844 3845 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops, num_to_process); 3846 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3847 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3848 copy_reference_mldts_op(ops, num_to_process, 0, bufs->inputs, bufs->harq_inputs, 3849 bufs->hard_outputs, tp->op_params->ref_mldts_op); 3850 3851 /* Set counter to validate the ordering */ 3852 for (j = 0; j < num_to_process; ++j) 3853 ops[j]->opaque_data = (void *)(uintptr_t)j; 3854 3855 for (j = 0; j < TEST_REPETITIONS; ++j) { 3856 for (i = 0; i < num_to_process; ++i) 3857 mbuf_reset(ops[i]->mldts.output.data); 3858 3859 tp->start_time = rte_rdtsc_precise(); 3860 for (enqueued = 0; enqueued < num_to_process;) { 3861 num_to_enq = burst_sz; 3862 3863 if (unlikely(num_to_process - enqueued < num_to_enq)) 3864 num_to_enq = num_to_process - enqueued; 3865 3866 enq = 0; 3867 do { 3868 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 3869 queue_id, &ops[enqueued], num_to_enq); 3870 } while (unlikely(enq != num_to_enq)); 3871 enqueued += enq; 3872 3873 /* Write to thread burst_sz current number of enqueued 3874 * descriptors. It ensures that proper number of 3875 * descriptors will be dequeued in callback 3876 * function - needed for last batch in case where 3877 * the number of operations is not a multiple of 3878 * burst size. 3879 */ 3880 rte_atomic_store_explicit(&tp->burst_sz, num_to_enq, 3881 rte_memory_order_relaxed); 3882 3883 /* Wait until processing of previous batch is 3884 * completed 3885 */ 3886 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->nb_dequeued, enqueued, 3887 rte_memory_order_relaxed); 3888 } 3889 if (j != TEST_REPETITIONS - 1) 3890 rte_atomic_store_explicit(&tp->nb_dequeued, 0, rte_memory_order_relaxed); 3891 } 3892 3893 return TEST_SUCCESS; 3894 } 3895 3896 static int 3897 throughput_pmd_lcore_dec(void *arg) 3898 { 3899 struct thread_params *tp = arg; 3900 uint16_t enq, deq; 3901 uint64_t total_time = 0, start_time; 3902 const uint16_t queue_id = tp->queue_id; 3903 const uint16_t burst_sz = tp->op_params->burst_sz; 3904 const uint16_t num_ops = tp->op_params->num_to_process; 3905 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3906 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3907 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3908 struct test_buffers *bufs = NULL; 3909 int i, j, ret; 3910 struct rte_bbdev_info info; 3911 uint16_t num_to_enq; 3912 bool so_enable; 3913 3914 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3915 "BURST_SIZE should be <= %u", MAX_BURST); 3916 3917 rte_bbdev_info_get(tp->dev_id, &info); 3918 3919 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3920 "NUM_OPS cannot exceed %u for this device", 3921 info.drv.queue_size_lim); 3922 3923 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3924 3925 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 3926 rte_memory_order_relaxed); 3927 3928 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3929 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3930 ref_op->turbo_dec.iter_max = get_iter_max(); 3931 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3932 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3933 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3934 3935 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 3936 3937 /* Set counter to validate the ordering */ 3938 for (j = 0; j < num_ops; ++j) 3939 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3940 3941 for (i = 0; i < TEST_REPETITIONS; ++i) { 3942 uint32_t time_out = 0; 3943 for (j = 0; j < num_ops; ++j) 3944 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3945 if (so_enable) 3946 for (j = 0; j < num_ops; ++j) 3947 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 3948 3949 start_time = rte_rdtsc_precise(); 3950 3951 for (enq = 0, deq = 0; enq < num_ops;) { 3952 num_to_enq = burst_sz; 3953 3954 if (unlikely(num_ops - enq < num_to_enq)) 3955 num_to_enq = num_ops - enq; 3956 3957 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3958 queue_id, &ops_enq[enq], num_to_enq); 3959 3960 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3961 queue_id, &ops_deq[deq], enq - deq); 3962 time_out++; 3963 if (time_out >= TIME_OUT_POLL) { 3964 timeout_exit(tp->dev_id); 3965 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 3966 } 3967 } 3968 3969 /* dequeue the remaining */ 3970 time_out = 0; 3971 while (deq < enq) { 3972 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3973 queue_id, &ops_deq[deq], enq - deq); 3974 time_out++; 3975 if (time_out >= TIME_OUT_POLL) { 3976 timeout_exit(tp->dev_id); 3977 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 3978 } 3979 } 3980 3981 total_time += rte_rdtsc_precise() - start_time; 3982 } 3983 3984 tp->iter_count = 0; 3985 /* get the max of iter_count for all dequeued ops */ 3986 for (i = 0; i < num_ops; ++i) { 3987 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3988 tp->iter_count); 3989 } 3990 3991 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3992 ret = validate_dec_op(ops_deq, num_ops, ref_op); 3993 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3994 } 3995 3996 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3997 3998 double tb_len_bits = calc_dec_TB_size(ref_op); 3999 4000 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4001 ((double)total_time / (double)rte_get_tsc_hz()); 4002 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4003 1000000.0) / ((double)total_time / 4004 (double)rte_get_tsc_hz()); 4005 4006 return TEST_SUCCESS; 4007 } 4008 4009 static int 4010 bler_pmd_lcore_ldpc_dec(void *arg) 4011 { 4012 struct thread_params *tp = arg; 4013 uint16_t enq, deq; 4014 uint64_t total_time = 0, start_time; 4015 const uint16_t queue_id = tp->queue_id; 4016 const uint16_t burst_sz = tp->op_params->burst_sz; 4017 const uint16_t num_ops = tp->op_params->num_to_process; 4018 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4019 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4020 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4021 struct test_buffers *bufs = NULL; 4022 int i, j, ret; 4023 float parity_bler = 0; 4024 struct rte_bbdev_info info; 4025 uint16_t num_to_enq; 4026 bool extDdr = check_bit(ldpc_cap_flags, 4027 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4028 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4029 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4030 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4031 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4032 4033 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4034 "BURST_SIZE should be <= %u", MAX_BURST); 4035 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4036 4037 rte_bbdev_info_get(tp->dev_id, &info); 4038 4039 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4040 "NUM_OPS cannot exceed %u for this device", 4041 info.drv.queue_size_lim); 4042 4043 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4044 4045 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4046 rte_memory_order_relaxed); 4047 4048 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4049 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4050 4051 /* For BLER tests we need to enable early termination */ 4052 if (!check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4053 ref_op->ldpc_dec.op_flags += RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4054 4055 ref_op->ldpc_dec.iter_max = get_iter_max(); 4056 4057 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4058 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4059 bufs->hard_outputs, bufs->soft_outputs, 4060 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4061 generate_llr_input(num_ops, bufs->inputs, ref_op); 4062 4063 /* Set counter to validate the ordering */ 4064 for (j = 0; j < num_ops; ++j) 4065 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4066 4067 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4068 uint32_t time_out = 0; 4069 for (j = 0; j < num_ops; ++j) { 4070 if (!loopback) 4071 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4072 if (hc_out || loopback) 4073 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4074 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4075 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4076 } 4077 if (extDdr) 4078 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4079 num_ops, true); 4080 start_time = rte_rdtsc_precise(); 4081 4082 for (enq = 0, deq = 0; enq < num_ops;) { 4083 num_to_enq = burst_sz; 4084 4085 if (unlikely(num_ops - enq < num_to_enq)) 4086 num_to_enq = num_ops - enq; 4087 4088 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4089 queue_id, &ops_enq[enq], num_to_enq); 4090 4091 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4092 queue_id, &ops_deq[deq], enq - deq); 4093 time_out++; 4094 if (time_out >= TIME_OUT_POLL) { 4095 timeout_exit(tp->dev_id); 4096 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4097 } 4098 } 4099 4100 /* dequeue the remaining */ 4101 time_out = 0; 4102 while (deq < enq) { 4103 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4104 queue_id, &ops_deq[deq], enq - deq); 4105 time_out++; 4106 if (time_out >= TIME_OUT_POLL) { 4107 timeout_exit(tp->dev_id); 4108 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4109 } 4110 } 4111 4112 total_time += rte_rdtsc_precise() - start_time; 4113 } 4114 4115 tp->iter_count = 0; 4116 tp->iter_average = 0; 4117 /* get the max of iter_count for all dequeued ops */ 4118 for (i = 0; i < num_ops; ++i) { 4119 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4120 tp->iter_count); 4121 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 4122 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 4123 parity_bler += 1.0; 4124 } 4125 4126 parity_bler /= num_ops; /* This one is based on SYND */ 4127 tp->iter_average /= num_ops; 4128 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 4129 4130 if (test_vector.op_type != RTE_BBDEV_OP_NONE 4131 && tp->bler == 0 4132 && parity_bler == 0 4133 && !hc_out) { 4134 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4135 tp->op_params->vector_mask); 4136 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4137 } 4138 4139 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4140 4141 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4142 tp->ops_per_sec = ((double)num_ops * 1) / 4143 ((double)total_time / (double)rte_get_tsc_hz()); 4144 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4145 1000000.0) / ((double)total_time / 4146 (double)rte_get_tsc_hz()); 4147 4148 return TEST_SUCCESS; 4149 } 4150 4151 4152 static int 4153 bler_pmd_lcore_turbo_dec(void *arg) 4154 { 4155 struct thread_params *tp = arg; 4156 uint16_t enq, deq; 4157 uint64_t total_time = 0, start_time; 4158 const uint16_t queue_id = tp->queue_id; 4159 const uint16_t burst_sz = tp->op_params->burst_sz; 4160 const uint16_t num_ops = tp->op_params->num_to_process; 4161 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4162 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4163 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4164 struct test_buffers *bufs = NULL; 4165 int i, j, ret; 4166 struct rte_bbdev_info info; 4167 uint16_t num_to_enq; 4168 4169 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4170 "BURST_SIZE should be <= %u", MAX_BURST); 4171 TEST_ASSERT_SUCCESS((num_ops == 0), "NUM_OPS must be greater than 0"); 4172 4173 rte_bbdev_info_get(tp->dev_id, &info); 4174 4175 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4176 "NUM_OPS cannot exceed %u for this device", 4177 info.drv.queue_size_lim); 4178 4179 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4180 4181 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4182 rte_memory_order_relaxed); 4183 4184 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4185 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4186 4187 /* For BLER tests we need to enable early termination */ 4188 if (!check_bit(ref_op->turbo_dec.op_flags, RTE_BBDEV_TURBO_EARLY_TERMINATION)) 4189 ref_op->turbo_dec.op_flags += RTE_BBDEV_TURBO_EARLY_TERMINATION; 4190 4191 ref_op->turbo_dec.iter_max = get_iter_max(); 4192 4193 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4194 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4195 bufs->hard_outputs, bufs->soft_outputs, 4196 ref_op); 4197 generate_turbo_llr_input(num_ops, bufs->inputs, ref_op); 4198 4199 /* Set counter to validate the ordering */ 4200 for (j = 0; j < num_ops; ++j) 4201 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4202 4203 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 4204 uint32_t time_out = 0; 4205 for (j = 0; j < num_ops; ++j) { 4206 mbuf_reset( 4207 ops_enq[j]->turbo_dec.hard_output.data); 4208 } 4209 4210 start_time = rte_rdtsc_precise(); 4211 4212 for (enq = 0, deq = 0; enq < num_ops;) { 4213 num_to_enq = burst_sz; 4214 4215 if (unlikely(num_ops - enq < num_to_enq)) 4216 num_to_enq = num_ops - enq; 4217 4218 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 4219 queue_id, &ops_enq[enq], num_to_enq); 4220 4221 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4222 queue_id, &ops_deq[deq], enq - deq); 4223 time_out++; 4224 if (time_out >= TIME_OUT_POLL) { 4225 timeout_exit(tp->dev_id); 4226 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4227 } 4228 } 4229 4230 /* dequeue the remaining */ 4231 time_out = 0; 4232 while (deq < enq) { 4233 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 4234 queue_id, &ops_deq[deq], enq - deq); 4235 time_out++; 4236 if (time_out >= TIME_OUT_POLL) { 4237 timeout_exit(tp->dev_id); 4238 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4239 } 4240 } 4241 4242 total_time += rte_rdtsc_precise() - start_time; 4243 } 4244 4245 tp->iter_count = 0; 4246 tp->iter_average = 0; 4247 /* get the max of iter_count for all dequeued ops */ 4248 for (i = 0; i < num_ops; ++i) { 4249 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 4250 tp->iter_count); 4251 tp->iter_average += (double) ops_enq[i]->turbo_dec.iter_count; 4252 } 4253 4254 tp->iter_average /= num_ops; 4255 tp->bler = (double) validate_turbo_bler(ops_deq, num_ops) / num_ops; 4256 4257 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4258 4259 double tb_len_bits = calc_dec_TB_size(ref_op); 4260 tp->ops_per_sec = ((double)num_ops * 1) / 4261 ((double)total_time / (double)rte_get_tsc_hz()); 4262 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 4263 1000000.0) / ((double)total_time / 4264 (double)rte_get_tsc_hz()); 4265 printf("TBS %.0f Time %.0f\n", tb_len_bits, 1000000.0 * 4266 ((double)total_time / (double)rte_get_tsc_hz())); 4267 4268 return TEST_SUCCESS; 4269 } 4270 4271 static int 4272 throughput_pmd_lcore_ldpc_dec(void *arg) 4273 { 4274 struct thread_params *tp = arg; 4275 uint16_t enq, deq; 4276 uint64_t total_time = 0, start_time; 4277 const uint16_t queue_id = tp->queue_id; 4278 const uint16_t burst_sz = tp->op_params->burst_sz; 4279 const uint16_t num_ops = tp->op_params->num_to_process; 4280 struct rte_bbdev_dec_op *ops_enq[num_ops]; 4281 struct rte_bbdev_dec_op *ops_deq[num_ops]; 4282 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 4283 struct test_buffers *bufs = NULL; 4284 int i, j, ret; 4285 struct rte_bbdev_info info; 4286 uint16_t num_to_enq; 4287 bool extDdr = check_bit(ldpc_cap_flags, 4288 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 4289 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 4290 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 4291 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 4292 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 4293 4294 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4295 "BURST_SIZE should be <= %u", MAX_BURST); 4296 4297 rte_bbdev_info_get(tp->dev_id, &info); 4298 4299 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4300 "NUM_OPS cannot exceed %u for this device", 4301 info.drv.queue_size_lim); 4302 4303 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4304 4305 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4306 rte_memory_order_relaxed); 4307 4308 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4309 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4310 4311 /* For throughput tests we need to disable early termination */ 4312 if (check_bit(ref_op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4313 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4314 4315 ref_op->ldpc_dec.iter_max = get_iter_max(); 4316 /* Since ET is disabled, the expected iter_count is iter_max */ 4317 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4318 4319 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4320 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 4321 bufs->hard_outputs, bufs->soft_outputs, 4322 bufs->harq_inputs, bufs->harq_outputs, ref_op); 4323 4324 /* Set counter to validate the ordering */ 4325 for (j = 0; j < num_ops; ++j) 4326 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4327 4328 for (i = 0; i < TEST_REPETITIONS; ++i) { 4329 uint32_t time_out = 0; 4330 for (j = 0; j < num_ops; ++j) { 4331 if (!loopback) 4332 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 4333 if (hc_out || loopback) 4334 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 4335 if (ops_enq[j]->ldpc_dec.soft_output.data != NULL) 4336 mbuf_reset(ops_enq[j]->ldpc_dec.soft_output.data); 4337 } 4338 if (extDdr) 4339 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 4340 num_ops, true); 4341 start_time = rte_rdtsc_precise(); 4342 4343 for (enq = 0, deq = 0; enq < num_ops;) { 4344 num_to_enq = burst_sz; 4345 4346 if (unlikely(num_ops - enq < num_to_enq)) 4347 num_to_enq = num_ops - enq; 4348 4349 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 4350 queue_id, &ops_enq[enq], num_to_enq); 4351 4352 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4353 queue_id, &ops_deq[deq], enq - deq); 4354 time_out++; 4355 if (time_out >= TIME_OUT_POLL) { 4356 timeout_exit(tp->dev_id); 4357 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4358 } 4359 } 4360 4361 /* dequeue the remaining */ 4362 time_out = 0; 4363 while (deq < enq) { 4364 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 4365 queue_id, &ops_deq[deq], enq - deq); 4366 time_out++; 4367 if (time_out >= TIME_OUT_POLL) { 4368 timeout_exit(tp->dev_id); 4369 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4370 } 4371 } 4372 4373 total_time += rte_rdtsc_precise() - start_time; 4374 } 4375 4376 tp->iter_count = 0; 4377 /* get the max of iter_count for all dequeued ops */ 4378 for (i = 0; i < num_ops; ++i) { 4379 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 4380 tp->iter_count); 4381 } 4382 if (extDdr) { 4383 /* Read loopback is not thread safe */ 4384 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 4385 } 4386 4387 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4388 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 4389 tp->op_params->vector_mask); 4390 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4391 } 4392 4393 ret = rte_bbdev_queue_stop(tp->dev_id, queue_id); 4394 if (ret != 0) 4395 printf("Failed to stop queue on dev %u q_id: %u\n", tp->dev_id, queue_id); 4396 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 4397 4398 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 4399 4400 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4401 ((double)total_time / (double)rte_get_tsc_hz()); 4402 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4403 1000000.0) / ((double)total_time / 4404 (double)rte_get_tsc_hz()); 4405 4406 return TEST_SUCCESS; 4407 } 4408 4409 static int 4410 throughput_pmd_lcore_enc(void *arg) 4411 { 4412 struct thread_params *tp = arg; 4413 uint16_t enq, deq; 4414 uint64_t total_time = 0, start_time; 4415 const uint16_t queue_id = tp->queue_id; 4416 const uint16_t burst_sz = tp->op_params->burst_sz; 4417 const uint16_t num_ops = tp->op_params->num_to_process; 4418 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4419 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4420 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4421 struct test_buffers *bufs = NULL; 4422 int i, j, ret; 4423 struct rte_bbdev_info info; 4424 uint16_t num_to_enq; 4425 4426 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4427 "BURST_SIZE should be <= %u", MAX_BURST); 4428 4429 rte_bbdev_info_get(tp->dev_id, &info); 4430 4431 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4432 "NUM_OPS cannot exceed %u for this device", 4433 info.drv.queue_size_lim); 4434 4435 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4436 4437 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4438 rte_memory_order_relaxed); 4439 4440 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4441 num_ops); 4442 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4443 num_ops); 4444 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4445 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4446 bufs->hard_outputs, ref_op); 4447 4448 /* Set counter to validate the ordering */ 4449 for (j = 0; j < num_ops; ++j) 4450 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4451 4452 for (i = 0; i < TEST_REPETITIONS; ++i) { 4453 uint32_t time_out = 0; 4454 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4455 for (j = 0; j < num_ops; ++j) 4456 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4457 4458 start_time = rte_rdtsc_precise(); 4459 4460 for (enq = 0, deq = 0; enq < num_ops;) { 4461 num_to_enq = burst_sz; 4462 4463 if (unlikely(num_ops - enq < num_to_enq)) 4464 num_to_enq = num_ops - enq; 4465 4466 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 4467 queue_id, &ops_enq[enq], num_to_enq); 4468 4469 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4470 queue_id, &ops_deq[deq], enq - deq); 4471 time_out++; 4472 if (time_out >= TIME_OUT_POLL) { 4473 timeout_exit(tp->dev_id); 4474 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4475 } 4476 } 4477 4478 /* dequeue the remaining */ 4479 time_out = 0; 4480 while (deq < enq) { 4481 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 4482 queue_id, &ops_deq[deq], enq - deq); 4483 time_out++; 4484 if (time_out >= TIME_OUT_POLL) { 4485 timeout_exit(tp->dev_id); 4486 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4487 } 4488 } 4489 4490 total_time += rte_rdtsc_precise() - start_time; 4491 } 4492 4493 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4494 ret = validate_enc_op(ops_deq, num_ops, ref_op); 4495 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4496 } 4497 4498 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4499 4500 double tb_len_bits = calc_enc_TB_size(ref_op); 4501 4502 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4503 ((double)total_time / (double)rte_get_tsc_hz()); 4504 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4505 / 1000000.0) / ((double)total_time / 4506 (double)rte_get_tsc_hz()); 4507 4508 return TEST_SUCCESS; 4509 } 4510 4511 static int 4512 throughput_pmd_lcore_ldpc_enc(void *arg) 4513 { 4514 struct thread_params *tp = arg; 4515 uint16_t enq, deq; 4516 uint64_t total_time = 0, start_time; 4517 const uint16_t queue_id = tp->queue_id; 4518 const uint16_t burst_sz = tp->op_params->burst_sz; 4519 const uint16_t num_ops = tp->op_params->num_to_process; 4520 struct rte_bbdev_enc_op *ops_enq[num_ops]; 4521 struct rte_bbdev_enc_op *ops_deq[num_ops]; 4522 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 4523 struct test_buffers *bufs = NULL; 4524 int i, j, ret; 4525 struct rte_bbdev_info info; 4526 uint16_t num_to_enq; 4527 4528 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4529 "BURST_SIZE should be <= %u", MAX_BURST); 4530 4531 rte_bbdev_info_get(tp->dev_id, &info); 4532 4533 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4534 "NUM_OPS cannot exceed %u for this device", 4535 info.drv.queue_size_lim); 4536 4537 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4538 4539 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4540 rte_memory_order_relaxed); 4541 4542 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 4543 num_ops); 4544 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 4545 num_ops); 4546 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4547 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 4548 bufs->hard_outputs, ref_op); 4549 4550 /* Set counter to validate the ordering */ 4551 for (j = 0; j < num_ops; ++j) 4552 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4553 4554 for (i = 0; i < TEST_REPETITIONS; ++i) { 4555 uint32_t time_out = 0; 4556 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4557 for (j = 0; j < num_ops; ++j) 4558 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 4559 4560 start_time = rte_rdtsc_precise(); 4561 4562 for (enq = 0, deq = 0; enq < num_ops;) { 4563 num_to_enq = burst_sz; 4564 4565 if (unlikely(num_ops - enq < num_to_enq)) 4566 num_to_enq = num_ops - enq; 4567 4568 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4569 queue_id, &ops_enq[enq], num_to_enq); 4570 4571 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4572 queue_id, &ops_deq[deq], enq - deq); 4573 time_out++; 4574 if (time_out >= TIME_OUT_POLL) { 4575 timeout_exit(tp->dev_id); 4576 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4577 } 4578 } 4579 4580 /* dequeue the remaining */ 4581 time_out = 0; 4582 while (deq < enq) { 4583 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4584 queue_id, &ops_deq[deq], enq - deq); 4585 time_out++; 4586 if (time_out >= TIME_OUT_POLL) { 4587 timeout_exit(tp->dev_id); 4588 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4589 } 4590 } 4591 4592 total_time += rte_rdtsc_precise() - start_time; 4593 } 4594 4595 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4596 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4597 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4598 } 4599 4600 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4601 4602 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4603 4604 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4605 ((double)total_time / (double)rte_get_tsc_hz()); 4606 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4607 / 1000000.0) / ((double)total_time / 4608 (double)rte_get_tsc_hz()); 4609 4610 return TEST_SUCCESS; 4611 } 4612 4613 static int 4614 throughput_pmd_lcore_fft(void *arg) 4615 { 4616 struct thread_params *tp = arg; 4617 uint16_t enq, deq; 4618 uint64_t total_time = 0, start_time; 4619 const uint16_t queue_id = tp->queue_id; 4620 const uint16_t burst_sz = tp->op_params->burst_sz; 4621 const uint16_t num_ops = tp->op_params->num_to_process; 4622 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4623 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4624 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4625 struct test_buffers *bufs = NULL; 4626 int i, j, ret; 4627 struct rte_bbdev_info info; 4628 uint16_t num_to_enq; 4629 4630 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4631 "BURST_SIZE should be <= %u", MAX_BURST); 4632 4633 rte_bbdev_info_get(tp->dev_id, &info); 4634 4635 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4636 "NUM_OPS cannot exceed %u for this device", 4637 info.drv.queue_size_lim); 4638 4639 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4640 4641 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4642 rte_memory_order_relaxed); 4643 4644 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4645 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4646 4647 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4648 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4649 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, ref_op); 4650 4651 /* Set counter to validate the ordering */ 4652 for (j = 0; j < num_ops; ++j) 4653 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4654 4655 for (i = 0; i < TEST_REPETITIONS; ++i) { 4656 uint32_t time_out = 0; 4657 for (j = 0; j < num_ops; ++j) 4658 mbuf_reset(ops_enq[j]->fft.base_output.data); 4659 4660 start_time = rte_rdtsc_precise(); 4661 4662 for (enq = 0, deq = 0; enq < num_ops;) { 4663 num_to_enq = burst_sz; 4664 4665 if (unlikely(num_ops - enq < num_to_enq)) 4666 num_to_enq = num_ops - enq; 4667 4668 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4669 queue_id, &ops_enq[enq], num_to_enq); 4670 4671 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4672 queue_id, &ops_deq[deq], enq - deq); 4673 time_out++; 4674 if (time_out >= TIME_OUT_POLL) { 4675 timeout_exit(tp->dev_id); 4676 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4677 } 4678 } 4679 4680 /* dequeue the remaining */ 4681 time_out = 0; 4682 while (deq < enq) { 4683 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4684 queue_id, &ops_deq[deq], enq - deq); 4685 time_out++; 4686 if (time_out >= TIME_OUT_POLL) { 4687 timeout_exit(tp->dev_id); 4688 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4689 } 4690 } 4691 4692 total_time += rte_rdtsc_precise() - start_time; 4693 } 4694 4695 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4696 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4697 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4698 } 4699 4700 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4701 4702 double tb_len_bits = calc_fft_size(ref_op); 4703 4704 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4705 ((double)total_time / (double)rte_get_tsc_hz()); 4706 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4707 1000000.0) / ((double)total_time / 4708 (double)rte_get_tsc_hz()); 4709 4710 return TEST_SUCCESS; 4711 } 4712 4713 static int 4714 throughput_pmd_lcore_mldts(void *arg) 4715 { 4716 struct thread_params *tp = arg; 4717 uint16_t enq, deq; 4718 uint64_t total_time = 0, start_time; 4719 const uint16_t queue_id = tp->queue_id; 4720 const uint16_t burst_sz = tp->op_params->burst_sz; 4721 const uint16_t num_ops = tp->op_params->num_to_process; 4722 struct rte_bbdev_mldts_op *ops_enq[num_ops]; 4723 struct rte_bbdev_mldts_op *ops_deq[num_ops]; 4724 struct rte_bbdev_mldts_op *ref_op = tp->op_params->ref_mldts_op; 4725 struct test_buffers *bufs = NULL; 4726 int i, j, ret; 4727 struct rte_bbdev_info info; 4728 uint16_t num_to_enq; 4729 4730 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); 4731 4732 rte_bbdev_info_get(tp->dev_id, &info); 4733 4734 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4735 "NUM_OPS cannot exceed %u for this device", 4736 info.drv.queue_size_lim); 4737 4738 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4739 4740 rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tp->op_params->sync, SYNC_START, 4741 rte_memory_order_relaxed); 4742 4743 ret = rte_bbdev_mldts_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4744 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4745 4746 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4747 copy_reference_mldts_op(ops_enq, num_ops, 0, bufs->inputs, bufs->harq_inputs, 4748 bufs->hard_outputs, ref_op); 4749 4750 /* Set counter to validate the ordering */ 4751 for (j = 0; j < num_ops; ++j) 4752 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4753 4754 for (i = 0; i < TEST_REPETITIONS; ++i) { 4755 uint32_t time_out = 0; 4756 for (j = 0; j < num_ops; ++j) 4757 mbuf_reset(ops_enq[j]->mldts.output.data); 4758 4759 start_time = rte_rdtsc_precise(); 4760 4761 for (enq = 0, deq = 0; enq < num_ops;) { 4762 num_to_enq = burst_sz; 4763 4764 if (unlikely(num_ops - enq < num_to_enq)) 4765 num_to_enq = num_ops - enq; 4766 4767 enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, 4768 queue_id, &ops_enq[enq], num_to_enq); 4769 4770 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4771 queue_id, &ops_deq[deq], enq - deq); 4772 time_out++; 4773 if (time_out >= TIME_OUT_POLL) { 4774 timeout_exit(tp->dev_id); 4775 TEST_ASSERT_SUCCESS(TEST_FAILED, "Enqueue timeout!"); 4776 } 4777 } 4778 4779 /* dequeue the remaining */ 4780 time_out = 0; 4781 while (deq < enq) { 4782 deq += rte_bbdev_dequeue_mldts_ops(tp->dev_id, 4783 queue_id, &ops_deq[deq], enq - deq); 4784 time_out++; 4785 if (time_out >= TIME_OUT_POLL) { 4786 timeout_exit(tp->dev_id); 4787 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 4788 } 4789 } 4790 4791 total_time += rte_rdtsc_precise() - start_time; 4792 } 4793 4794 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4795 ret = validate_mldts_op(ops_deq, num_ops, ref_op); 4796 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4797 } 4798 4799 rte_bbdev_mldts_op_free_bulk(ops_enq, num_ops); 4800 4801 double tb_len_bits = calc_mldts_size(ref_op); 4802 4803 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4804 ((double)total_time / (double)rte_get_tsc_hz()); 4805 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4806 1000000.0) / ((double)total_time / 4807 (double)rte_get_tsc_hz()); 4808 4809 return TEST_SUCCESS; 4810 } 4811 4812 static void 4813 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4814 { 4815 unsigned int iter = 0; 4816 double total_mops = 0, total_mbps = 0; 4817 4818 for (iter = 0; iter < used_cores; iter++) { 4819 printf( 4820 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4821 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4822 t_params[iter].mbps); 4823 total_mops += t_params[iter].ops_per_sec; 4824 total_mbps += t_params[iter].mbps; 4825 } 4826 printf( 4827 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4828 used_cores, total_mops, total_mbps); 4829 } 4830 4831 /* Aggregate the performance results over the number of cores used */ 4832 static void 4833 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4834 { 4835 unsigned int core_idx = 0; 4836 double total_mops = 0, total_mbps = 0; 4837 uint8_t iter_count = 0; 4838 4839 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4840 printf( 4841 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4842 t_params[core_idx].lcore_id, 4843 t_params[core_idx].ops_per_sec, 4844 t_params[core_idx].mbps, 4845 t_params[core_idx].iter_count); 4846 total_mops += t_params[core_idx].ops_per_sec; 4847 total_mbps += t_params[core_idx].mbps; 4848 iter_count = RTE_MAX(iter_count, 4849 t_params[core_idx].iter_count); 4850 } 4851 printf( 4852 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4853 used_cores, total_mops, total_mbps, iter_count); 4854 } 4855 4856 /* Aggregate the performance results over the number of cores used */ 4857 static void 4858 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4859 { 4860 unsigned int core_idx = 0; 4861 double total_mbps = 0, total_bler = 0, total_iter = 0; 4862 double snr = get_snr(); 4863 4864 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4865 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4866 t_params[core_idx].lcore_id, 4867 t_params[core_idx].bler * 100, 4868 t_params[core_idx].iter_average, 4869 t_params[core_idx].mbps, 4870 get_vector_filename()); 4871 total_mbps += t_params[core_idx].mbps; 4872 total_bler += t_params[core_idx].bler; 4873 total_iter += t_params[core_idx].iter_average; 4874 } 4875 total_bler /= used_cores; 4876 total_iter /= used_cores; 4877 4878 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.3f Mbps %s\n", 4879 snr, total_bler * 100, total_iter, get_iter_max(), 4880 total_mbps, get_vector_filename()); 4881 } 4882 4883 /* 4884 * Test function that determines BLER wireless performance 4885 */ 4886 static int 4887 bler_test(struct active_device *ad, 4888 struct test_op_params *op_params) 4889 { 4890 int ret; 4891 unsigned int lcore_id, used_cores = 0; 4892 struct thread_params *t_params; 4893 struct rte_bbdev_info info; 4894 lcore_function_t *bler_function; 4895 uint16_t num_lcores; 4896 const char *op_type_str; 4897 4898 rte_bbdev_info_get(ad->dev_id, &info); 4899 4900 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4901 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4902 test_vector.op_type); 4903 4904 printf("+ ------------------------------------------------------- +\n"); 4905 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4906 info.dev_name, ad->nb_queues, op_params->burst_sz, 4907 op_params->num_to_process, op_params->num_lcores, 4908 op_type_str, 4909 intr_enabled ? "Interrupt mode" : "PMD mode", 4910 (double)rte_get_tsc_hz() / 1000000000.0); 4911 4912 /* Set number of lcores */ 4913 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4914 ? ad->nb_queues 4915 : op_params->num_lcores; 4916 4917 /* Allocate memory for thread parameters structure */ 4918 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4919 RTE_CACHE_LINE_SIZE); 4920 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4921 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4922 RTE_CACHE_LINE_SIZE)); 4923 4924 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 4925 !check_bit(test_vector.ldpc_dec.op_flags, 4926 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 4927 && !check_bit(test_vector.ldpc_dec.op_flags, 4928 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 4929 bler_function = bler_pmd_lcore_ldpc_dec; 4930 else if ((test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) && 4931 !check_bit(test_vector.turbo_dec.op_flags, 4932 RTE_BBDEV_TURBO_SOFT_OUTPUT)) 4933 bler_function = bler_pmd_lcore_turbo_dec; 4934 else 4935 return TEST_SKIPPED; 4936 4937 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 4938 4939 /* Main core is set at first entry */ 4940 t_params[0].dev_id = ad->dev_id; 4941 t_params[0].lcore_id = rte_lcore_id(); 4942 t_params[0].op_params = op_params; 4943 t_params[0].queue_id = ad->queue_ids[used_cores++]; 4944 t_params[0].iter_count = 0; 4945 4946 RTE_LCORE_FOREACH_WORKER(lcore_id) { 4947 if (used_cores >= num_lcores) 4948 break; 4949 4950 t_params[used_cores].dev_id = ad->dev_id; 4951 t_params[used_cores].lcore_id = lcore_id; 4952 t_params[used_cores].op_params = op_params; 4953 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 4954 t_params[used_cores].iter_count = 0; 4955 4956 rte_eal_remote_launch(bler_function, 4957 &t_params[used_cores++], lcore_id); 4958 } 4959 4960 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 4961 ret = bler_function(&t_params[0]); 4962 4963 /* Main core is always used */ 4964 for (used_cores = 1; used_cores < num_lcores; used_cores++) 4965 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 4966 4967 print_dec_bler(t_params, num_lcores); 4968 4969 /* Return if test failed */ 4970 if (ret) { 4971 rte_free(t_params); 4972 return ret; 4973 } 4974 4975 /* Function to print something here*/ 4976 rte_free(t_params); 4977 return ret; 4978 } 4979 4980 /* 4981 * Test function that determines how long an enqueue + dequeue of a burst 4982 * takes on available lcores. 4983 */ 4984 static int 4985 throughput_test(struct active_device *ad, 4986 struct test_op_params *op_params) 4987 { 4988 int ret; 4989 unsigned int lcore_id, used_cores = 0; 4990 struct thread_params *t_params, *tp; 4991 struct rte_bbdev_info info; 4992 lcore_function_t *throughput_function; 4993 uint16_t num_lcores; 4994 const char *op_type_str; 4995 4996 rte_bbdev_info_get(ad->dev_id, &info); 4997 4998 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4999 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 5000 test_vector.op_type); 5001 5002 printf("+ ------------------------------------------------------- +\n"); 5003 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 5004 info.dev_name, ad->nb_queues, op_params->burst_sz, 5005 op_params->num_to_process, op_params->num_lcores, 5006 op_type_str, 5007 intr_enabled ? "Interrupt mode" : "PMD mode", 5008 (double)rte_get_tsc_hz() / 1000000000.0); 5009 5010 /* Set number of lcores */ 5011 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 5012 ? ad->nb_queues 5013 : op_params->num_lcores; 5014 5015 /* Allocate memory for thread parameters structure */ 5016 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 5017 RTE_CACHE_LINE_SIZE); 5018 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 5019 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 5020 RTE_CACHE_LINE_SIZE)); 5021 5022 if (intr_enabled) { 5023 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5024 throughput_function = throughput_intr_lcore_dec; 5025 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5026 throughput_function = throughput_intr_lcore_ldpc_dec; 5027 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5028 throughput_function = throughput_intr_lcore_enc; 5029 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5030 throughput_function = throughput_intr_lcore_ldpc_enc; 5031 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5032 throughput_function = throughput_intr_lcore_fft; 5033 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5034 throughput_function = throughput_intr_lcore_mldts; 5035 else 5036 throughput_function = throughput_intr_lcore_enc; 5037 5038 /* Dequeue interrupt callback registration */ 5039 ret = rte_bbdev_callback_register(ad->dev_id, 5040 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 5041 t_params); 5042 if (ret < 0) { 5043 rte_free(t_params); 5044 return ret; 5045 } 5046 } else { 5047 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 5048 throughput_function = throughput_pmd_lcore_dec; 5049 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5050 throughput_function = throughput_pmd_lcore_ldpc_dec; 5051 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 5052 throughput_function = throughput_pmd_lcore_enc; 5053 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5054 throughput_function = throughput_pmd_lcore_ldpc_enc; 5055 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 5056 throughput_function = throughput_pmd_lcore_fft; 5057 else if (test_vector.op_type == RTE_BBDEV_OP_MLDTS) 5058 throughput_function = throughput_pmd_lcore_mldts; 5059 else 5060 throughput_function = throughput_pmd_lcore_enc; 5061 } 5062 5063 rte_atomic_store_explicit(&op_params->sync, SYNC_WAIT, rte_memory_order_relaxed); 5064 5065 /* Main core is set at first entry */ 5066 t_params[0].dev_id = ad->dev_id; 5067 t_params[0].lcore_id = rte_lcore_id(); 5068 t_params[0].op_params = op_params; 5069 t_params[0].queue_id = ad->queue_ids[used_cores++]; 5070 t_params[0].iter_count = 0; 5071 5072 RTE_LCORE_FOREACH_WORKER(lcore_id) { 5073 if (used_cores >= num_lcores) 5074 break; 5075 5076 t_params[used_cores].dev_id = ad->dev_id; 5077 t_params[used_cores].lcore_id = lcore_id; 5078 t_params[used_cores].op_params = op_params; 5079 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 5080 t_params[used_cores].iter_count = 0; 5081 5082 rte_eal_remote_launch(throughput_function, 5083 &t_params[used_cores++], lcore_id); 5084 } 5085 5086 rte_atomic_store_explicit(&op_params->sync, SYNC_START, rte_memory_order_relaxed); 5087 ret = throughput_function(&t_params[0]); 5088 5089 /* Main core is always used */ 5090 for (used_cores = 1; used_cores < num_lcores; used_cores++) 5091 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 5092 5093 /* Return if test failed */ 5094 if (ret) { 5095 rte_free(t_params); 5096 return ret; 5097 } 5098 5099 /* Print throughput if interrupts are disabled and test passed */ 5100 if (!intr_enabled) { 5101 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5102 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5103 print_dec_throughput(t_params, num_lcores); 5104 else 5105 print_enc_throughput(t_params, num_lcores); 5106 rte_free(t_params); 5107 return ret; 5108 } 5109 5110 /* In interrupt TC we need to wait for the interrupt callback to deqeue 5111 * all pending operations. Skip waiting for queues which reported an 5112 * error using processing_status variable. 5113 * Wait for main lcore operations. 5114 */ 5115 tp = &t_params[0]; 5116 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5117 op_params->num_to_process) && 5118 (rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed) != 5119 TEST_FAILED)) 5120 rte_pause(); 5121 5122 tp->ops_per_sec /= TEST_REPETITIONS; 5123 tp->mbps /= TEST_REPETITIONS; 5124 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, rte_memory_order_relaxed); 5125 5126 /* Wait for worker lcores operations */ 5127 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 5128 tp = &t_params[used_cores]; 5129 5130 while ((rte_atomic_load_explicit(&tp->nb_dequeued, rte_memory_order_relaxed) < 5131 op_params->num_to_process) && 5132 (rte_atomic_load_explicit(&tp->processing_status, 5133 rte_memory_order_relaxed) != TEST_FAILED)) 5134 rte_pause(); 5135 5136 tp->ops_per_sec /= TEST_REPETITIONS; 5137 tp->mbps /= TEST_REPETITIONS; 5138 ret |= (int)rte_atomic_load_explicit(&tp->processing_status, 5139 rte_memory_order_relaxed); 5140 } 5141 5142 /* Print throughput if test passed */ 5143 if (!ret) { 5144 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 5145 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 5146 print_dec_throughput(t_params, num_lcores); 5147 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 5148 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 5149 print_enc_throughput(t_params, num_lcores); 5150 } 5151 5152 rte_free(t_params); 5153 return ret; 5154 } 5155 5156 static int 5157 latency_test_dec(struct rte_mempool *mempool, 5158 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5159 uint16_t dev_id, uint16_t queue_id, 5160 const uint16_t num_to_process, uint16_t burst_sz, 5161 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, bool disable_et) 5162 { 5163 int ret = TEST_SUCCESS; 5164 uint16_t i, j, dequeued; 5165 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5166 uint64_t start_time = 0, last_time = 0; 5167 5168 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5169 uint16_t enq = 0, deq = 0; 5170 uint32_t time_out = 0; 5171 bool first_time = true; 5172 last_time = 0; 5173 5174 if (unlikely(num_to_process - dequeued < burst_sz)) 5175 burst_sz = num_to_process - dequeued; 5176 5177 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5178 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5179 5180 ref_op->turbo_dec.iter_max = get_iter_max(); 5181 /* For validation tests we want to enable early termination */ 5182 if (!disable_et && !check_bit(ref_op->turbo_dec.op_flags, 5183 RTE_BBDEV_TURBO_EARLY_TERMINATION)) 5184 ref_op->turbo_dec.op_flags |= RTE_BBDEV_TURBO_EARLY_TERMINATION; 5185 5186 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5187 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5188 bufs->inputs, 5189 bufs->hard_outputs, 5190 bufs->soft_outputs, 5191 ref_op); 5192 5193 /* Set counter to validate the ordering */ 5194 for (j = 0; j < burst_sz; ++j) 5195 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5196 5197 start_time = rte_rdtsc_precise(); 5198 5199 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 5200 burst_sz); 5201 TEST_ASSERT(enq == burst_sz, 5202 "Error enqueueing burst, expected %u, got %u", 5203 burst_sz, enq); 5204 5205 /* Dequeue */ 5206 do { 5207 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5208 &ops_deq[deq], burst_sz - deq); 5209 if (likely(first_time && (deq > 0))) { 5210 last_time = rte_rdtsc_precise() - start_time; 5211 first_time = false; 5212 } 5213 time_out++; 5214 if (time_out >= TIME_OUT_POLL) { 5215 timeout_exit(dev_id); 5216 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5217 } 5218 } while (unlikely(burst_sz != deq)); 5219 5220 *max_time = RTE_MAX(*max_time, last_time); 5221 *min_time = RTE_MIN(*min_time, last_time); 5222 *total_time += last_time; 5223 5224 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5225 ret = validate_dec_op(ops_deq, burst_sz, ref_op); 5226 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5227 } 5228 5229 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5230 dequeued += deq; 5231 } 5232 5233 return i; 5234 } 5235 5236 /* Test case for latency/validation for LDPC Decoder */ 5237 static int 5238 latency_test_ldpc_dec(struct rte_mempool *mempool, 5239 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 5240 int vector_mask, uint16_t dev_id, uint16_t queue_id, 5241 const uint16_t num_to_process, uint16_t burst_sz, 5242 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 5243 bool disable_et) 5244 { 5245 int ret = TEST_SUCCESS; 5246 uint16_t i, j, dequeued; 5247 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5248 uint64_t start_time = 0, last_time = 0; 5249 bool extDdr = ldpc_cap_flags & 5250 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5251 5252 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5253 uint16_t enq = 0, deq = 0; 5254 uint32_t time_out = 0; 5255 bool first_time = true; 5256 last_time = 0; 5257 5258 if (unlikely(num_to_process - dequeued < burst_sz)) 5259 burst_sz = num_to_process - dequeued; 5260 5261 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5262 TEST_ASSERT_SUCCESS(ret, 5263 "rte_bbdev_dec_op_alloc_bulk() failed"); 5264 5265 /* For latency tests we need to disable early termination */ 5266 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 5267 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 5268 ref_op->ldpc_dec.op_flags -= RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 5269 5270 ref_op->ldpc_dec.iter_max = get_iter_max(); 5271 /* When ET is disabled, the expected iter_count is iter_max */ 5272 if (disable_et) 5273 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 5274 5275 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5276 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5277 bufs->inputs, 5278 bufs->hard_outputs, 5279 bufs->soft_outputs, 5280 bufs->harq_inputs, 5281 bufs->harq_outputs, 5282 ref_op); 5283 5284 if (extDdr) 5285 preload_harq_ddr(dev_id, queue_id, ops_enq, 5286 burst_sz, true); 5287 5288 /* Set counter to validate the ordering */ 5289 for (j = 0; j < burst_sz; ++j) 5290 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5291 5292 start_time = rte_rdtsc_precise(); 5293 5294 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5295 &ops_enq[enq], burst_sz); 5296 TEST_ASSERT(enq == burst_sz, 5297 "Error enqueueing burst, expected %u, got %u", 5298 burst_sz, enq); 5299 5300 /* Dequeue */ 5301 do { 5302 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5303 &ops_deq[deq], burst_sz - deq); 5304 if (likely(first_time && (deq > 0))) { 5305 last_time = rte_rdtsc_precise() - start_time; 5306 first_time = false; 5307 } 5308 time_out++; 5309 if (time_out >= TIME_OUT_POLL) { 5310 timeout_exit(dev_id); 5311 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5312 } 5313 } while (unlikely(burst_sz != deq)); 5314 5315 *max_time = RTE_MAX(*max_time, last_time); 5316 *min_time = RTE_MIN(*min_time, last_time); 5317 *total_time += last_time; 5318 5319 if (extDdr) 5320 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5321 5322 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5323 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, vector_mask); 5324 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5325 } 5326 5327 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5328 dequeued += deq; 5329 } 5330 return i; 5331 } 5332 5333 static int 5334 latency_test_enc(struct rte_mempool *mempool, 5335 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5336 uint16_t dev_id, uint16_t queue_id, 5337 const uint16_t num_to_process, uint16_t burst_sz, 5338 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5339 { 5340 int ret = TEST_SUCCESS; 5341 uint16_t i, j, dequeued; 5342 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5343 uint64_t start_time = 0, last_time = 0; 5344 5345 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5346 uint16_t enq = 0, deq = 0; 5347 uint32_t time_out = 0; 5348 bool first_time = true; 5349 last_time = 0; 5350 5351 if (unlikely(num_to_process - dequeued < burst_sz)) 5352 burst_sz = num_to_process - dequeued; 5353 5354 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5355 TEST_ASSERT_SUCCESS(ret, 5356 "rte_bbdev_enc_op_alloc_bulk() failed"); 5357 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5358 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5359 bufs->inputs, 5360 bufs->hard_outputs, 5361 ref_op); 5362 5363 /* Set counter to validate the ordering */ 5364 for (j = 0; j < burst_sz; ++j) 5365 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5366 5367 start_time = rte_rdtsc_precise(); 5368 5369 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 5370 burst_sz); 5371 TEST_ASSERT(enq == burst_sz, 5372 "Error enqueueing burst, expected %u, got %u", 5373 burst_sz, enq); 5374 5375 /* Dequeue */ 5376 do { 5377 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5378 &ops_deq[deq], burst_sz - deq); 5379 if (likely(first_time && (deq > 0))) { 5380 last_time += rte_rdtsc_precise() - start_time; 5381 first_time = false; 5382 } 5383 time_out++; 5384 if (time_out >= TIME_OUT_POLL) { 5385 timeout_exit(dev_id); 5386 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5387 } 5388 } while (unlikely(burst_sz != deq)); 5389 5390 *max_time = RTE_MAX(*max_time, last_time); 5391 *min_time = RTE_MIN(*min_time, last_time); 5392 *total_time += last_time; 5393 5394 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5395 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5396 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5397 } 5398 5399 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5400 dequeued += deq; 5401 } 5402 5403 return i; 5404 } 5405 5406 static int 5407 latency_test_ldpc_enc(struct rte_mempool *mempool, 5408 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 5409 uint16_t dev_id, uint16_t queue_id, 5410 const uint16_t num_to_process, uint16_t burst_sz, 5411 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5412 { 5413 int ret = TEST_SUCCESS; 5414 uint16_t i, j, dequeued; 5415 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5416 uint64_t start_time = 0, last_time = 0; 5417 5418 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5419 uint16_t enq = 0, deq = 0; 5420 uint32_t time_out = 0; 5421 bool first_time = true; 5422 last_time = 0; 5423 5424 if (unlikely(num_to_process - dequeued < burst_sz)) 5425 burst_sz = num_to_process - dequeued; 5426 5427 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5428 TEST_ASSERT_SUCCESS(ret, 5429 "rte_bbdev_enc_op_alloc_bulk() failed"); 5430 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5431 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5432 bufs->inputs, 5433 bufs->hard_outputs, 5434 ref_op); 5435 5436 /* Set counter to validate the ordering */ 5437 for (j = 0; j < burst_sz; ++j) 5438 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5439 5440 start_time = rte_rdtsc_precise(); 5441 5442 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5443 &ops_enq[enq], burst_sz); 5444 TEST_ASSERT(enq == burst_sz, 5445 "Error enqueueing burst, expected %u, got %u", 5446 burst_sz, enq); 5447 5448 /* Dequeue */ 5449 do { 5450 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5451 &ops_deq[deq], burst_sz - deq); 5452 if (likely(first_time && (deq > 0))) { 5453 last_time += rte_rdtsc_precise() - start_time; 5454 first_time = false; 5455 } 5456 time_out++; 5457 if (time_out >= TIME_OUT_POLL) { 5458 timeout_exit(dev_id); 5459 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5460 } 5461 } while (unlikely(burst_sz != deq)); 5462 5463 *max_time = RTE_MAX(*max_time, last_time); 5464 *min_time = RTE_MIN(*min_time, last_time); 5465 *total_time += last_time; 5466 5467 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5468 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 5469 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5470 } 5471 5472 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5473 dequeued += deq; 5474 } 5475 5476 return i; 5477 } 5478 5479 5480 static int 5481 latency_test_fft(struct rte_mempool *mempool, 5482 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 5483 uint16_t dev_id, uint16_t queue_id, 5484 const uint16_t num_to_process, uint16_t burst_sz, 5485 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5486 { 5487 int ret = TEST_SUCCESS; 5488 uint16_t i, j, dequeued; 5489 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5490 uint64_t start_time = 0, last_time = 0; 5491 5492 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5493 uint16_t enq = 0, deq = 0; 5494 uint32_t time_out = 0; 5495 bool first_time = true; 5496 last_time = 0; 5497 5498 if (unlikely(num_to_process - dequeued < burst_sz)) 5499 burst_sz = num_to_process - dequeued; 5500 5501 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5502 TEST_ASSERT_SUCCESS(ret, 5503 "rte_bbdev_fft_op_alloc_bulk() failed"); 5504 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5505 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5506 bufs->inputs, 5507 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5508 ref_op); 5509 5510 /* Set counter to validate the ordering */ 5511 for (j = 0; j < burst_sz; ++j) 5512 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5513 5514 start_time = rte_rdtsc_precise(); 5515 5516 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5517 &ops_enq[enq], burst_sz); 5518 TEST_ASSERT(enq == burst_sz, 5519 "Error enqueueing burst, expected %u, got %u", 5520 burst_sz, enq); 5521 5522 /* Dequeue */ 5523 do { 5524 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5525 &ops_deq[deq], burst_sz - deq); 5526 if (likely(first_time && (deq > 0))) { 5527 last_time += rte_rdtsc_precise() - start_time; 5528 first_time = false; 5529 } 5530 time_out++; 5531 if (time_out >= TIME_OUT_POLL) { 5532 timeout_exit(dev_id); 5533 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5534 } 5535 } while (unlikely(burst_sz != deq)); 5536 5537 *max_time = RTE_MAX(*max_time, last_time); 5538 *min_time = RTE_MIN(*min_time, last_time); 5539 *total_time += last_time; 5540 5541 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5542 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 5543 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5544 } 5545 5546 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5547 dequeued += deq; 5548 } 5549 5550 return i; 5551 } 5552 5553 static int 5554 latency_test_mldts(struct rte_mempool *mempool, 5555 struct test_buffers *bufs, struct rte_bbdev_mldts_op *ref_op, 5556 uint16_t dev_id, uint16_t queue_id, 5557 const uint16_t num_to_process, uint16_t burst_sz, 5558 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 5559 { 5560 int ret = TEST_SUCCESS; 5561 uint16_t i, j, dequeued; 5562 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5563 uint64_t start_time = 0, last_time = 0; 5564 5565 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5566 uint16_t enq = 0, deq = 0; 5567 uint32_t time_out = 0; 5568 bool first_time = true; 5569 last_time = 0; 5570 5571 if (unlikely(num_to_process - dequeued < burst_sz)) 5572 burst_sz = num_to_process - dequeued; 5573 5574 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5575 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5576 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5577 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5578 bufs->inputs, bufs->harq_inputs, 5579 bufs->hard_outputs, 5580 ref_op); 5581 5582 /* Set counter to validate the ordering */ 5583 for (j = 0; j < burst_sz; ++j) 5584 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 5585 5586 start_time = rte_rdtsc_precise(); 5587 5588 enq = rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, &ops_enq[enq], burst_sz); 5589 TEST_ASSERT(enq == burst_sz, 5590 "Error enqueueing burst, expected %u, got %u", 5591 burst_sz, enq); 5592 5593 /* Dequeue */ 5594 do { 5595 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5596 &ops_deq[deq], burst_sz - deq); 5597 if (likely(first_time && (deq > 0))) { 5598 last_time += rte_rdtsc_precise() - start_time; 5599 first_time = false; 5600 } 5601 time_out++; 5602 if (time_out >= TIME_OUT_POLL) { 5603 timeout_exit(dev_id); 5604 TEST_ASSERT_SUCCESS(TEST_FAILED, "Dequeue timeout!"); 5605 } 5606 } while (unlikely(burst_sz != deq)); 5607 5608 *max_time = RTE_MAX(*max_time, last_time); 5609 *min_time = RTE_MIN(*min_time, last_time); 5610 *total_time += last_time; 5611 5612 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 5613 ret = validate_mldts_op(ops_deq, burst_sz, ref_op); 5614 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 5615 } 5616 5617 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5618 dequeued += deq; 5619 } 5620 5621 return i; 5622 } 5623 5624 /* Common function for running validation and latency test cases */ 5625 static int 5626 validation_latency_test(struct active_device *ad, 5627 struct test_op_params *op_params, bool latency_flag) 5628 { 5629 int iter; 5630 uint16_t burst_sz = op_params->burst_sz; 5631 const uint16_t num_to_process = op_params->num_to_process; 5632 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5633 const uint16_t queue_id = ad->queue_ids[0]; 5634 struct test_buffers *bufs = NULL; 5635 struct rte_bbdev_info info; 5636 uint64_t total_time, min_time, max_time; 5637 const char *op_type_str; 5638 5639 total_time = max_time = 0; 5640 min_time = UINT64_MAX; 5641 5642 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5643 "BURST_SIZE should be <= %u", MAX_BURST); 5644 5645 rte_bbdev_info_get(ad->dev_id, &info); 5646 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5647 5648 op_type_str = rte_bbdev_op_type_str(op_type); 5649 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5650 5651 printf("+ ------------------------------------------------------- +\n"); 5652 if (latency_flag) 5653 printf("== test: latency\ndev:"); 5654 else 5655 printf("== test: validation\ndev:"); 5656 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 5657 info.dev_name, burst_sz, num_to_process, op_type_str); 5658 5659 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5660 iter = latency_test_dec(op_params->mp, bufs, 5661 op_params->ref_dec_op, ad->dev_id, queue_id, 5662 num_to_process, burst_sz, &total_time, 5663 &min_time, &max_time, latency_flag); 5664 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5665 iter = latency_test_ldpc_enc(op_params->mp, bufs, 5666 op_params->ref_enc_op, ad->dev_id, queue_id, 5667 num_to_process, burst_sz, &total_time, 5668 &min_time, &max_time); 5669 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5670 iter = latency_test_ldpc_dec(op_params->mp, bufs, 5671 op_params->ref_dec_op, op_params->vector_mask, 5672 ad->dev_id, queue_id, num_to_process, 5673 burst_sz, &total_time, &min_time, &max_time, 5674 latency_flag); 5675 else if (op_type == RTE_BBDEV_OP_FFT) 5676 iter = latency_test_fft(op_params->mp, bufs, 5677 op_params->ref_fft_op, 5678 ad->dev_id, queue_id, 5679 num_to_process, burst_sz, &total_time, 5680 &min_time, &max_time); 5681 else if (op_type == RTE_BBDEV_OP_MLDTS) 5682 iter = latency_test_mldts(op_params->mp, bufs, 5683 op_params->ref_mldts_op, 5684 ad->dev_id, queue_id, 5685 num_to_process, burst_sz, &total_time, 5686 &min_time, &max_time); 5687 else /* RTE_BBDEV_OP_TURBO_ENC */ 5688 iter = latency_test_enc(op_params->mp, bufs, 5689 op_params->ref_enc_op, 5690 ad->dev_id, queue_id, 5691 num_to_process, burst_sz, &total_time, 5692 &min_time, &max_time); 5693 5694 if (iter <= 0) 5695 return TEST_FAILED; 5696 5697 printf("Operation latency:\n" 5698 "\tavg: %lg cycles, %lg us\n" 5699 "\tmin: %lg cycles, %lg us\n" 5700 "\tmax: %lg cycles, %lg us\n", 5701 (double)total_time / (double)iter, 5702 (double)(total_time * 1000000) / (double)iter / 5703 (double)rte_get_tsc_hz(), (double)min_time, 5704 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 5705 (double)max_time, (double)(max_time * 1000000) / 5706 (double)rte_get_tsc_hz()); 5707 5708 return TEST_SUCCESS; 5709 } 5710 5711 static int 5712 latency_test(struct active_device *ad, struct test_op_params *op_params) 5713 { 5714 return validation_latency_test(ad, op_params, true); 5715 } 5716 5717 static int 5718 validation_test(struct active_device *ad, struct test_op_params *op_params) 5719 { 5720 return validation_latency_test(ad, op_params, false); 5721 } 5722 5723 static int 5724 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 5725 struct rte_bbdev_stats *stats) 5726 { 5727 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 5728 struct rte_bbdev_stats *q_stats; 5729 5730 if (queue_id >= dev->data->num_queues) 5731 return -1; 5732 5733 q_stats = &dev->data->queues[queue_id].queue_stats; 5734 5735 stats->enqueued_count = q_stats->enqueued_count; 5736 stats->dequeued_count = q_stats->dequeued_count; 5737 stats->enqueue_err_count = q_stats->enqueue_err_count; 5738 stats->dequeue_err_count = q_stats->dequeue_err_count; 5739 stats->enqueue_warn_count = q_stats->enqueue_warn_count; 5740 stats->dequeue_warn_count = q_stats->dequeue_warn_count; 5741 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 5742 5743 return 0; 5744 } 5745 5746 static int 5747 offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, 5748 struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, 5749 uint16_t queue_id, const uint16_t num_to_process, 5750 uint16_t burst_sz, struct test_time_stats *time_st) 5751 { 5752 int i, dequeued, ret; 5753 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5754 uint64_t enq_start_time, deq_start_time; 5755 uint64_t enq_sw_last_time, deq_last_time; 5756 struct rte_bbdev_stats stats; 5757 5758 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5759 uint16_t enq = 0, deq = 0; 5760 5761 if (unlikely(num_to_process - dequeued < burst_sz)) 5762 burst_sz = num_to_process - dequeued; 5763 5764 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 5765 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_fft_op_alloc_bulk() failed"); 5766 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5767 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 5768 bufs->inputs, 5769 bufs->hard_outputs, bufs->soft_outputs, bufs->harq_inputs, 5770 ref_op); 5771 5772 /* Start time meas for enqueue function offload latency */ 5773 enq_start_time = rte_rdtsc_precise(); 5774 do { 5775 enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 5776 &ops_enq[enq], burst_sz - enq); 5777 } while (unlikely(burst_sz != enq)); 5778 5779 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5780 TEST_ASSERT_SUCCESS(ret, 5781 "Failed to get stats for queue (%u) of device (%u)", 5782 queue_id, dev_id); 5783 5784 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5785 stats.acc_offload_cycles; 5786 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5787 enq_sw_last_time); 5788 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5789 enq_sw_last_time); 5790 time_st->enq_sw_total_time += enq_sw_last_time; 5791 5792 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5793 stats.acc_offload_cycles); 5794 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5795 stats.acc_offload_cycles); 5796 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5797 5798 /* give time for device to process ops */ 5799 rte_delay_us(WAIT_OFFLOAD_US); 5800 5801 /* Start time meas for dequeue function offload latency */ 5802 deq_start_time = rte_rdtsc_precise(); 5803 /* Dequeue one operation */ 5804 do { 5805 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5806 &ops_deq[deq], enq); 5807 } while (unlikely(deq == 0)); 5808 5809 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5810 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5811 deq_last_time); 5812 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5813 deq_last_time); 5814 time_st->deq_total_time += deq_last_time; 5815 5816 /* Dequeue remaining operations if needed*/ 5817 while (burst_sz != deq) 5818 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 5819 &ops_deq[deq], burst_sz - deq); 5820 5821 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 5822 dequeued += deq; 5823 } 5824 5825 return i; 5826 } 5827 5828 static int 5829 offload_latency_test_mldts(struct rte_mempool *mempool, struct test_buffers *bufs, 5830 struct rte_bbdev_mldts_op *ref_op, uint16_t dev_id, 5831 uint16_t queue_id, const uint16_t num_to_process, 5832 uint16_t burst_sz, struct test_time_stats *time_st) 5833 { 5834 int i, dequeued, ret; 5835 struct rte_bbdev_mldts_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5836 uint64_t enq_start_time, deq_start_time; 5837 uint64_t enq_sw_last_time, deq_last_time; 5838 struct rte_bbdev_stats stats; 5839 5840 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5841 uint16_t enq = 0, deq = 0; 5842 5843 if (unlikely(num_to_process - dequeued < burst_sz)) 5844 burst_sz = num_to_process - dequeued; 5845 5846 ret = rte_bbdev_mldts_op_alloc_bulk(mempool, ops_enq, burst_sz); 5847 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_mldts_op_alloc_bulk() failed"); 5848 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5849 copy_reference_mldts_op(ops_enq, burst_sz, dequeued, 5850 bufs->inputs, bufs->harq_inputs, 5851 bufs->hard_outputs, 5852 ref_op); 5853 5854 /* Start time meas for enqueue function offload latency */ 5855 enq_start_time = rte_rdtsc_precise(); 5856 do { 5857 enq += rte_bbdev_enqueue_mldts_ops(dev_id, queue_id, 5858 &ops_enq[enq], burst_sz - enq); 5859 } while (unlikely(burst_sz != enq)); 5860 5861 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5862 TEST_ASSERT_SUCCESS(ret, 5863 "Failed to get stats for queue (%u) of device (%u)", 5864 queue_id, dev_id); 5865 5866 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5867 stats.acc_offload_cycles; 5868 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5869 enq_sw_last_time); 5870 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5871 enq_sw_last_time); 5872 time_st->enq_sw_total_time += enq_sw_last_time; 5873 5874 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5875 stats.acc_offload_cycles); 5876 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5877 stats.acc_offload_cycles); 5878 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5879 5880 /* give time for device to process ops */ 5881 rte_delay_us(WAIT_OFFLOAD_US); 5882 5883 /* Start time meas for dequeue function offload latency */ 5884 deq_start_time = rte_rdtsc_precise(); 5885 /* Dequeue one operation */ 5886 do { 5887 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, &ops_deq[deq], enq); 5888 } while (unlikely(deq == 0)); 5889 5890 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5891 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, deq_last_time); 5892 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, deq_last_time); 5893 time_st->deq_total_time += deq_last_time; 5894 5895 /* Dequeue remaining operations if needed*/ 5896 while (burst_sz != deq) 5897 deq += rte_bbdev_dequeue_mldts_ops(dev_id, queue_id, 5898 &ops_deq[deq], burst_sz - deq); 5899 5900 rte_bbdev_mldts_op_free_bulk(ops_enq, deq); 5901 dequeued += deq; 5902 } 5903 5904 return i; 5905 } 5906 5907 static int 5908 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 5909 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5910 uint16_t queue_id, const uint16_t num_to_process, 5911 uint16_t burst_sz, struct test_time_stats *time_st) 5912 { 5913 int i, dequeued, ret; 5914 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5915 uint64_t enq_start_time, deq_start_time; 5916 uint64_t enq_sw_last_time, deq_last_time; 5917 struct rte_bbdev_stats stats; 5918 5919 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5920 uint16_t enq = 0, deq = 0; 5921 5922 if (unlikely(num_to_process - dequeued < burst_sz)) 5923 burst_sz = num_to_process - dequeued; 5924 5925 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5926 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 5927 ref_op->turbo_dec.iter_max = get_iter_max(); 5928 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5929 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 5930 bufs->inputs, 5931 bufs->hard_outputs, 5932 bufs->soft_outputs, 5933 ref_op); 5934 5935 /* Start time meas for enqueue function offload latency */ 5936 enq_start_time = rte_rdtsc_precise(); 5937 do { 5938 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 5939 &ops_enq[enq], burst_sz - enq); 5940 } while (unlikely(burst_sz != enq)); 5941 5942 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5943 TEST_ASSERT_SUCCESS(ret, 5944 "Failed to get stats for queue (%u) of device (%u)", 5945 queue_id, dev_id); 5946 5947 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 5948 stats.acc_offload_cycles; 5949 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5950 enq_sw_last_time); 5951 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5952 enq_sw_last_time); 5953 time_st->enq_sw_total_time += enq_sw_last_time; 5954 5955 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5956 stats.acc_offload_cycles); 5957 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5958 stats.acc_offload_cycles); 5959 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5960 5961 /* give time for device to process ops */ 5962 rte_delay_us(WAIT_OFFLOAD_US); 5963 5964 /* Start time meas for dequeue function offload latency */ 5965 deq_start_time = rte_rdtsc_precise(); 5966 /* Dequeue one operation */ 5967 do { 5968 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5969 &ops_deq[deq], enq); 5970 } while (unlikely(deq == 0)); 5971 5972 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5973 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5974 deq_last_time); 5975 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5976 deq_last_time); 5977 time_st->deq_total_time += deq_last_time; 5978 5979 /* Dequeue remaining operations if needed*/ 5980 while (burst_sz != deq) 5981 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5982 &ops_deq[deq], burst_sz - deq); 5983 5984 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5985 dequeued += deq; 5986 } 5987 5988 return i; 5989 } 5990 5991 static int 5992 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 5993 struct test_buffers *bufs, 5994 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5995 uint16_t queue_id, const uint16_t num_to_process, 5996 uint16_t burst_sz, struct test_time_stats *time_st) 5997 { 5998 int i, dequeued, ret; 5999 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6000 uint64_t enq_start_time, deq_start_time; 6001 uint64_t enq_sw_last_time, deq_last_time; 6002 struct rte_bbdev_stats stats; 6003 bool extDdr = ldpc_cap_flags & 6004 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 6005 6006 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6007 uint16_t enq = 0, deq = 0; 6008 6009 if (unlikely(num_to_process - dequeued < burst_sz)) 6010 burst_sz = num_to_process - dequeued; 6011 6012 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 6013 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_dec_op_alloc_bulk() failed"); 6014 ref_op->ldpc_dec.iter_max = get_iter_max(); 6015 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6016 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 6017 bufs->inputs, 6018 bufs->hard_outputs, 6019 bufs->soft_outputs, 6020 bufs->harq_inputs, 6021 bufs->harq_outputs, 6022 ref_op); 6023 6024 if (extDdr) 6025 preload_harq_ddr(dev_id, queue_id, ops_enq, 6026 burst_sz, true); 6027 6028 /* Start time meas for enqueue function offload latency */ 6029 enq_start_time = rte_rdtsc_precise(); 6030 do { 6031 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 6032 &ops_enq[enq], burst_sz - enq); 6033 } while (unlikely(burst_sz != enq)); 6034 6035 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6036 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6037 TEST_ASSERT_SUCCESS(ret, 6038 "Failed to get stats for queue (%u) of device (%u)", 6039 queue_id, dev_id); 6040 6041 enq_sw_last_time -= stats.acc_offload_cycles; 6042 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6043 enq_sw_last_time); 6044 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6045 enq_sw_last_time); 6046 time_st->enq_sw_total_time += enq_sw_last_time; 6047 6048 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6049 stats.acc_offload_cycles); 6050 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6051 stats.acc_offload_cycles); 6052 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6053 6054 /* give time for device to process ops */ 6055 rte_delay_us(WAIT_OFFLOAD_US); 6056 6057 /* Start time meas for dequeue function offload latency */ 6058 deq_start_time = rte_rdtsc_precise(); 6059 /* Dequeue one operation */ 6060 do { 6061 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6062 &ops_deq[deq], enq); 6063 } while (unlikely(deq == 0)); 6064 6065 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6066 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6067 deq_last_time); 6068 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6069 deq_last_time); 6070 time_st->deq_total_time += deq_last_time; 6071 6072 /* Dequeue remaining operations if needed*/ 6073 while (burst_sz != deq) 6074 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 6075 &ops_deq[deq], burst_sz - deq); 6076 6077 if (extDdr) { 6078 /* Read loopback is not thread safe */ 6079 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 6080 } 6081 6082 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 6083 dequeued += deq; 6084 } 6085 6086 return i; 6087 } 6088 6089 static int 6090 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 6091 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6092 uint16_t queue_id, const uint16_t num_to_process, 6093 uint16_t burst_sz, struct test_time_stats *time_st) 6094 { 6095 int i, dequeued, ret; 6096 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6097 uint64_t enq_start_time, deq_start_time; 6098 uint64_t enq_sw_last_time, deq_last_time; 6099 struct rte_bbdev_stats stats; 6100 6101 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6102 uint16_t enq = 0, deq = 0; 6103 6104 if (unlikely(num_to_process - dequeued < burst_sz)) 6105 burst_sz = num_to_process - dequeued; 6106 6107 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6108 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6109 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6110 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 6111 bufs->inputs, 6112 bufs->hard_outputs, 6113 ref_op); 6114 6115 /* Start time meas for enqueue function offload latency */ 6116 enq_start_time = rte_rdtsc_precise(); 6117 do { 6118 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 6119 &ops_enq[enq], burst_sz - enq); 6120 } while (unlikely(burst_sz != enq)); 6121 6122 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6123 6124 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6125 TEST_ASSERT_SUCCESS(ret, 6126 "Failed to get stats for queue (%u) of device (%u)", 6127 queue_id, dev_id); 6128 enq_sw_last_time -= stats.acc_offload_cycles; 6129 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6130 enq_sw_last_time); 6131 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6132 enq_sw_last_time); 6133 time_st->enq_sw_total_time += enq_sw_last_time; 6134 6135 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6136 stats.acc_offload_cycles); 6137 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6138 stats.acc_offload_cycles); 6139 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6140 6141 /* give time for device to process ops */ 6142 rte_delay_us(WAIT_OFFLOAD_US); 6143 6144 /* Start time meas for dequeue function offload latency */ 6145 deq_start_time = rte_rdtsc_precise(); 6146 /* Dequeue one operation */ 6147 do { 6148 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6149 &ops_deq[deq], enq); 6150 } while (unlikely(deq == 0)); 6151 6152 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6153 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6154 deq_last_time); 6155 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6156 deq_last_time); 6157 time_st->deq_total_time += deq_last_time; 6158 6159 while (burst_sz != deq) 6160 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 6161 &ops_deq[deq], burst_sz - deq); 6162 6163 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6164 dequeued += deq; 6165 } 6166 6167 return i; 6168 } 6169 6170 static int 6171 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 6172 struct test_buffers *bufs, 6173 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 6174 uint16_t queue_id, const uint16_t num_to_process, 6175 uint16_t burst_sz, struct test_time_stats *time_st) 6176 { 6177 int i, dequeued, ret; 6178 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 6179 uint64_t enq_start_time, deq_start_time; 6180 uint64_t enq_sw_last_time, deq_last_time; 6181 struct rte_bbdev_stats stats; 6182 6183 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 6184 uint16_t enq = 0, deq = 0; 6185 6186 if (unlikely(num_to_process - dequeued < burst_sz)) 6187 burst_sz = num_to_process - dequeued; 6188 6189 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 6190 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_enc_op_alloc_bulk() failed"); 6191 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 6192 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 6193 bufs->inputs, 6194 bufs->hard_outputs, 6195 ref_op); 6196 6197 /* Start time meas for enqueue function offload latency */ 6198 enq_start_time = rte_rdtsc_precise(); 6199 do { 6200 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 6201 &ops_enq[enq], burst_sz - enq); 6202 } while (unlikely(burst_sz != enq)); 6203 6204 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 6205 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 6206 TEST_ASSERT_SUCCESS(ret, 6207 "Failed to get stats for queue (%u) of device (%u)", 6208 queue_id, dev_id); 6209 6210 enq_sw_last_time -= stats.acc_offload_cycles; 6211 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 6212 enq_sw_last_time); 6213 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 6214 enq_sw_last_time); 6215 time_st->enq_sw_total_time += enq_sw_last_time; 6216 6217 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 6218 stats.acc_offload_cycles); 6219 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 6220 stats.acc_offload_cycles); 6221 time_st->enq_acc_total_time += stats.acc_offload_cycles; 6222 6223 /* give time for device to process ops */ 6224 rte_delay_us(WAIT_OFFLOAD_US); 6225 6226 /* Start time meas for dequeue function offload latency */ 6227 deq_start_time = rte_rdtsc_precise(); 6228 /* Dequeue one operation */ 6229 do { 6230 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6231 &ops_deq[deq], enq); 6232 } while (unlikely(deq == 0)); 6233 6234 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6235 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 6236 deq_last_time); 6237 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 6238 deq_last_time); 6239 time_st->deq_total_time += deq_last_time; 6240 6241 while (burst_sz != deq) 6242 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 6243 &ops_deq[deq], burst_sz - deq); 6244 6245 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 6246 dequeued += deq; 6247 } 6248 6249 return i; 6250 } 6251 6252 static int 6253 offload_cost_test(struct active_device *ad, 6254 struct test_op_params *op_params) 6255 { 6256 int iter, ret; 6257 uint16_t burst_sz = op_params->burst_sz; 6258 const uint16_t num_to_process = op_params->num_to_process; 6259 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6260 const uint16_t queue_id = ad->queue_ids[0]; 6261 struct test_buffers *bufs = NULL; 6262 struct rte_bbdev_info info; 6263 const char *op_type_str; 6264 struct test_time_stats time_st; 6265 6266 memset(&time_st, 0, sizeof(struct test_time_stats)); 6267 time_st.enq_sw_min_time = UINT64_MAX; 6268 time_st.enq_acc_min_time = UINT64_MAX; 6269 time_st.deq_min_time = UINT64_MAX; 6270 6271 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6272 "BURST_SIZE should be <= %u", MAX_BURST); 6273 6274 rte_bbdev_info_get(ad->dev_id, &info); 6275 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 6276 6277 op_type_str = rte_bbdev_op_type_str(op_type); 6278 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6279 6280 printf("+ ------------------------------------------------------- +\n"); 6281 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6282 info.dev_name, burst_sz, num_to_process, op_type_str); 6283 6284 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 6285 iter = offload_latency_test_dec(op_params->mp, bufs, 6286 op_params->ref_dec_op, ad->dev_id, queue_id, 6287 num_to_process, burst_sz, &time_st); 6288 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 6289 iter = offload_latency_test_enc(op_params->mp, bufs, 6290 op_params->ref_enc_op, ad->dev_id, queue_id, 6291 num_to_process, burst_sz, &time_st); 6292 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6293 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 6294 op_params->ref_enc_op, ad->dev_id, queue_id, 6295 num_to_process, burst_sz, &time_st); 6296 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6297 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 6298 op_params->ref_dec_op, ad->dev_id, queue_id, 6299 num_to_process, burst_sz, &time_st); 6300 else if (op_type == RTE_BBDEV_OP_FFT) 6301 iter = offload_latency_test_fft(op_params->mp, bufs, 6302 op_params->ref_fft_op, ad->dev_id, queue_id, 6303 num_to_process, burst_sz, &time_st); 6304 else if (op_type == RTE_BBDEV_OP_MLDTS) 6305 iter = offload_latency_test_mldts(op_params->mp, bufs, 6306 op_params->ref_mldts_op, ad->dev_id, queue_id, 6307 num_to_process, burst_sz, &time_st); 6308 else 6309 iter = offload_latency_test_enc(op_params->mp, bufs, 6310 op_params->ref_enc_op, ad->dev_id, queue_id, 6311 num_to_process, burst_sz, &time_st); 6312 6313 if (iter <= 0) 6314 return TEST_FAILED; 6315 6316 printf("Enqueue driver offload cost latency:\n" 6317 "\tavg: %lg cycles, %lg us\n" 6318 "\tmin: %lg cycles, %lg us\n" 6319 "\tmax: %lg cycles, %lg us\n" 6320 "Enqueue accelerator offload cost latency:\n" 6321 "\tavg: %lg cycles, %lg us\n" 6322 "\tmin: %lg cycles, %lg us\n" 6323 "\tmax: %lg cycles, %lg us\n", 6324 (double)time_st.enq_sw_total_time / (double)iter, 6325 (double)(time_st.enq_sw_total_time * 1000000) / 6326 (double)iter / (double)rte_get_tsc_hz(), 6327 (double)time_st.enq_sw_min_time, 6328 (double)(time_st.enq_sw_min_time * 1000000) / 6329 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 6330 (double)(time_st.enq_sw_max_time * 1000000) / 6331 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 6332 (double)iter, 6333 (double)(time_st.enq_acc_total_time * 1000000) / 6334 (double)iter / (double)rte_get_tsc_hz(), 6335 (double)time_st.enq_acc_min_time, 6336 (double)(time_st.enq_acc_min_time * 1000000) / 6337 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 6338 (double)(time_st.enq_acc_max_time * 1000000) / 6339 rte_get_tsc_hz()); 6340 6341 printf("Dequeue offload cost latency - one op:\n" 6342 "\tavg: %lg cycles, %lg us\n" 6343 "\tmin: %lg cycles, %lg us\n" 6344 "\tmax: %lg cycles, %lg us\n", 6345 (double)time_st.deq_total_time / (double)iter, 6346 (double)(time_st.deq_total_time * 1000000) / 6347 (double)iter / (double)rte_get_tsc_hz(), 6348 (double)time_st.deq_min_time, 6349 (double)(time_st.deq_min_time * 1000000) / 6350 rte_get_tsc_hz(), (double)time_st.deq_max_time, 6351 (double)(time_st.deq_max_time * 1000000) / 6352 rte_get_tsc_hz()); 6353 6354 struct rte_bbdev_stats stats = {0}; 6355 ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 6356 TEST_ASSERT_SUCCESS(ret, 6357 "Failed to get stats for queue (%u) of device (%u)", 6358 queue_id, ad->dev_id); 6359 if (stats.enqueue_warn_count > 0) 6360 printf("Warning reported on the queue : %10"PRIu64"\n", 6361 stats.enqueue_warn_count); 6362 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 6363 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 6364 "Mismatch in enqueue count %10"PRIu64" %d", 6365 stats.enqueued_count, num_to_process); 6366 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 6367 "Mismatch in dequeue count %10"PRIu64" %d", 6368 stats.dequeued_count, num_to_process); 6369 } 6370 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 6371 "Enqueue count Error %10"PRIu64"", 6372 stats.enqueue_err_count); 6373 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 6374 "Dequeue count Error (%10"PRIu64"", 6375 stats.dequeue_err_count); 6376 6377 return TEST_SUCCESS; 6378 } 6379 6380 static int 6381 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 6382 const uint16_t num_to_process, uint16_t burst_sz, 6383 uint64_t *deq_total_time, uint64_t *deq_min_time, 6384 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6385 { 6386 int i, deq_total; 6387 struct rte_bbdev_dec_op *ops[MAX_BURST]; 6388 uint64_t deq_start_time, deq_last_time; 6389 6390 /* Test deq offload latency from an empty queue */ 6391 6392 for (i = 0, deq_total = 0; deq_total < num_to_process; 6393 ++i, deq_total += burst_sz) { 6394 deq_start_time = rte_rdtsc_precise(); 6395 6396 if (unlikely(num_to_process - deq_total < burst_sz)) 6397 burst_sz = num_to_process - deq_total; 6398 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 6399 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 6400 burst_sz); 6401 else 6402 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 6403 burst_sz); 6404 6405 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6406 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6407 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6408 *deq_total_time += deq_last_time; 6409 } 6410 6411 return i; 6412 } 6413 6414 static int 6415 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 6416 const uint16_t num_to_process, uint16_t burst_sz, 6417 uint64_t *deq_total_time, uint64_t *deq_min_time, 6418 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 6419 { 6420 int i, deq_total; 6421 struct rte_bbdev_enc_op *ops[MAX_BURST]; 6422 uint64_t deq_start_time, deq_last_time; 6423 6424 /* Test deq offload latency from an empty queue */ 6425 for (i = 0, deq_total = 0; deq_total < num_to_process; 6426 ++i, deq_total += burst_sz) { 6427 deq_start_time = rte_rdtsc_precise(); 6428 6429 if (unlikely(num_to_process - deq_total < burst_sz)) 6430 burst_sz = num_to_process - deq_total; 6431 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 6432 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 6433 burst_sz); 6434 else 6435 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 6436 burst_sz); 6437 6438 deq_last_time = rte_rdtsc_precise() - deq_start_time; 6439 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 6440 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 6441 *deq_total_time += deq_last_time; 6442 } 6443 6444 return i; 6445 } 6446 6447 static int 6448 offload_latency_empty_q_test(struct active_device *ad, 6449 struct test_op_params *op_params) 6450 { 6451 int iter; 6452 uint64_t deq_total_time, deq_min_time, deq_max_time; 6453 uint16_t burst_sz = op_params->burst_sz; 6454 const uint16_t num_to_process = op_params->num_to_process; 6455 const enum rte_bbdev_op_type op_type = test_vector.op_type; 6456 const uint16_t queue_id = ad->queue_ids[0]; 6457 struct rte_bbdev_info info; 6458 const char *op_type_str; 6459 6460 deq_total_time = deq_max_time = 0; 6461 deq_min_time = UINT64_MAX; 6462 6463 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 6464 "BURST_SIZE should be <= %u", MAX_BURST); 6465 6466 rte_bbdev_info_get(ad->dev_id, &info); 6467 6468 op_type_str = rte_bbdev_op_type_str(op_type); 6469 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 6470 6471 printf("+ ------------------------------------------------------- +\n"); 6472 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 6473 info.dev_name, burst_sz, num_to_process, op_type_str); 6474 6475 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 6476 op_type == RTE_BBDEV_OP_LDPC_DEC) 6477 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 6478 num_to_process, burst_sz, &deq_total_time, 6479 &deq_min_time, &deq_max_time, op_type); 6480 else 6481 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 6482 num_to_process, burst_sz, &deq_total_time, 6483 &deq_min_time, &deq_max_time, op_type); 6484 6485 if (iter <= 0) 6486 return TEST_FAILED; 6487 6488 printf("Empty dequeue offload:\n" 6489 "\tavg: %lg cycles, %lg us\n" 6490 "\tmin: %lg cycles, %lg us\n" 6491 "\tmax: %lg cycles, %lg us\n", 6492 (double)deq_total_time / (double)iter, 6493 (double)(deq_total_time * 1000000) / (double)iter / 6494 (double)rte_get_tsc_hz(), (double)deq_min_time, 6495 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 6496 (double)deq_max_time, (double)(deq_max_time * 1000000) / 6497 rte_get_tsc_hz()); 6498 6499 return TEST_SUCCESS; 6500 } 6501 6502 static int 6503 bler_tc(void) 6504 { 6505 return run_test_case(bler_test); 6506 } 6507 6508 static int 6509 throughput_tc(void) 6510 { 6511 return run_test_case(throughput_test); 6512 } 6513 6514 static int 6515 offload_cost_tc(void) 6516 { 6517 return run_test_case(offload_cost_test); 6518 } 6519 6520 static int 6521 offload_latency_empty_q_tc(void) 6522 { 6523 return run_test_case(offload_latency_empty_q_test); 6524 } 6525 6526 static int 6527 latency_tc(void) 6528 { 6529 return run_test_case(latency_test); 6530 } 6531 6532 static int 6533 validation_tc(void) 6534 { 6535 return run_test_case(validation_test); 6536 } 6537 6538 static int 6539 interrupt_tc(void) 6540 { 6541 return run_test_case(throughput_test); 6542 } 6543 6544 static struct unit_test_suite bbdev_bler_testsuite = { 6545 .suite_name = "BBdev BLER Tests", 6546 .setup = testsuite_setup, 6547 .teardown = testsuite_teardown, 6548 .unit_test_cases = { 6549 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 6550 TEST_CASES_END() /**< NULL terminate unit test array */ 6551 } 6552 }; 6553 6554 static struct unit_test_suite bbdev_throughput_testsuite = { 6555 .suite_name = "BBdev Throughput Tests", 6556 .setup = testsuite_setup, 6557 .teardown = testsuite_teardown, 6558 .unit_test_cases = { 6559 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 6560 TEST_CASES_END() /**< NULL terminate unit test array */ 6561 } 6562 }; 6563 6564 static struct unit_test_suite bbdev_validation_testsuite = { 6565 .suite_name = "BBdev Validation Tests", 6566 .setup = testsuite_setup, 6567 .teardown = testsuite_teardown, 6568 .unit_test_cases = { 6569 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 6570 TEST_CASES_END() /**< NULL terminate unit test array */ 6571 } 6572 }; 6573 6574 static struct unit_test_suite bbdev_latency_testsuite = { 6575 .suite_name = "BBdev Latency Tests", 6576 .setup = testsuite_setup, 6577 .teardown = testsuite_teardown, 6578 .unit_test_cases = { 6579 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 6580 TEST_CASES_END() /**< NULL terminate unit test array */ 6581 } 6582 }; 6583 6584 static struct unit_test_suite bbdev_offload_cost_testsuite = { 6585 .suite_name = "BBdev Offload Cost Tests", 6586 .setup = testsuite_setup, 6587 .teardown = testsuite_teardown, 6588 .unit_test_cases = { 6589 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 6590 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 6591 TEST_CASES_END() /**< NULL terminate unit test array */ 6592 } 6593 }; 6594 6595 static struct unit_test_suite bbdev_interrupt_testsuite = { 6596 .suite_name = "BBdev Interrupt Tests", 6597 .setup = interrupt_testsuite_setup, 6598 .teardown = testsuite_teardown, 6599 .unit_test_cases = { 6600 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 6601 TEST_CASES_END() /**< NULL terminate unit test array */ 6602 } 6603 }; 6604 6605 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 6606 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 6607 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 6608 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 6609 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 6610 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 6611