1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <math.h> 9 10 #include <rte_eal.h> 11 #include <rte_common.h> 12 #include <rte_dev.h> 13 #include <rte_launch.h> 14 #include <rte_bbdev.h> 15 #include <rte_cycles.h> 16 #include <rte_lcore.h> 17 #include <rte_malloc.h> 18 #include <rte_random.h> 19 #include <rte_hexdump.h> 20 #include <rte_interrupts.h> 21 22 #include "main.h" 23 #include "test_bbdev_vector.h" 24 25 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 26 27 #define MAX_QUEUES RTE_MAX_LCORE 28 #define TEST_REPETITIONS 100 29 #define WAIT_OFFLOAD_US 1000 30 31 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 32 #include <fpga_lte_fec.h> 33 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 34 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 35 #define VF_UL_4G_QUEUE_VALUE 4 36 #define VF_DL_4G_QUEUE_VALUE 4 37 #define UL_4G_BANDWIDTH 3 38 #define DL_4G_BANDWIDTH 3 39 #define UL_4G_LOAD_BALANCE 128 40 #define DL_4G_LOAD_BALANCE 128 41 #define FLR_4G_TIMEOUT 610 42 #endif 43 44 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 45 #include <rte_pmd_fpga_5gnr_fec.h> 46 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 47 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 48 #define VF_UL_5G_QUEUE_VALUE 4 49 #define VF_DL_5G_QUEUE_VALUE 4 50 #define UL_5G_BANDWIDTH 3 51 #define DL_5G_BANDWIDTH 3 52 #define UL_5G_LOAD_BALANCE 128 53 #define DL_5G_LOAD_BALANCE 128 54 #endif 55 56 #ifdef RTE_BASEBAND_ACC 57 #include <rte_acc_cfg.h> 58 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 59 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 60 #define ACC100_QMGR_NUM_AQS 16 61 #define ACC100_QMGR_NUM_QGS 2 62 #define ACC100_QMGR_AQ_DEPTH 5 63 #define ACC100_QMGR_INVALID_IDX -1 64 #define ACC100_QMGR_RR 1 65 #define ACC100_QOS_GBR 0 66 #define ACC200PF_DRIVER_NAME ("intel_acc200_pf") 67 #define ACC200VF_DRIVER_NAME ("intel_acc200_vf") 68 #define ACC200_QMGR_NUM_AQS 16 69 #define ACC200_QMGR_NUM_QGS 2 70 #define ACC200_QMGR_AQ_DEPTH 5 71 #define ACC200_QMGR_INVALID_IDX -1 72 #define ACC200_QMGR_RR 1 73 #define ACC200_QOS_GBR 0 74 #endif 75 76 #define OPS_CACHE_SIZE 256U 77 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 78 79 #define SYNC_WAIT 0 80 #define SYNC_START 1 81 #define INVALID_OPAQUE -1 82 83 #define INVALID_QUEUE_ID -1 84 /* Increment for next code block in external HARQ memory */ 85 #define HARQ_INCR 32768 86 /* Headroom for filler LLRs insertion in HARQ buffer */ 87 #define FILLER_HEADROOM 1024 88 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 89 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 90 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 91 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 92 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 93 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 94 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 95 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 96 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 97 98 static struct test_bbdev_vector test_vector; 99 100 /* Switch between PMD and Interrupt for throughput TC */ 101 static bool intr_enabled; 102 103 /* LLR arithmetic representation for numerical conversion */ 104 static int ldpc_llr_decimals; 105 static int ldpc_llr_size; 106 /* Keep track of the LDPC decoder device capability flag */ 107 static uint32_t ldpc_cap_flags; 108 109 /* Represents tested active devices */ 110 static struct active_device { 111 const char *driver_name; 112 uint8_t dev_id; 113 uint16_t supported_ops; 114 uint16_t queue_ids[MAX_QUEUES]; 115 uint16_t nb_queues; 116 struct rte_mempool *ops_mempool; 117 struct rte_mempool *in_mbuf_pool; 118 struct rte_mempool *hard_out_mbuf_pool; 119 struct rte_mempool *soft_out_mbuf_pool; 120 struct rte_mempool *harq_in_mbuf_pool; 121 struct rte_mempool *harq_out_mbuf_pool; 122 } active_devs[RTE_BBDEV_MAX_DEVS]; 123 124 static uint8_t nb_active_devs; 125 126 /* Data buffers used by BBDEV ops */ 127 struct test_buffers { 128 struct rte_bbdev_op_data *inputs; 129 struct rte_bbdev_op_data *hard_outputs; 130 struct rte_bbdev_op_data *soft_outputs; 131 struct rte_bbdev_op_data *harq_inputs; 132 struct rte_bbdev_op_data *harq_outputs; 133 }; 134 135 /* Operation parameters specific for given test case */ 136 struct test_op_params { 137 struct rte_mempool *mp; 138 struct rte_bbdev_dec_op *ref_dec_op; 139 struct rte_bbdev_enc_op *ref_enc_op; 140 struct rte_bbdev_fft_op *ref_fft_op; 141 uint16_t burst_sz; 142 uint16_t num_to_process; 143 uint16_t num_lcores; 144 int vector_mask; 145 uint16_t sync; 146 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 147 }; 148 149 /* Contains per lcore params */ 150 struct thread_params { 151 uint8_t dev_id; 152 uint16_t queue_id; 153 uint32_t lcore_id; 154 uint64_t start_time; 155 double ops_per_sec; 156 double mbps; 157 uint8_t iter_count; 158 double iter_average; 159 double bler; 160 uint16_t nb_dequeued; 161 int16_t processing_status; 162 uint16_t burst_sz; 163 struct test_op_params *op_params; 164 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 165 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 166 struct rte_bbdev_fft_op *fft_ops[MAX_BURST]; 167 }; 168 169 #ifdef RTE_BBDEV_OFFLOAD_COST 170 /* Stores time statistics */ 171 struct test_time_stats { 172 /* Stores software enqueue total working time */ 173 uint64_t enq_sw_total_time; 174 /* Stores minimum value of software enqueue working time */ 175 uint64_t enq_sw_min_time; 176 /* Stores maximum value of software enqueue working time */ 177 uint64_t enq_sw_max_time; 178 /* Stores turbo enqueue total working time */ 179 uint64_t enq_acc_total_time; 180 /* Stores minimum value of accelerator enqueue working time */ 181 uint64_t enq_acc_min_time; 182 /* Stores maximum value of accelerator enqueue working time */ 183 uint64_t enq_acc_max_time; 184 /* Stores dequeue total working time */ 185 uint64_t deq_total_time; 186 /* Stores minimum value of dequeue working time */ 187 uint64_t deq_min_time; 188 /* Stores maximum value of dequeue working time */ 189 uint64_t deq_max_time; 190 }; 191 #endif 192 193 typedef int (test_case_function)(struct active_device *ad, 194 struct test_op_params *op_params); 195 196 static inline void 197 mbuf_reset(struct rte_mbuf *m) 198 { 199 m->pkt_len = 0; 200 201 do { 202 m->data_len = 0; 203 m = m->next; 204 } while (m != NULL); 205 } 206 207 /* Read flag value 0/1 from bitmap */ 208 static inline bool 209 check_bit(uint32_t bitmap, uint32_t bitmask) 210 { 211 return bitmap & bitmask; 212 } 213 214 static inline void 215 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 216 { 217 ad->supported_ops |= (1 << op_type); 218 } 219 220 static inline bool 221 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 222 { 223 return ad->supported_ops & (1 << op_type); 224 } 225 226 static inline bool 227 flags_match(uint32_t flags_req, uint32_t flags_present) 228 { 229 return (flags_req & flags_present) == flags_req; 230 } 231 232 static void 233 clear_soft_out_cap(uint32_t *op_flags) 234 { 235 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 236 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 237 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 238 } 239 240 /* This API is to convert all the test vector op data entries 241 * to big endian format. It is used when the device supports 242 * the input in the big endian format. 243 */ 244 static inline void 245 convert_op_data_to_be(void) 246 { 247 struct op_data_entries *op; 248 enum op_data_type type; 249 uint8_t nb_segs, *rem_data, temp; 250 uint32_t *data, len; 251 int complete, rem, i, j; 252 253 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 254 nb_segs = test_vector.entries[type].nb_segments; 255 op = &test_vector.entries[type]; 256 257 /* Invert byte endianness for all the segments */ 258 for (i = 0; i < nb_segs; ++i) { 259 len = op->segments[i].length; 260 data = op->segments[i].addr; 261 262 /* Swap complete u32 bytes */ 263 complete = len / 4; 264 for (j = 0; j < complete; j++) 265 data[j] = rte_bswap32(data[j]); 266 267 /* Swap any remaining bytes */ 268 rem = len % 4; 269 rem_data = (uint8_t *)&data[j]; 270 for (j = 0; j < rem/2; j++) { 271 temp = rem_data[j]; 272 rem_data[j] = rem_data[rem - j - 1]; 273 rem_data[rem - j - 1] = temp; 274 } 275 } 276 } 277 } 278 279 static int 280 check_dev_cap(const struct rte_bbdev_info *dev_info) 281 { 282 unsigned int i; 283 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 284 nb_harq_inputs, nb_harq_outputs; 285 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 286 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 287 288 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 289 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 290 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 291 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 292 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 293 294 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 295 if (op_cap->type != test_vector.op_type) 296 continue; 297 298 if (dev_data_endianness == RTE_BIG_ENDIAN) 299 convert_op_data_to_be(); 300 301 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 302 const struct rte_bbdev_op_cap_turbo_dec *cap = 303 &op_cap->cap.turbo_dec; 304 /* Ignore lack of soft output capability, just skip 305 * checking if soft output is valid. 306 */ 307 if ((test_vector.turbo_dec.op_flags & 308 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 309 !(cap->capability_flags & 310 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 311 printf( 312 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 313 dev_info->dev_name); 314 clear_soft_out_cap( 315 &test_vector.turbo_dec.op_flags); 316 } 317 318 if (!flags_match(test_vector.turbo_dec.op_flags, 319 cap->capability_flags)) 320 return TEST_FAILED; 321 if (nb_inputs > cap->num_buffers_src) { 322 printf("Too many inputs defined: %u, max: %u\n", 323 nb_inputs, cap->num_buffers_src); 324 return TEST_FAILED; 325 } 326 if (nb_soft_outputs > cap->num_buffers_soft_out && 327 (test_vector.turbo_dec.op_flags & 328 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 329 printf( 330 "Too many soft outputs defined: %u, max: %u\n", 331 nb_soft_outputs, 332 cap->num_buffers_soft_out); 333 return TEST_FAILED; 334 } 335 if (nb_hard_outputs > cap->num_buffers_hard_out) { 336 printf( 337 "Too many hard outputs defined: %u, max: %u\n", 338 nb_hard_outputs, 339 cap->num_buffers_hard_out); 340 return TEST_FAILED; 341 } 342 if (intr_enabled && !(cap->capability_flags & 343 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 344 printf( 345 "Dequeue interrupts are not supported!\n"); 346 return TEST_FAILED; 347 } 348 349 return TEST_SUCCESS; 350 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 351 const struct rte_bbdev_op_cap_turbo_enc *cap = 352 &op_cap->cap.turbo_enc; 353 354 if (!flags_match(test_vector.turbo_enc.op_flags, 355 cap->capability_flags)) 356 return TEST_FAILED; 357 if (nb_inputs > cap->num_buffers_src) { 358 printf("Too many inputs defined: %u, max: %u\n", 359 nb_inputs, cap->num_buffers_src); 360 return TEST_FAILED; 361 } 362 if (nb_hard_outputs > cap->num_buffers_dst) { 363 printf( 364 "Too many hard outputs defined: %u, max: %u\n", 365 nb_hard_outputs, cap->num_buffers_dst); 366 return TEST_FAILED; 367 } 368 if (intr_enabled && !(cap->capability_flags & 369 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 370 printf( 371 "Dequeue interrupts are not supported!\n"); 372 return TEST_FAILED; 373 } 374 375 return TEST_SUCCESS; 376 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 377 const struct rte_bbdev_op_cap_ldpc_enc *cap = 378 &op_cap->cap.ldpc_enc; 379 380 if (!flags_match(test_vector.ldpc_enc.op_flags, 381 cap->capability_flags)){ 382 printf("Flag Mismatch\n"); 383 return TEST_FAILED; 384 } 385 if (nb_inputs > cap->num_buffers_src) { 386 printf("Too many inputs defined: %u, max: %u\n", 387 nb_inputs, cap->num_buffers_src); 388 return TEST_FAILED; 389 } 390 if (nb_hard_outputs > cap->num_buffers_dst) { 391 printf( 392 "Too many hard outputs defined: %u, max: %u\n", 393 nb_hard_outputs, cap->num_buffers_dst); 394 return TEST_FAILED; 395 } 396 if (intr_enabled && !(cap->capability_flags & 397 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 398 printf( 399 "Dequeue interrupts are not supported!\n"); 400 return TEST_FAILED; 401 } 402 403 return TEST_SUCCESS; 404 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 405 const struct rte_bbdev_op_cap_ldpc_dec *cap = 406 &op_cap->cap.ldpc_dec; 407 408 if (!flags_match(test_vector.ldpc_dec.op_flags, 409 cap->capability_flags)){ 410 printf("Flag Mismatch\n"); 411 return TEST_FAILED; 412 } 413 if (nb_inputs > cap->num_buffers_src) { 414 printf("Too many inputs defined: %u, max: %u\n", 415 nb_inputs, cap->num_buffers_src); 416 return TEST_FAILED; 417 } 418 if (nb_hard_outputs > cap->num_buffers_hard_out) { 419 printf( 420 "Too many hard outputs defined: %u, max: %u\n", 421 nb_hard_outputs, 422 cap->num_buffers_hard_out); 423 return TEST_FAILED; 424 } 425 if (nb_harq_inputs > cap->num_buffers_hard_out) { 426 printf( 427 "Too many HARQ inputs defined: %u, max: %u\n", 428 nb_harq_inputs, 429 cap->num_buffers_hard_out); 430 return TEST_FAILED; 431 } 432 if (nb_harq_outputs > cap->num_buffers_hard_out) { 433 printf( 434 "Too many HARQ outputs defined: %u, max: %u\n", 435 nb_harq_outputs, 436 cap->num_buffers_hard_out); 437 return TEST_FAILED; 438 } 439 if (intr_enabled && !(cap->capability_flags & 440 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 441 printf( 442 "Dequeue interrupts are not supported!\n"); 443 return TEST_FAILED; 444 } 445 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 446 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 447 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 448 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 449 ))) { 450 printf("Skip loop-back with interrupt\n"); 451 return TEST_FAILED; 452 } 453 return TEST_SUCCESS; 454 } else if (op_cap->type == RTE_BBDEV_OP_FFT) { 455 const struct rte_bbdev_op_cap_fft *cap = &op_cap->cap.fft; 456 457 if (!flags_match(test_vector.fft.op_flags, cap->capability_flags)) { 458 printf("Flag Mismatch\n"); 459 return TEST_FAILED; 460 } 461 if (nb_inputs > cap->num_buffers_src) { 462 printf("Too many inputs defined: %u, max: %u\n", 463 nb_inputs, cap->num_buffers_src); 464 return TEST_FAILED; 465 } 466 return TEST_SUCCESS; 467 } 468 } 469 470 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 471 return TEST_SUCCESS; /* Special case for NULL device */ 472 473 return TEST_FAILED; 474 } 475 476 /* calculates optimal mempool size not smaller than the val */ 477 static unsigned int 478 optimal_mempool_size(unsigned int val) 479 { 480 return rte_align32pow2(val + 1) - 1; 481 } 482 483 /* allocates mbuf mempool for inputs and outputs */ 484 static struct rte_mempool * 485 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 486 int socket_id, unsigned int mbuf_pool_size, 487 const char *op_type_str) 488 { 489 unsigned int i; 490 uint32_t max_seg_sz = 0; 491 char pool_name[RTE_MEMPOOL_NAMESIZE]; 492 493 /* find max input segment size */ 494 for (i = 0; i < entries->nb_segments; ++i) 495 if (entries->segments[i].length > max_seg_sz) 496 max_seg_sz = entries->segments[i].length; 497 498 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 499 dev_id); 500 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 501 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 502 + FILLER_HEADROOM, 503 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 504 } 505 506 static int 507 create_mempools(struct active_device *ad, int socket_id, 508 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 509 { 510 struct rte_mempool *mp; 511 unsigned int ops_pool_size, mbuf_pool_size = 0; 512 char pool_name[RTE_MEMPOOL_NAMESIZE]; 513 const char *op_type_str; 514 enum rte_bbdev_op_type op_type = org_op_type; 515 516 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 517 struct op_data_entries *hard_out = 518 &test_vector.entries[DATA_HARD_OUTPUT]; 519 struct op_data_entries *soft_out = 520 &test_vector.entries[DATA_SOFT_OUTPUT]; 521 struct op_data_entries *harq_in = 522 &test_vector.entries[DATA_HARQ_INPUT]; 523 struct op_data_entries *harq_out = 524 &test_vector.entries[DATA_HARQ_OUTPUT]; 525 526 /* allocate ops mempool */ 527 ops_pool_size = optimal_mempool_size(RTE_MAX( 528 /* Ops used plus 1 reference op */ 529 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 530 /* Minimal cache size plus 1 reference op */ 531 (unsigned int)(1.5 * rte_lcore_count() * 532 OPS_CACHE_SIZE + 1)), 533 OPS_POOL_SIZE_MIN)); 534 535 if (org_op_type == RTE_BBDEV_OP_NONE) 536 op_type = RTE_BBDEV_OP_TURBO_ENC; 537 538 op_type_str = rte_bbdev_op_type_str(op_type); 539 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 540 541 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 542 ad->dev_id); 543 mp = rte_bbdev_op_pool_create(pool_name, op_type, 544 ops_pool_size, OPS_CACHE_SIZE, socket_id); 545 TEST_ASSERT_NOT_NULL(mp, 546 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 547 ops_pool_size, 548 ad->dev_id, 549 socket_id); 550 ad->ops_mempool = mp; 551 552 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 553 if (org_op_type == RTE_BBDEV_OP_NONE) 554 return TEST_SUCCESS; 555 556 /* Inputs */ 557 if (in->nb_segments > 0) { 558 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 559 in->nb_segments); 560 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 561 mbuf_pool_size, "in"); 562 TEST_ASSERT_NOT_NULL(mp, 563 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 564 mbuf_pool_size, 565 ad->dev_id, 566 socket_id); 567 ad->in_mbuf_pool = mp; 568 } 569 570 /* Hard outputs */ 571 if (hard_out->nb_segments > 0) { 572 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 573 hard_out->nb_segments); 574 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 575 mbuf_pool_size, 576 "hard_out"); 577 TEST_ASSERT_NOT_NULL(mp, 578 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 579 mbuf_pool_size, 580 ad->dev_id, 581 socket_id); 582 ad->hard_out_mbuf_pool = mp; 583 } 584 585 /* Soft outputs */ 586 if (soft_out->nb_segments > 0) { 587 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 588 soft_out->nb_segments); 589 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 590 mbuf_pool_size, 591 "soft_out"); 592 TEST_ASSERT_NOT_NULL(mp, 593 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 594 mbuf_pool_size, 595 ad->dev_id, 596 socket_id); 597 ad->soft_out_mbuf_pool = mp; 598 } 599 600 /* HARQ inputs */ 601 if (harq_in->nb_segments > 0) { 602 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 603 harq_in->nb_segments); 604 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 605 mbuf_pool_size, 606 "harq_in"); 607 TEST_ASSERT_NOT_NULL(mp, 608 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 609 mbuf_pool_size, 610 ad->dev_id, 611 socket_id); 612 ad->harq_in_mbuf_pool = mp; 613 } 614 615 /* HARQ outputs */ 616 if (harq_out->nb_segments > 0) { 617 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 618 harq_out->nb_segments); 619 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 620 mbuf_pool_size, 621 "harq_out"); 622 TEST_ASSERT_NOT_NULL(mp, 623 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 624 mbuf_pool_size, 625 ad->dev_id, 626 socket_id); 627 ad->harq_out_mbuf_pool = mp; 628 } 629 630 return TEST_SUCCESS; 631 } 632 633 static int 634 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 635 struct test_bbdev_vector *vector) 636 { 637 int ret; 638 unsigned int queue_id; 639 struct rte_bbdev_queue_conf qconf; 640 struct active_device *ad = &active_devs[nb_active_devs]; 641 unsigned int nb_queues; 642 enum rte_bbdev_op_type op_type = vector->op_type; 643 644 /* Configure fpga lte fec with PF & VF values 645 * if '-i' flag is set and using fpga device 646 */ 647 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 648 if ((get_init_device() == true) && 649 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 650 struct rte_fpga_lte_fec_conf conf; 651 unsigned int i; 652 653 printf("Configure FPGA LTE FEC Driver %s with default values\n", 654 info->drv.driver_name); 655 656 /* clear default configuration before initialization */ 657 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 658 659 /* Set PF mode : 660 * true if PF is used for data plane 661 * false for VFs 662 */ 663 conf.pf_mode_en = true; 664 665 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 666 /* Number of UL queues per VF (fpga supports 8 VFs) */ 667 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 668 /* Number of DL queues per VF (fpga supports 8 VFs) */ 669 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 670 } 671 672 /* UL bandwidth. Needed for schedule algorithm */ 673 conf.ul_bandwidth = UL_4G_BANDWIDTH; 674 /* DL bandwidth */ 675 conf.dl_bandwidth = DL_4G_BANDWIDTH; 676 677 /* UL & DL load Balance Factor to 64 */ 678 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 679 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 680 681 /**< FLR timeout value */ 682 conf.flr_time_out = FLR_4G_TIMEOUT; 683 684 /* setup FPGA PF with configuration information */ 685 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 686 TEST_ASSERT_SUCCESS(ret, 687 "Failed to configure 4G FPGA PF for bbdev %s", 688 info->dev_name); 689 } 690 #endif 691 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 692 if ((get_init_device() == true) && 693 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 694 struct rte_fpga_5gnr_fec_conf conf; 695 unsigned int i; 696 697 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 698 info->drv.driver_name); 699 700 /* clear default configuration before initialization */ 701 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 702 703 /* Set PF mode : 704 * true if PF is used for data plane 705 * false for VFs 706 */ 707 conf.pf_mode_en = true; 708 709 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 710 /* Number of UL queues per VF (fpga supports 8 VFs) */ 711 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 712 /* Number of DL queues per VF (fpga supports 8 VFs) */ 713 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 714 } 715 716 /* UL bandwidth. Needed for schedule algorithm */ 717 conf.ul_bandwidth = UL_5G_BANDWIDTH; 718 /* DL bandwidth */ 719 conf.dl_bandwidth = DL_5G_BANDWIDTH; 720 721 /* UL & DL load Balance Factor to 64 */ 722 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 723 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 724 725 /* setup FPGA PF with configuration information */ 726 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 727 TEST_ASSERT_SUCCESS(ret, 728 "Failed to configure 5G FPGA PF for bbdev %s", 729 info->dev_name); 730 } 731 #endif 732 #ifdef RTE_BASEBAND_ACC 733 if ((get_init_device() == true) && 734 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 735 struct rte_acc_conf conf; 736 unsigned int i; 737 738 printf("Configure ACC100/ACC101 FEC Driver %s with default values\n", 739 info->drv.driver_name); 740 741 /* clear default configuration before initialization */ 742 memset(&conf, 0, sizeof(struct rte_acc_conf)); 743 744 /* Always set in PF mode for built-in configuration */ 745 conf.pf_mode_en = true; 746 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 747 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 748 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 749 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 750 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 751 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 752 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 753 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 754 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 755 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 756 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 757 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 758 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 759 } 760 761 conf.input_pos_llr_1_bit = true; 762 conf.output_pos_llr_1_bit = true; 763 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 764 765 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 766 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 767 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 768 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 769 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 770 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 771 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 772 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 773 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 774 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 775 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 776 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 777 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 778 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 779 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 780 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 781 782 /* setup PF with configuration information */ 783 ret = rte_acc_configure(info->dev_name, &conf); 784 TEST_ASSERT_SUCCESS(ret, 785 "Failed to configure ACC100 PF for bbdev %s", 786 info->dev_name); 787 } 788 if ((get_init_device() == true) && 789 (!strcmp(info->drv.driver_name, ACC200PF_DRIVER_NAME))) { 790 struct rte_acc_conf conf; 791 unsigned int i; 792 793 printf("Configure ACC200 FEC Driver %s with default values\n", 794 info->drv.driver_name); 795 796 /* clear default configuration before initialization */ 797 memset(&conf, 0, sizeof(struct rte_acc_conf)); 798 799 /* Always set in PF mode for built-in configuration */ 800 conf.pf_mode_en = true; 801 for (i = 0; i < RTE_ACC_NUM_VFS; ++i) { 802 conf.arb_dl_4g[i].gbr_threshold1 = ACC200_QOS_GBR; 803 conf.arb_dl_4g[i].gbr_threshold1 = ACC200_QOS_GBR; 804 conf.arb_dl_4g[i].round_robin_weight = ACC200_QMGR_RR; 805 conf.arb_ul_4g[i].gbr_threshold1 = ACC200_QOS_GBR; 806 conf.arb_ul_4g[i].gbr_threshold1 = ACC200_QOS_GBR; 807 conf.arb_ul_4g[i].round_robin_weight = ACC200_QMGR_RR; 808 conf.arb_dl_5g[i].gbr_threshold1 = ACC200_QOS_GBR; 809 conf.arb_dl_5g[i].gbr_threshold1 = ACC200_QOS_GBR; 810 conf.arb_dl_5g[i].round_robin_weight = ACC200_QMGR_RR; 811 conf.arb_ul_5g[i].gbr_threshold1 = ACC200_QOS_GBR; 812 conf.arb_ul_5g[i].gbr_threshold1 = ACC200_QOS_GBR; 813 conf.arb_ul_5g[i].round_robin_weight = ACC200_QMGR_RR; 814 conf.arb_fft[i].gbr_threshold1 = ACC200_QOS_GBR; 815 conf.arb_fft[i].gbr_threshold1 = ACC200_QOS_GBR; 816 conf.arb_fft[i].round_robin_weight = ACC200_QMGR_RR; 817 } 818 819 conf.input_pos_llr_1_bit = true; 820 conf.output_pos_llr_1_bit = true; 821 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 822 823 conf.q_ul_4g.num_qgroups = ACC200_QMGR_NUM_QGS; 824 conf.q_ul_4g.first_qgroup_index = ACC200_QMGR_INVALID_IDX; 825 conf.q_ul_4g.num_aqs_per_groups = ACC200_QMGR_NUM_AQS; 826 conf.q_ul_4g.aq_depth_log2 = ACC200_QMGR_AQ_DEPTH; 827 conf.q_dl_4g.num_qgroups = ACC200_QMGR_NUM_QGS; 828 conf.q_dl_4g.first_qgroup_index = ACC200_QMGR_INVALID_IDX; 829 conf.q_dl_4g.num_aqs_per_groups = ACC200_QMGR_NUM_AQS; 830 conf.q_dl_4g.aq_depth_log2 = ACC200_QMGR_AQ_DEPTH; 831 conf.q_ul_5g.num_qgroups = ACC200_QMGR_NUM_QGS; 832 conf.q_ul_5g.first_qgroup_index = ACC200_QMGR_INVALID_IDX; 833 conf.q_ul_5g.num_aqs_per_groups = ACC200_QMGR_NUM_AQS; 834 conf.q_ul_5g.aq_depth_log2 = ACC200_QMGR_AQ_DEPTH; 835 conf.q_dl_5g.num_qgroups = ACC200_QMGR_NUM_QGS; 836 conf.q_dl_5g.first_qgroup_index = ACC200_QMGR_INVALID_IDX; 837 conf.q_dl_5g.num_aqs_per_groups = ACC200_QMGR_NUM_AQS; 838 conf.q_dl_5g.aq_depth_log2 = ACC200_QMGR_AQ_DEPTH; 839 conf.q_fft.num_qgroups = ACC200_QMGR_NUM_QGS; 840 conf.q_fft.first_qgroup_index = ACC200_QMGR_INVALID_IDX; 841 conf.q_fft.num_aqs_per_groups = ACC200_QMGR_NUM_AQS; 842 conf.q_fft.aq_depth_log2 = ACC200_QMGR_AQ_DEPTH; 843 844 /* setup PF with configuration information */ 845 ret = rte_acc_configure(info->dev_name, &conf); 846 TEST_ASSERT_SUCCESS(ret, 847 "Failed to configure ACC200 PF for bbdev %s", 848 info->dev_name); 849 } 850 #endif 851 /* Let's refresh this now this is configured */ 852 rte_bbdev_info_get(dev_id, info); 853 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 854 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 855 856 /* setup device */ 857 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 858 if (ret < 0) { 859 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 860 dev_id, nb_queues, info->socket_id, ret); 861 return TEST_FAILED; 862 } 863 864 /* configure interrupts if needed */ 865 if (intr_enabled) { 866 ret = rte_bbdev_intr_enable(dev_id); 867 if (ret < 0) { 868 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 869 ret); 870 return TEST_FAILED; 871 } 872 } 873 874 /* setup device queues */ 875 qconf.socket = info->socket_id; 876 qconf.queue_size = info->drv.default_queue_conf.queue_size; 877 qconf.priority = 0; 878 qconf.deferred_start = 0; 879 qconf.op_type = op_type; 880 881 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 882 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 883 if (ret != 0) { 884 printf( 885 "Allocated all queues (id=%u) at prio%u on dev%u\n", 886 queue_id, qconf.priority, dev_id); 887 qconf.priority++; 888 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 889 &qconf); 890 } 891 if (ret != 0) { 892 printf("All queues on dev %u allocated: %u\n", 893 dev_id, queue_id); 894 break; 895 } 896 ad->queue_ids[queue_id] = queue_id; 897 } 898 TEST_ASSERT(queue_id != 0, 899 "ERROR Failed to configure any queues on dev %u", 900 dev_id); 901 ad->nb_queues = queue_id; 902 903 set_avail_op(ad, op_type); 904 905 return TEST_SUCCESS; 906 } 907 908 static int 909 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 910 struct test_bbdev_vector *vector) 911 { 912 int ret; 913 914 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 915 active_devs[nb_active_devs].dev_id = dev_id; 916 917 ret = add_bbdev_dev(dev_id, info, vector); 918 if (ret == TEST_SUCCESS) 919 ++nb_active_devs; 920 return ret; 921 } 922 923 static uint8_t 924 populate_active_devices(void) 925 { 926 int ret; 927 uint8_t dev_id; 928 uint8_t nb_devs_added = 0; 929 struct rte_bbdev_info info; 930 931 RTE_BBDEV_FOREACH(dev_id) { 932 rte_bbdev_info_get(dev_id, &info); 933 934 if (check_dev_cap(&info)) { 935 printf( 936 "Device %d (%s) does not support specified capabilities\n", 937 dev_id, info.dev_name); 938 continue; 939 } 940 941 ret = add_active_device(dev_id, &info, &test_vector); 942 if (ret != 0) { 943 printf("Adding active bbdev %s skipped\n", 944 info.dev_name); 945 continue; 946 } 947 nb_devs_added++; 948 } 949 950 return nb_devs_added; 951 } 952 953 static int 954 read_test_vector(void) 955 { 956 int ret; 957 958 memset(&test_vector, 0, sizeof(test_vector)); 959 printf("Test vector file = %s\n", get_vector_filename()); 960 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 961 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 962 get_vector_filename()); 963 964 return TEST_SUCCESS; 965 } 966 967 static int 968 testsuite_setup(void) 969 { 970 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 971 972 if (populate_active_devices() == 0) { 973 printf("No suitable devices found!\n"); 974 return TEST_SKIPPED; 975 } 976 977 return TEST_SUCCESS; 978 } 979 980 static int 981 interrupt_testsuite_setup(void) 982 { 983 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 984 985 /* Enable interrupts */ 986 intr_enabled = true; 987 988 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 989 if (populate_active_devices() == 0 || 990 test_vector.op_type == RTE_BBDEV_OP_NONE) { 991 intr_enabled = false; 992 printf("No suitable devices found!\n"); 993 return TEST_SKIPPED; 994 } 995 996 return TEST_SUCCESS; 997 } 998 999 static void 1000 testsuite_teardown(void) 1001 { 1002 uint8_t dev_id; 1003 1004 /* Unconfigure devices */ 1005 RTE_BBDEV_FOREACH(dev_id) 1006 rte_bbdev_close(dev_id); 1007 1008 /* Clear active devices structs. */ 1009 memset(active_devs, 0, sizeof(active_devs)); 1010 nb_active_devs = 0; 1011 1012 /* Disable interrupts */ 1013 intr_enabled = false; 1014 } 1015 1016 static int 1017 ut_setup(void) 1018 { 1019 uint8_t i, dev_id; 1020 1021 for (i = 0; i < nb_active_devs; i++) { 1022 dev_id = active_devs[i].dev_id; 1023 /* reset bbdev stats */ 1024 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 1025 "Failed to reset stats of bbdev %u", dev_id); 1026 /* start the device */ 1027 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 1028 "Failed to start bbdev %u", dev_id); 1029 } 1030 1031 return TEST_SUCCESS; 1032 } 1033 1034 static void 1035 ut_teardown(void) 1036 { 1037 uint8_t i, dev_id; 1038 struct rte_bbdev_stats stats; 1039 1040 for (i = 0; i < nb_active_devs; i++) { 1041 dev_id = active_devs[i].dev_id; 1042 /* read stats and print */ 1043 rte_bbdev_stats_get(dev_id, &stats); 1044 /* Stop the device */ 1045 rte_bbdev_stop(dev_id); 1046 } 1047 } 1048 1049 static int 1050 init_op_data_objs(struct rte_bbdev_op_data *bufs, 1051 struct op_data_entries *ref_entries, 1052 struct rte_mempool *mbuf_pool, const uint16_t n, 1053 enum op_data_type op_type, uint16_t min_alignment) 1054 { 1055 int ret; 1056 unsigned int i, j; 1057 bool large_input = false; 1058 1059 for (i = 0; i < n; ++i) { 1060 char *data; 1061 struct op_data_buf *seg = &ref_entries->segments[0]; 1062 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 1063 TEST_ASSERT_NOT_NULL(m_head, 1064 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1065 op_type, n * ref_entries->nb_segments, 1066 mbuf_pool->size); 1067 1068 if ((seg->length + RTE_PKTMBUF_HEADROOM) > RTE_BBDEV_LDPC_E_MAX_MBUF) { 1069 /* 1070 * Special case when DPDK mbuf cannot handle 1071 * the required input size 1072 */ 1073 printf("Warning: Larger input size than DPDK mbuf %d\n", 1074 seg->length); 1075 large_input = true; 1076 } 1077 bufs[i].data = m_head; 1078 bufs[i].offset = 0; 1079 bufs[i].length = 0; 1080 1081 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1082 if (large_input) { 1083 /* Allocate a fake overused mbuf */ 1084 data = rte_malloc(NULL, seg->length, 0); 1085 TEST_ASSERT_NOT_NULL(data, 1086 "rte malloc failed with %u bytes", 1087 seg->length); 1088 memcpy(data, seg->addr, seg->length); 1089 m_head->buf_addr = data; 1090 rte_mbuf_iova_set(m_head, rte_malloc_virt2iova(data)); 1091 m_head->data_off = 0; 1092 m_head->data_len = seg->length; 1093 } else { 1094 data = rte_pktmbuf_append(m_head, seg->length); 1095 TEST_ASSERT_NOT_NULL(data, 1096 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1097 seg->length, op_type); 1098 1099 TEST_ASSERT(data == RTE_PTR_ALIGN( 1100 data, min_alignment), 1101 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1102 data, min_alignment); 1103 rte_memcpy(data, seg->addr, seg->length); 1104 } 1105 1106 bufs[i].length += seg->length; 1107 1108 for (j = 1; j < ref_entries->nb_segments; ++j) { 1109 struct rte_mbuf *m_tail = 1110 rte_pktmbuf_alloc(mbuf_pool); 1111 TEST_ASSERT_NOT_NULL(m_tail, 1112 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1113 op_type, 1114 n * ref_entries->nb_segments, 1115 mbuf_pool->size); 1116 seg += 1; 1117 1118 data = rte_pktmbuf_append(m_tail, seg->length); 1119 TEST_ASSERT_NOT_NULL(data, 1120 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1121 seg->length, op_type); 1122 1123 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1124 min_alignment), 1125 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1126 data, min_alignment); 1127 rte_memcpy(data, seg->addr, seg->length); 1128 bufs[i].length += seg->length; 1129 1130 ret = rte_pktmbuf_chain(m_head, m_tail); 1131 TEST_ASSERT_SUCCESS(ret, 1132 "Couldn't chain mbufs from %d data type mbuf pool", 1133 op_type); 1134 } 1135 } else { 1136 if (((op_type == DATA_HARD_OUTPUT) || (op_type == DATA_SOFT_OUTPUT)) 1137 && ((seg->length + RTE_PKTMBUF_HEADROOM) 1138 > RTE_BBDEV_LDPC_E_MAX_MBUF)) { 1139 /* Allocate a fake overused mbuf + margin */ 1140 data = rte_malloc(NULL, seg->length + 1024, 0); 1141 TEST_ASSERT_NOT_NULL(data, 1142 "rte malloc failed with %u bytes", 1143 seg->length + 1024); 1144 m_head->buf_addr = data; 1145 m_head->buf_iova = rte_malloc_virt2iova(data); 1146 m_head->data_off = 0; 1147 m_head->data_len = seg->length; 1148 } else { 1149 /* allocate chained-mbuf for output buffer */ 1150 for (j = 1; j < ref_entries->nb_segments; ++j) { 1151 struct rte_mbuf *m_tail = 1152 rte_pktmbuf_alloc(mbuf_pool); 1153 TEST_ASSERT_NOT_NULL(m_tail, 1154 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1155 op_type, 1156 n * ref_entries->nb_segments, 1157 mbuf_pool->size); 1158 1159 ret = rte_pktmbuf_chain(m_head, m_tail); 1160 TEST_ASSERT_SUCCESS(ret, 1161 "Couldn't chain mbufs from %d data type mbuf pool", 1162 op_type); 1163 } 1164 } 1165 bufs[i].length += seg->length; 1166 } 1167 } 1168 1169 return 0; 1170 } 1171 1172 static int 1173 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1174 const int socket) 1175 { 1176 int i; 1177 1178 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1179 if (*buffers == NULL) { 1180 printf("WARNING: Failed to allocate op_data on socket %d\n", 1181 socket); 1182 /* try to allocate memory on other detected sockets */ 1183 for (i = 0; i < socket; i++) { 1184 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1185 if (*buffers != NULL) 1186 break; 1187 } 1188 } 1189 1190 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1191 } 1192 1193 static void 1194 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1195 const uint16_t n, const int8_t max_llr_modulus) 1196 { 1197 uint16_t i, byte_idx; 1198 1199 for (i = 0; i < n; ++i) { 1200 struct rte_mbuf *m = input_ops[i].data; 1201 while (m != NULL) { 1202 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1203 input_ops[i].offset); 1204 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1205 ++byte_idx) 1206 llr[byte_idx] = round((double)max_llr_modulus * 1207 llr[byte_idx] / INT8_MAX); 1208 1209 m = m->next; 1210 } 1211 } 1212 } 1213 1214 /* 1215 * We may have to insert filler bits 1216 * when they are required by the HARQ assumption 1217 */ 1218 static void 1219 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1220 const uint16_t n, struct test_op_params *op_params) 1221 { 1222 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1223 1224 if (input_ops == NULL) 1225 return; 1226 /* No need to add filler if not required by device */ 1227 if (!(ldpc_cap_flags & 1228 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1229 return; 1230 /* No need to add filler for loopback operation */ 1231 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1232 return; 1233 1234 uint16_t i, j, parity_offset; 1235 for (i = 0; i < n; ++i) { 1236 struct rte_mbuf *m = input_ops[i].data; 1237 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1238 input_ops[i].offset); 1239 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1240 * dec.z_c - dec.n_filler; 1241 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1242 m->data_len = new_hin_size; 1243 input_ops[i].length = new_hin_size; 1244 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1245 j--) 1246 llr[j] = llr[j - dec.n_filler]; 1247 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1248 for (j = 0; j < dec.n_filler; j++) 1249 llr[parity_offset + j] = llr_max_pre_scaling; 1250 } 1251 } 1252 1253 static void 1254 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1255 const uint16_t n, const int8_t llr_size, 1256 const int8_t llr_decimals) 1257 { 1258 if (input_ops == NULL) 1259 return; 1260 1261 uint16_t i, byte_idx; 1262 1263 int16_t llr_max, llr_min, llr_tmp; 1264 llr_max = (1 << (llr_size - 1)) - 1; 1265 llr_min = -llr_max; 1266 for (i = 0; i < n; ++i) { 1267 struct rte_mbuf *m = input_ops[i].data; 1268 while (m != NULL) { 1269 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1270 input_ops[i].offset); 1271 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1272 ++byte_idx) { 1273 1274 llr_tmp = llr[byte_idx]; 1275 if (llr_decimals == 4) 1276 llr_tmp *= 8; 1277 else if (llr_decimals == 2) 1278 llr_tmp *= 2; 1279 else if (llr_decimals == 0) 1280 llr_tmp /= 2; 1281 llr_tmp = RTE_MIN(llr_max, 1282 RTE_MAX(llr_min, llr_tmp)); 1283 llr[byte_idx] = (int8_t) llr_tmp; 1284 } 1285 1286 m = m->next; 1287 } 1288 } 1289 } 1290 1291 1292 1293 static int 1294 fill_queue_buffers(struct test_op_params *op_params, 1295 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1296 struct rte_mempool *soft_out_mp, 1297 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1298 uint16_t queue_id, 1299 const struct rte_bbdev_op_cap *capabilities, 1300 uint16_t min_alignment, const int socket_id) 1301 { 1302 int ret; 1303 enum op_data_type type; 1304 const uint16_t n = op_params->num_to_process; 1305 1306 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1307 in_mp, 1308 soft_out_mp, 1309 hard_out_mp, 1310 harq_in_mp, 1311 harq_out_mp, 1312 }; 1313 1314 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1315 &op_params->q_bufs[socket_id][queue_id].inputs, 1316 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1317 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1318 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1319 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1320 }; 1321 1322 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1323 struct op_data_entries *ref_entries = 1324 &test_vector.entries[type]; 1325 if (ref_entries->nb_segments == 0) 1326 continue; 1327 1328 ret = allocate_buffers_on_socket(queue_ops[type], 1329 n * sizeof(struct rte_bbdev_op_data), 1330 socket_id); 1331 TEST_ASSERT_SUCCESS(ret, 1332 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1333 1334 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1335 mbuf_pools[type], n, type, min_alignment); 1336 TEST_ASSERT_SUCCESS(ret, 1337 "Couldn't init rte_bbdev_op_data structs"); 1338 } 1339 1340 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1341 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1342 capabilities->cap.turbo_dec.max_llr_modulus); 1343 1344 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1345 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1346 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1347 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1348 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1349 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1350 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1351 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1352 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1353 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1354 if (!loopback && !llr_comp) 1355 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1356 ldpc_llr_size, ldpc_llr_decimals); 1357 if (!loopback && !harq_comp) 1358 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1359 ldpc_llr_size, ldpc_llr_decimals); 1360 if (!loopback) 1361 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1362 op_params); 1363 } 1364 1365 return 0; 1366 } 1367 1368 static void 1369 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1370 { 1371 unsigned int i, j; 1372 1373 rte_mempool_free(ad->ops_mempool); 1374 rte_mempool_free(ad->in_mbuf_pool); 1375 rte_mempool_free(ad->hard_out_mbuf_pool); 1376 rte_mempool_free(ad->soft_out_mbuf_pool); 1377 rte_mempool_free(ad->harq_in_mbuf_pool); 1378 rte_mempool_free(ad->harq_out_mbuf_pool); 1379 1380 for (i = 0; i < rte_lcore_count(); ++i) { 1381 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1382 rte_free(op_params->q_bufs[j][i].inputs); 1383 rte_free(op_params->q_bufs[j][i].hard_outputs); 1384 rte_free(op_params->q_bufs[j][i].soft_outputs); 1385 rte_free(op_params->q_bufs[j][i].harq_inputs); 1386 rte_free(op_params->q_bufs[j][i].harq_outputs); 1387 } 1388 } 1389 } 1390 1391 static void 1392 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1393 unsigned int start_idx, 1394 struct rte_bbdev_op_data *inputs, 1395 struct rte_bbdev_op_data *hard_outputs, 1396 struct rte_bbdev_op_data *soft_outputs, 1397 struct rte_bbdev_dec_op *ref_op) 1398 { 1399 unsigned int i; 1400 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1401 1402 for (i = 0; i < n; ++i) { 1403 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1404 ops[i]->turbo_dec.tb_params.ea = 1405 turbo_dec->tb_params.ea; 1406 ops[i]->turbo_dec.tb_params.eb = 1407 turbo_dec->tb_params.eb; 1408 ops[i]->turbo_dec.tb_params.k_pos = 1409 turbo_dec->tb_params.k_pos; 1410 ops[i]->turbo_dec.tb_params.k_neg = 1411 turbo_dec->tb_params.k_neg; 1412 ops[i]->turbo_dec.tb_params.c = 1413 turbo_dec->tb_params.c; 1414 ops[i]->turbo_dec.tb_params.c_neg = 1415 turbo_dec->tb_params.c_neg; 1416 ops[i]->turbo_dec.tb_params.cab = 1417 turbo_dec->tb_params.cab; 1418 ops[i]->turbo_dec.tb_params.r = 1419 turbo_dec->tb_params.r; 1420 } else { 1421 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1422 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1423 } 1424 1425 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1426 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1427 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1428 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1429 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1430 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1431 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1432 1433 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1434 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1435 if (soft_outputs != NULL) 1436 ops[i]->turbo_dec.soft_output = 1437 soft_outputs[start_idx + i]; 1438 } 1439 } 1440 1441 static void 1442 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1443 unsigned int start_idx, 1444 struct rte_bbdev_op_data *inputs, 1445 struct rte_bbdev_op_data *outputs, 1446 struct rte_bbdev_enc_op *ref_op) 1447 { 1448 unsigned int i; 1449 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1450 for (i = 0; i < n; ++i) { 1451 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1452 ops[i]->turbo_enc.tb_params.ea = 1453 turbo_enc->tb_params.ea; 1454 ops[i]->turbo_enc.tb_params.eb = 1455 turbo_enc->tb_params.eb; 1456 ops[i]->turbo_enc.tb_params.k_pos = 1457 turbo_enc->tb_params.k_pos; 1458 ops[i]->turbo_enc.tb_params.k_neg = 1459 turbo_enc->tb_params.k_neg; 1460 ops[i]->turbo_enc.tb_params.c = 1461 turbo_enc->tb_params.c; 1462 ops[i]->turbo_enc.tb_params.c_neg = 1463 turbo_enc->tb_params.c_neg; 1464 ops[i]->turbo_enc.tb_params.cab = 1465 turbo_enc->tb_params.cab; 1466 ops[i]->turbo_enc.tb_params.ncb_pos = 1467 turbo_enc->tb_params.ncb_pos; 1468 ops[i]->turbo_enc.tb_params.ncb_neg = 1469 turbo_enc->tb_params.ncb_neg; 1470 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1471 } else { 1472 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1473 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1474 ops[i]->turbo_enc.cb_params.ncb = 1475 turbo_enc->cb_params.ncb; 1476 } 1477 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1478 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1479 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1480 1481 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1482 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1483 } 1484 } 1485 1486 1487 /* Returns a random number drawn from a normal distribution 1488 * with mean of 0 and variance of 1 1489 * Marsaglia algorithm 1490 */ 1491 static double 1492 randn(int n) 1493 { 1494 double S, Z, U1, U2, u, v, fac; 1495 1496 do { 1497 U1 = (double)rand() / RAND_MAX; 1498 U2 = (double)rand() / RAND_MAX; 1499 u = 2. * U1 - 1.; 1500 v = 2. * U2 - 1.; 1501 S = u * u + v * v; 1502 } while (S >= 1 || S == 0); 1503 fac = sqrt(-2. * log(S) / S); 1504 Z = (n % 2) ? u * fac : v * fac; 1505 return Z; 1506 } 1507 1508 static inline double 1509 maxstar(double A, double B) 1510 { 1511 if (fabs(A - B) > 5) 1512 return RTE_MAX(A, B); 1513 else 1514 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1515 } 1516 1517 /* 1518 * Generate Qm LLRS for Qm==8 1519 * Modulation, AWGN and LLR estimation from max log development 1520 */ 1521 static void 1522 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1523 { 1524 int qm = 8; 1525 int qam = 256; 1526 int m, k; 1527 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1528 /* 5.1.4 of TS38.211 */ 1529 const double symbols_I[256] = { 1530 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1531 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1532 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1533 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1534 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1535 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1536 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1537 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1538 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1539 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1540 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1541 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1542 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1543 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1544 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1545 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1546 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1547 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1548 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1549 -13, -13, -15, -15, -13, -13, -15, -15}; 1550 const double symbols_Q[256] = { 1551 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1552 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1553 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1554 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1555 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1556 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1557 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1558 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1559 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1560 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1561 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1562 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1563 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1564 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1565 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1566 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1567 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1568 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1569 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1570 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1571 /* Average constellation point energy */ 1572 N0 *= 170.0; 1573 for (k = 0; k < qm; k++) 1574 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1575 /* 5.1.4 of TS38.211 */ 1576 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1577 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1578 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1579 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1580 /* AWGN channel */ 1581 I += sqrt(N0 / 2) * randn(0); 1582 Q += sqrt(N0 / 2) * randn(1); 1583 /* 1584 * Calculate the log of the probability that each of 1585 * the constellation points was transmitted 1586 */ 1587 for (m = 0; m < qam; m++) 1588 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1589 + pow(Q - symbols_Q[m], 2.0)) / N0; 1590 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1591 for (k = 0; k < qm; k++) { 1592 p0 = -999999; 1593 p1 = -999999; 1594 /* For each constellation point */ 1595 for (m = 0; m < qam; m++) { 1596 if ((m >> (qm - k - 1)) & 1) 1597 p1 = maxstar(p1, log_syml_prob[m]); 1598 else 1599 p0 = maxstar(p0, log_syml_prob[m]); 1600 } 1601 /* Calculate the LLR */ 1602 llr_ = p0 - p1; 1603 llr_ *= (1 << ldpc_llr_decimals); 1604 llr_ = round(llr_); 1605 if (llr_ > llr_max) 1606 llr_ = llr_max; 1607 if (llr_ < -llr_max) 1608 llr_ = -llr_max; 1609 llrs[qm * i + k] = (int8_t) llr_; 1610 } 1611 } 1612 1613 1614 /* 1615 * Generate Qm LLRS for Qm==6 1616 * Modulation, AWGN and LLR estimation from max log development 1617 */ 1618 static void 1619 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1620 { 1621 int qm = 6; 1622 int qam = 64; 1623 int m, k; 1624 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1625 /* 5.1.4 of TS38.211 */ 1626 const double symbols_I[64] = { 1627 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1628 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1629 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1630 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1631 -5, -5, -7, -7, -5, -5, -7, -7}; 1632 const double symbols_Q[64] = { 1633 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1634 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1635 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1636 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1637 -3, -1, -3, -1, -5, -7, -5, -7}; 1638 /* Average constellation point energy */ 1639 N0 *= 42.0; 1640 for (k = 0; k < qm; k++) 1641 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1642 /* 5.1.4 of TS38.211 */ 1643 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1644 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1645 /* AWGN channel */ 1646 I += sqrt(N0 / 2) * randn(0); 1647 Q += sqrt(N0 / 2) * randn(1); 1648 /* 1649 * Calculate the log of the probability that each of 1650 * the constellation points was transmitted 1651 */ 1652 for (m = 0; m < qam; m++) 1653 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1654 + pow(Q - symbols_Q[m], 2.0)) / N0; 1655 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1656 for (k = 0; k < qm; k++) { 1657 p0 = -999999; 1658 p1 = -999999; 1659 /* For each constellation point */ 1660 for (m = 0; m < qam; m++) { 1661 if ((m >> (qm - k - 1)) & 1) 1662 p1 = maxstar(p1, log_syml_prob[m]); 1663 else 1664 p0 = maxstar(p0, log_syml_prob[m]); 1665 } 1666 /* Calculate the LLR */ 1667 llr_ = p0 - p1; 1668 llr_ *= (1 << ldpc_llr_decimals); 1669 llr_ = round(llr_); 1670 if (llr_ > llr_max) 1671 llr_ = llr_max; 1672 if (llr_ < -llr_max) 1673 llr_ = -llr_max; 1674 llrs[qm * i + k] = (int8_t) llr_; 1675 } 1676 } 1677 1678 /* 1679 * Generate Qm LLRS for Qm==4 1680 * Modulation, AWGN and LLR estimation from max log development 1681 */ 1682 static void 1683 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1684 { 1685 int qm = 4; 1686 int qam = 16; 1687 int m, k; 1688 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1689 /* 5.1.4 of TS38.211 */ 1690 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1691 -1, -1, -3, -3, -1, -1, -3, -3}; 1692 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1693 1, 3, 1, 3, -1, -3, -1, -3}; 1694 /* Average constellation point energy */ 1695 N0 *= 10.0; 1696 for (k = 0; k < qm; k++) 1697 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1698 /* 5.1.4 of TS38.211 */ 1699 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1700 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1701 /* AWGN channel */ 1702 I += sqrt(N0 / 2) * randn(0); 1703 Q += sqrt(N0 / 2) * randn(1); 1704 /* 1705 * Calculate the log of the probability that each of 1706 * the constellation points was transmitted 1707 */ 1708 for (m = 0; m < qam; m++) 1709 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1710 + pow(Q - symbols_Q[m], 2.0)) / N0; 1711 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1712 for (k = 0; k < qm; k++) { 1713 p0 = -999999; 1714 p1 = -999999; 1715 /* For each constellation point */ 1716 for (m = 0; m < qam; m++) { 1717 if ((m >> (qm - k - 1)) & 1) 1718 p1 = maxstar(p1, log_syml_prob[m]); 1719 else 1720 p0 = maxstar(p0, log_syml_prob[m]); 1721 } 1722 /* Calculate the LLR */ 1723 llr_ = p0 - p1; 1724 llr_ *= (1 << ldpc_llr_decimals); 1725 llr_ = round(llr_); 1726 if (llr_ > llr_max) 1727 llr_ = llr_max; 1728 if (llr_ < -llr_max) 1729 llr_ = -llr_max; 1730 llrs[qm * i + k] = (int8_t) llr_; 1731 } 1732 } 1733 1734 static void 1735 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1736 { 1737 double b, b1, n; 1738 double coeff = 2.0 * sqrt(N0); 1739 1740 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1741 if (llrs[j] < 8 && llrs[j] > -8) 1742 return; 1743 1744 /* Note don't change sign here */ 1745 n = randn(j % 2); 1746 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1747 + coeff * n) / N0; 1748 b = b1 * (1 << ldpc_llr_decimals); 1749 b = round(b); 1750 if (b > llr_max) 1751 b = llr_max; 1752 if (b < -llr_max) 1753 b = -llr_max; 1754 llrs[j] = (int8_t) b; 1755 } 1756 1757 /* Generate LLR for a given SNR */ 1758 static void 1759 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1760 struct rte_bbdev_dec_op *ref_op) 1761 { 1762 struct rte_mbuf *m; 1763 uint16_t qm; 1764 uint32_t i, j, e, range; 1765 double N0, llr_max; 1766 1767 e = ref_op->ldpc_dec.cb_params.e; 1768 qm = ref_op->ldpc_dec.q_m; 1769 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1770 range = e / qm; 1771 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1772 1773 for (i = 0; i < n; ++i) { 1774 m = inputs[i].data; 1775 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1776 if (qm == 8) { 1777 for (j = 0; j < range; ++j) 1778 gen_qm8_llr(llrs, j, N0, llr_max); 1779 } else if (qm == 6) { 1780 for (j = 0; j < range; ++j) 1781 gen_qm6_llr(llrs, j, N0, llr_max); 1782 } else if (qm == 4) { 1783 for (j = 0; j < range; ++j) 1784 gen_qm4_llr(llrs, j, N0, llr_max); 1785 } else { 1786 for (j = 0; j < e; ++j) 1787 gen_qm2_llr(llrs, j, N0, llr_max); 1788 } 1789 } 1790 } 1791 1792 static void 1793 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1794 unsigned int start_idx, 1795 struct rte_bbdev_op_data *inputs, 1796 struct rte_bbdev_op_data *hard_outputs, 1797 struct rte_bbdev_op_data *soft_outputs, 1798 struct rte_bbdev_op_data *harq_inputs, 1799 struct rte_bbdev_op_data *harq_outputs, 1800 struct rte_bbdev_dec_op *ref_op) 1801 { 1802 unsigned int i; 1803 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1804 1805 for (i = 0; i < n; ++i) { 1806 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1807 ops[i]->ldpc_dec.tb_params.ea = 1808 ldpc_dec->tb_params.ea; 1809 ops[i]->ldpc_dec.tb_params.eb = 1810 ldpc_dec->tb_params.eb; 1811 ops[i]->ldpc_dec.tb_params.c = 1812 ldpc_dec->tb_params.c; 1813 ops[i]->ldpc_dec.tb_params.cab = 1814 ldpc_dec->tb_params.cab; 1815 ops[i]->ldpc_dec.tb_params.r = 1816 ldpc_dec->tb_params.r; 1817 } else { 1818 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1819 } 1820 1821 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1822 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1823 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1824 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1825 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1826 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1827 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1828 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1829 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1830 1831 if (hard_outputs != NULL) 1832 ops[i]->ldpc_dec.hard_output = 1833 hard_outputs[start_idx + i]; 1834 if (inputs != NULL) 1835 ops[i]->ldpc_dec.input = 1836 inputs[start_idx + i]; 1837 if (soft_outputs != NULL) 1838 ops[i]->ldpc_dec.soft_output = 1839 soft_outputs[start_idx + i]; 1840 if (harq_inputs != NULL) 1841 ops[i]->ldpc_dec.harq_combined_input = 1842 harq_inputs[start_idx + i]; 1843 if (harq_outputs != NULL) 1844 ops[i]->ldpc_dec.harq_combined_output = 1845 harq_outputs[start_idx + i]; 1846 } 1847 } 1848 1849 1850 static void 1851 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1852 unsigned int start_idx, 1853 struct rte_bbdev_op_data *inputs, 1854 struct rte_bbdev_op_data *outputs, 1855 struct rte_bbdev_enc_op *ref_op) 1856 { 1857 unsigned int i; 1858 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1859 for (i = 0; i < n; ++i) { 1860 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1861 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1862 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1863 ops[i]->ldpc_enc.tb_params.cab = 1864 ldpc_enc->tb_params.cab; 1865 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1866 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1867 } else { 1868 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1869 } 1870 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1871 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1872 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1873 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1874 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1875 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1876 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1877 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1878 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1879 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1880 } 1881 } 1882 1883 static void 1884 copy_reference_fft_op(struct rte_bbdev_fft_op **ops, unsigned int n, 1885 unsigned int start_idx, struct rte_bbdev_op_data *inputs, 1886 struct rte_bbdev_op_data *outputs, struct rte_bbdev_op_data *pwrouts, 1887 struct rte_bbdev_fft_op *ref_op) 1888 { 1889 unsigned int i, j; 1890 struct rte_bbdev_op_fft *fft = &ref_op->fft; 1891 for (i = 0; i < n; i++) { 1892 ops[i]->fft.input_sequence_size = fft->input_sequence_size; 1893 ops[i]->fft.input_leading_padding = fft->input_leading_padding; 1894 ops[i]->fft.output_sequence_size = fft->output_sequence_size; 1895 ops[i]->fft.output_leading_depadding = 1896 fft->output_leading_depadding; 1897 for (j = 0; j < RTE_BBDEV_MAX_CS_2; j++) 1898 ops[i]->fft.window_index[j] = fft->window_index[j]; 1899 ops[i]->fft.cs_bitmap = fft->cs_bitmap; 1900 ops[i]->fft.num_antennas_log2 = fft->num_antennas_log2; 1901 ops[i]->fft.idft_log2 = fft->idft_log2; 1902 ops[i]->fft.dft_log2 = fft->dft_log2; 1903 ops[i]->fft.cs_time_adjustment = fft->cs_time_adjustment; 1904 ops[i]->fft.idft_shift = fft->idft_shift; 1905 ops[i]->fft.dft_shift = fft->dft_shift; 1906 ops[i]->fft.ncs_reciprocal = fft->ncs_reciprocal; 1907 ops[i]->fft.power_shift = fft->power_shift; 1908 ops[i]->fft.fp16_exp_adjust = fft->fp16_exp_adjust; 1909 ops[i]->fft.base_output = outputs[start_idx + i]; 1910 ops[i]->fft.base_input = inputs[start_idx + i]; 1911 if (pwrouts != NULL) 1912 ops[i]->fft.power_meas_output = pwrouts[start_idx + i]; 1913 ops[i]->fft.op_flags = fft->op_flags; 1914 } 1915 } 1916 1917 static int 1918 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1919 unsigned int order_idx, const int expected_status) 1920 { 1921 int status = op->status; 1922 /* ignore parity mismatch false alarms for long iterations */ 1923 if (get_iter_max() >= 10) { 1924 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1925 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1926 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1927 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 1928 } 1929 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1930 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1931 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1932 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 1933 } 1934 } 1935 1936 TEST_ASSERT(status == expected_status, 1937 "op_status (%d) != expected_status (%d)", 1938 op->status, expected_status); 1939 1940 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1941 "Ordering error, expected %p, got %p", 1942 (void *)(uintptr_t)order_idx, op->opaque_data); 1943 1944 return TEST_SUCCESS; 1945 } 1946 1947 static int 1948 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1949 unsigned int order_idx, const int expected_status) 1950 { 1951 TEST_ASSERT(op->status == expected_status, 1952 "op_status (%d) != expected_status (%d)", 1953 op->status, expected_status); 1954 1955 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) 1956 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1957 "Ordering error, expected %p, got %p", 1958 (void *)(uintptr_t)order_idx, op->opaque_data); 1959 1960 return TEST_SUCCESS; 1961 } 1962 1963 static int 1964 check_fft_status_and_ordering(struct rte_bbdev_fft_op *op, 1965 unsigned int order_idx, const int expected_status) 1966 { 1967 TEST_ASSERT(op->status == expected_status, 1968 "op_status (%d) != expected_status (%d)", 1969 op->status, expected_status); 1970 1971 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1972 "Ordering error, expected %p, got %p", 1973 (void *)(uintptr_t)order_idx, op->opaque_data); 1974 1975 return TEST_SUCCESS; 1976 } 1977 1978 static inline int 1979 validate_op_chain(struct rte_bbdev_op_data *op, 1980 struct op_data_entries *orig_op) 1981 { 1982 uint8_t i; 1983 struct rte_mbuf *m = op->data; 1984 uint8_t nb_dst_segments = orig_op->nb_segments; 1985 uint32_t total_data_size = 0; 1986 1987 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1988 "Number of segments differ in original (%u) and filled (%u) op", 1989 nb_dst_segments, m->nb_segs); 1990 1991 /* Validate each mbuf segment length */ 1992 for (i = 0; i < nb_dst_segments; ++i) { 1993 /* Apply offset to the first mbuf segment */ 1994 uint16_t offset = (i == 0) ? op->offset : 0; 1995 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1996 total_data_size += orig_op->segments[i].length; 1997 1998 TEST_ASSERT(orig_op->segments[i].length == data_len, 1999 "Length of segment differ in original (%u) and filled (%u) op", 2000 orig_op->segments[i].length, data_len); 2001 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 2002 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 2003 data_len, 2004 "Output buffers (CB=%u) are not equal", i); 2005 m = m->next; 2006 } 2007 2008 /* Validate total mbuf pkt length */ 2009 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2010 TEST_ASSERT(total_data_size == pkt_len, 2011 "Length of data differ in original (%u) and filled (%u) op", 2012 total_data_size, pkt_len); 2013 2014 return TEST_SUCCESS; 2015 } 2016 2017 /* 2018 * Compute K0 for a given configuration for HARQ output length computation 2019 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 2020 */ 2021 static inline uint16_t 2022 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 2023 { 2024 if (rv_index == 0) 2025 return 0; 2026 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 2027 if (n_cb == n) { 2028 if (rv_index == 1) 2029 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 2030 else if (rv_index == 2) 2031 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 2032 else 2033 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 2034 } 2035 /* LBRM case - includes a division by N */ 2036 if (rv_index == 1) 2037 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 2038 / n) * z_c; 2039 else if (rv_index == 2) 2040 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 2041 / n) * z_c; 2042 else 2043 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 2044 / n) * z_c; 2045 } 2046 2047 /* HARQ output length including the Filler bits */ 2048 static inline uint16_t 2049 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 2050 { 2051 uint16_t k0 = 0; 2052 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 2053 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 2054 /* Compute RM out size and number of rows */ 2055 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2056 * ops_ld->z_c - ops_ld->n_filler; 2057 uint16_t deRmOutSize = RTE_MIN( 2058 k0 + ops_ld->cb_params.e + 2059 ((k0 > parity_offset) ? 2060 0 : ops_ld->n_filler), 2061 ops_ld->n_cb); 2062 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 2063 / ops_ld->z_c); 2064 uint16_t harq_output_len = numRows * ops_ld->z_c; 2065 return harq_output_len; 2066 } 2067 2068 static inline int 2069 validate_op_harq_chain(struct rte_bbdev_op_data *op, 2070 struct op_data_entries *orig_op, 2071 struct rte_bbdev_op_ldpc_dec *ops_ld) 2072 { 2073 uint8_t i; 2074 uint32_t j, jj, k; 2075 struct rte_mbuf *m = op->data; 2076 uint8_t nb_dst_segments = orig_op->nb_segments; 2077 uint32_t total_data_size = 0; 2078 int8_t *harq_orig, *harq_out, abs_harq_origin; 2079 uint32_t byte_error = 0, cum_error = 0, error; 2080 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 2081 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 2082 uint16_t parity_offset; 2083 2084 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2085 "Number of segments differ in original (%u) and filled (%u) op", 2086 nb_dst_segments, m->nb_segs); 2087 2088 /* Validate each mbuf segment length */ 2089 for (i = 0; i < nb_dst_segments; ++i) { 2090 /* Apply offset to the first mbuf segment */ 2091 uint16_t offset = (i == 0) ? op->offset : 0; 2092 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2093 total_data_size += orig_op->segments[i].length; 2094 2095 TEST_ASSERT(orig_op->segments[i].length < 2096 (uint32_t)(data_len + 64), 2097 "Length of segment differ in original (%u) and filled (%u) op", 2098 orig_op->segments[i].length, data_len); 2099 harq_orig = (int8_t *) orig_op->segments[i].addr; 2100 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2101 2102 if (!(ldpc_cap_flags & 2103 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 2104 ) || (ops_ld->op_flags & 2105 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2106 data_len -= ops_ld->z_c; 2107 parity_offset = data_len; 2108 } else { 2109 /* Compute RM out size and number of rows */ 2110 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 2111 * ops_ld->z_c - ops_ld->n_filler; 2112 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 2113 ops_ld->n_filler; 2114 if (data_len > deRmOutSize) 2115 data_len = deRmOutSize; 2116 if (data_len > orig_op->segments[i].length) 2117 data_len = orig_op->segments[i].length; 2118 } 2119 /* 2120 * HARQ output can have minor differences 2121 * due to integer representation and related scaling 2122 */ 2123 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2124 if (j == parity_offset) { 2125 /* Special Handling of the filler bits */ 2126 for (k = 0; k < ops_ld->n_filler; k++) { 2127 if (harq_out[jj] != 2128 llr_max_pre_scaling) { 2129 printf("HARQ Filler issue %d: %d %d\n", 2130 jj, harq_out[jj], 2131 llr_max); 2132 byte_error++; 2133 } 2134 jj++; 2135 } 2136 } 2137 if (!(ops_ld->op_flags & 2138 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2139 if (ldpc_llr_decimals > 1) 2140 harq_out[jj] = (harq_out[jj] + 1) 2141 >> (ldpc_llr_decimals - 1); 2142 /* Saturated to S7 */ 2143 if (harq_orig[j] > llr_max) 2144 harq_orig[j] = llr_max; 2145 if (harq_orig[j] < -llr_max) 2146 harq_orig[j] = -llr_max; 2147 } 2148 if (harq_orig[j] != harq_out[jj]) { 2149 error = (harq_orig[j] > harq_out[jj]) ? 2150 harq_orig[j] - harq_out[jj] : 2151 harq_out[jj] - harq_orig[j]; 2152 abs_harq_origin = harq_orig[j] > 0 ? 2153 harq_orig[j] : 2154 -harq_orig[j]; 2155 /* Residual quantization error */ 2156 if ((error > 8 && (abs_harq_origin < 2157 (llr_max - 16))) || 2158 (error > 16)) { 2159 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2160 j, harq_orig[j], 2161 harq_out[jj], error); 2162 byte_error++; 2163 cum_error += error; 2164 } 2165 } 2166 } 2167 m = m->next; 2168 } 2169 2170 if (byte_error) 2171 TEST_ASSERT(byte_error <= 1, 2172 "HARQ output mismatch (%d) %d", 2173 byte_error, cum_error); 2174 2175 /* Validate total mbuf pkt length */ 2176 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2177 TEST_ASSERT(total_data_size < pkt_len + 64, 2178 "Length of data differ in original (%u) and filled (%u) op", 2179 total_data_size, pkt_len); 2180 2181 return TEST_SUCCESS; 2182 } 2183 2184 2185 static inline int 2186 validate_op_so_chain(struct rte_bbdev_op_data *op, 2187 struct op_data_entries *orig_op) 2188 { 2189 struct rte_mbuf *m = op->data; 2190 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2191 uint32_t j, jj; 2192 int8_t *so_orig, *so_out; 2193 uint32_t byte_error = 0, error, margin_error = 0; 2194 2195 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2196 "Number of segments differ in original (%u) and filled (%u) op", 2197 nb_dst_segments, m->nb_segs); 2198 2199 /* Validate each mbuf segment length. */ 2200 for (i = 0; i < nb_dst_segments; ++i) { 2201 /* Apply offset to the first mbuf segment. */ 2202 uint16_t offset = (i == 0) ? op->offset : 0; 2203 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 2204 2205 TEST_ASSERT(orig_op->segments[i].length == data_len, 2206 "Length of segment differ in original (%u) and filled (%u) op", 2207 orig_op->segments[i].length, data_len); 2208 so_orig = (int8_t *) orig_op->segments[i].addr; 2209 so_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 2210 margin_error += data_len / 8; /* Allow for few % errors. */ 2211 2212 /* SO output can have minor differences due to algorithm variations. */ 2213 for (j = 0, jj = 0; j < data_len; j++, jj++) { 2214 if (so_orig[j] != so_out[jj]) { 2215 error = (so_orig[j] > so_out[jj]) ? so_orig[j] - so_out[jj] : 2216 so_out[jj] - so_orig[j]; 2217 /* Residual quantization error. */ 2218 if (error > 32) { 2219 printf("Warning: Soft mismatch %d: exp %d act %d => %d\n", 2220 j, so_orig[j], so_out[jj], error); 2221 byte_error++; 2222 } 2223 } 2224 } 2225 m = m->next; 2226 } 2227 2228 if (byte_error > margin_error) 2229 TEST_ASSERT(byte_error <= 1, "Soft output mismatch (%d) %d", 2230 byte_error, margin_error); 2231 2232 return TEST_SUCCESS; 2233 } 2234 2235 static int 2236 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2237 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2238 { 2239 unsigned int i; 2240 int ret; 2241 struct op_data_entries *hard_data_orig = 2242 &test_vector.entries[DATA_HARD_OUTPUT]; 2243 struct op_data_entries *soft_data_orig = 2244 &test_vector.entries[DATA_SOFT_OUTPUT]; 2245 struct rte_bbdev_op_turbo_dec *ops_td; 2246 struct rte_bbdev_op_data *hard_output; 2247 struct rte_bbdev_op_data *soft_output; 2248 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 2249 2250 for (i = 0; i < n; ++i) { 2251 ops_td = &ops[i]->turbo_dec; 2252 hard_output = &ops_td->hard_output; 2253 soft_output = &ops_td->soft_output; 2254 2255 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2256 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2257 "Returned iter_count (%d) > expected iter_count (%d)", 2258 ops_td->iter_count, ref_td->iter_count); 2259 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2260 TEST_ASSERT_SUCCESS(ret, 2261 "Checking status and ordering for decoder failed"); 2262 2263 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2264 hard_data_orig), 2265 "Hard output buffers (CB=%u) are not equal", 2266 i); 2267 2268 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2269 TEST_ASSERT_SUCCESS(validate_op_so_chain(soft_output, 2270 soft_data_orig), 2271 "Soft output buffers (CB=%u) are not equal", 2272 i); 2273 } 2274 2275 return TEST_SUCCESS; 2276 } 2277 2278 /* Check Number of code blocks errors */ 2279 static int 2280 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2281 { 2282 unsigned int i; 2283 struct op_data_entries *hard_data_orig = 2284 &test_vector.entries[DATA_HARD_OUTPUT]; 2285 struct rte_bbdev_op_ldpc_dec *ops_td; 2286 struct rte_bbdev_op_data *hard_output; 2287 int errors = 0; 2288 struct rte_mbuf *m; 2289 2290 for (i = 0; i < n; ++i) { 2291 ops_td = &ops[i]->ldpc_dec; 2292 hard_output = &ops_td->hard_output; 2293 m = hard_output->data; 2294 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2295 hard_data_orig->segments[0].addr, 2296 hard_data_orig->segments[0].length)) 2297 errors++; 2298 } 2299 return errors; 2300 } 2301 2302 static int 2303 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2304 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2305 { 2306 unsigned int i; 2307 int ret; 2308 struct op_data_entries *hard_data_orig = 2309 &test_vector.entries[DATA_HARD_OUTPUT]; 2310 struct op_data_entries *soft_data_orig = 2311 &test_vector.entries[DATA_SOFT_OUTPUT]; 2312 struct op_data_entries *harq_data_orig = 2313 &test_vector.entries[DATA_HARQ_OUTPUT]; 2314 struct rte_bbdev_op_ldpc_dec *ops_td; 2315 struct rte_bbdev_op_data *hard_output; 2316 struct rte_bbdev_op_data *harq_output; 2317 struct rte_bbdev_op_data *soft_output; 2318 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2319 2320 for (i = 0; i < n; ++i) { 2321 ops_td = &ops[i]->ldpc_dec; 2322 hard_output = &ops_td->hard_output; 2323 harq_output = &ops_td->harq_combined_output; 2324 soft_output = &ops_td->soft_output; 2325 2326 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2327 TEST_ASSERT_SUCCESS(ret, 2328 "Checking status and ordering for decoder failed"); 2329 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2330 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2331 "Returned iter_count (%d) > expected iter_count (%d)", 2332 ops_td->iter_count, ref_td->iter_count); 2333 /* 2334 * We can ignore output data when the decoding failed to 2335 * converge or for loop-back cases 2336 */ 2337 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2338 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2339 ) && ( 2340 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2341 )) == 0) 2342 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2343 hard_data_orig), 2344 "Hard output buffers (CB=%u) are not equal", 2345 i); 2346 2347 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2348 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2349 soft_data_orig), 2350 "Soft output buffers (CB=%u) are not equal", 2351 i); 2352 if (ref_op->ldpc_dec.op_flags & 2353 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2354 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2355 harq_data_orig, ops_td), 2356 "HARQ output buffers (CB=%u) are not equal", 2357 i); 2358 } 2359 if (ref_op->ldpc_dec.op_flags & 2360 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2361 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2362 harq_data_orig, ops_td), 2363 "HARQ output buffers (CB=%u) are not equal", 2364 i); 2365 2366 } 2367 2368 return TEST_SUCCESS; 2369 } 2370 2371 2372 static int 2373 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2374 struct rte_bbdev_enc_op *ref_op) 2375 { 2376 unsigned int i; 2377 int ret; 2378 struct op_data_entries *hard_data_orig = 2379 &test_vector.entries[DATA_HARD_OUTPUT]; 2380 2381 for (i = 0; i < n; ++i) { 2382 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2383 TEST_ASSERT_SUCCESS(ret, 2384 "Checking status and ordering for encoder failed"); 2385 TEST_ASSERT_SUCCESS(validate_op_chain( 2386 &ops[i]->turbo_enc.output, 2387 hard_data_orig), 2388 "Output buffers (CB=%u) are not equal", 2389 i); 2390 } 2391 2392 return TEST_SUCCESS; 2393 } 2394 2395 static int 2396 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2397 struct rte_bbdev_enc_op *ref_op) 2398 { 2399 unsigned int i; 2400 int ret; 2401 struct op_data_entries *hard_data_orig = 2402 &test_vector.entries[DATA_HARD_OUTPUT]; 2403 2404 for (i = 0; i < n; ++i) { 2405 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2406 TEST_ASSERT_SUCCESS(ret, 2407 "Checking status and ordering for encoder failed"); 2408 TEST_ASSERT_SUCCESS(validate_op_chain( 2409 &ops[i]->ldpc_enc.output, 2410 hard_data_orig), 2411 "Output buffers (CB=%u) are not equal", 2412 i); 2413 } 2414 2415 return TEST_SUCCESS; 2416 } 2417 2418 2419 static inline int 2420 validate_op_fft_chain(struct rte_bbdev_op_data *op, struct op_data_entries *orig_op) 2421 { 2422 struct rte_mbuf *m = op->data; 2423 uint8_t i, nb_dst_segments = orig_op->nb_segments; 2424 int16_t delt, abs_delt, thres_hold = 3; 2425 uint32_t j, data_len_iq, error_num; 2426 int16_t *ref_out, *op_out; 2427 2428 TEST_ASSERT(nb_dst_segments == m->nb_segs, 2429 "Number of segments differ in original (%u) and filled (%u) op fft", 2430 nb_dst_segments, m->nb_segs); 2431 2432 /* Due to size limitation of mbuf, FFT doesn't use real mbuf. */ 2433 for (i = 0; i < nb_dst_segments; ++i) { 2434 uint16_t offset = (i == 0) ? op->offset : 0; 2435 uint32_t data_len = op->length; 2436 2437 TEST_ASSERT(orig_op->segments[i].length == data_len, 2438 "Length of segment differ in original (%u) and filled (%u) op fft", 2439 orig_op->segments[i].length, data_len); 2440 /* Divided by 2 to get the number of 16bits data. */ 2441 data_len_iq = data_len >> 1; 2442 ref_out = (int16_t *)(orig_op->segments[i].addr); 2443 op_out = rte_pktmbuf_mtod_offset(m, int16_t *, offset); 2444 error_num = 0; 2445 for (j = 0; j < data_len_iq; j++) { 2446 delt = ref_out[j] - op_out[j]; 2447 abs_delt = delt > 0 ? delt : -delt; 2448 error_num += (abs_delt > thres_hold ? 1 : 0); 2449 } 2450 if (error_num > 0) { 2451 rte_memdump(stdout, "Buffer A", ref_out, data_len); 2452 rte_memdump(stdout, "Buffer B", op_out, data_len); 2453 TEST_ASSERT(error_num == 0, 2454 "FFT Output are not matched total (%u) errors (%u)", 2455 data_len_iq, error_num); 2456 } 2457 2458 m = m->next; 2459 } 2460 2461 return TEST_SUCCESS; 2462 } 2463 2464 static int 2465 validate_fft_op(struct rte_bbdev_fft_op **ops, const uint16_t n, 2466 struct rte_bbdev_fft_op *ref_op) 2467 { 2468 unsigned int i; 2469 int ret; 2470 struct op_data_entries *fft_data_orig = &test_vector.entries[DATA_HARD_OUTPUT]; 2471 struct op_data_entries *fft_pwr_orig = &test_vector.entries[DATA_SOFT_OUTPUT]; 2472 2473 for (i = 0; i < n; ++i) { 2474 ret = check_fft_status_and_ordering(ops[i], i, ref_op->status); 2475 TEST_ASSERT_SUCCESS(ret, "Checking status and ordering for FFT failed"); 2476 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2477 &ops[i]->fft.base_output, fft_data_orig), 2478 "FFT Output buffers (op=%u) are not matched", i); 2479 if (check_bit(ops[i]->fft.op_flags, RTE_BBDEV_FFT_POWER_MEAS)) 2480 TEST_ASSERT_SUCCESS(validate_op_fft_chain( 2481 &ops[i]->fft.power_meas_output, fft_pwr_orig), 2482 "FFT Power Output buffers (op=%u) are not matched", i); 2483 } 2484 2485 return TEST_SUCCESS; 2486 } 2487 2488 static void 2489 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2490 { 2491 unsigned int i; 2492 struct op_data_entries *entry; 2493 2494 op->turbo_dec = test_vector.turbo_dec; 2495 entry = &test_vector.entries[DATA_INPUT]; 2496 for (i = 0; i < entry->nb_segments; ++i) 2497 op->turbo_dec.input.length += 2498 entry->segments[i].length; 2499 } 2500 2501 static void 2502 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2503 { 2504 unsigned int i; 2505 struct op_data_entries *entry; 2506 2507 op->ldpc_dec = test_vector.ldpc_dec; 2508 entry = &test_vector.entries[DATA_INPUT]; 2509 for (i = 0; i < entry->nb_segments; ++i) 2510 op->ldpc_dec.input.length += 2511 entry->segments[i].length; 2512 if (test_vector.ldpc_dec.op_flags & 2513 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2514 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2515 for (i = 0; i < entry->nb_segments; ++i) 2516 op->ldpc_dec.harq_combined_input.length += 2517 entry->segments[i].length; 2518 } 2519 } 2520 2521 static void 2522 create_reference_fft_op(struct rte_bbdev_fft_op *op) 2523 { 2524 unsigned int i; 2525 struct op_data_entries *entry; 2526 op->fft = test_vector.fft; 2527 entry = &test_vector.entries[DATA_INPUT]; 2528 for (i = 0; i < entry->nb_segments; ++i) 2529 op->fft.base_input.length += entry->segments[i].length; 2530 } 2531 2532 static void 2533 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2534 { 2535 unsigned int i; 2536 struct op_data_entries *entry; 2537 2538 op->turbo_enc = test_vector.turbo_enc; 2539 entry = &test_vector.entries[DATA_INPUT]; 2540 for (i = 0; i < entry->nb_segments; ++i) 2541 op->turbo_enc.input.length += 2542 entry->segments[i].length; 2543 } 2544 2545 static void 2546 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2547 { 2548 unsigned int i; 2549 struct op_data_entries *entry; 2550 2551 op->ldpc_enc = test_vector.ldpc_enc; 2552 entry = &test_vector.entries[DATA_INPUT]; 2553 for (i = 0; i < entry->nb_segments; ++i) 2554 op->ldpc_enc.input.length += 2555 entry->segments[i].length; 2556 } 2557 2558 static uint32_t 2559 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2560 { 2561 uint8_t i; 2562 uint32_t c, r, tb_size = 0; 2563 2564 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2565 tb_size = op->turbo_dec.tb_params.k_neg; 2566 } else { 2567 c = op->turbo_dec.tb_params.c; 2568 r = op->turbo_dec.tb_params.r; 2569 for (i = 0; i < c-r; i++) 2570 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2571 op->turbo_dec.tb_params.k_neg : 2572 op->turbo_dec.tb_params.k_pos; 2573 } 2574 return tb_size; 2575 } 2576 2577 static uint32_t 2578 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2579 { 2580 uint8_t i; 2581 uint32_t c, r, tb_size = 0; 2582 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2583 2584 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2585 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 2586 } else { 2587 c = op->ldpc_dec.tb_params.c; 2588 r = op->ldpc_dec.tb_params.r; 2589 for (i = 0; i < c-r; i++) 2590 tb_size += sys_cols * op->ldpc_dec.z_c 2591 - op->ldpc_dec.n_filler; 2592 } 2593 return tb_size; 2594 } 2595 2596 static uint32_t 2597 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2598 { 2599 uint8_t i; 2600 uint32_t c, r, tb_size = 0; 2601 2602 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2603 tb_size = op->turbo_enc.tb_params.k_neg; 2604 } else { 2605 c = op->turbo_enc.tb_params.c; 2606 r = op->turbo_enc.tb_params.r; 2607 for (i = 0; i < c-r; i++) 2608 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2609 op->turbo_enc.tb_params.k_neg : 2610 op->turbo_enc.tb_params.k_pos; 2611 } 2612 return tb_size; 2613 } 2614 2615 static uint32_t 2616 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2617 { 2618 uint8_t i; 2619 uint32_t c, r, tb_size = 0; 2620 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2621 2622 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2623 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 2624 } else { 2625 c = op->turbo_enc.tb_params.c; 2626 r = op->turbo_enc.tb_params.r; 2627 for (i = 0; i < c-r; i++) 2628 tb_size += sys_cols * op->ldpc_enc.z_c 2629 - op->ldpc_enc.n_filler; 2630 } 2631 return tb_size; 2632 } 2633 2634 static uint32_t 2635 calc_fft_size(struct rte_bbdev_fft_op *op) 2636 { 2637 uint32_t output_size; 2638 int num_cs = 0, i; 2639 for (i = 0; i < 12; i++) 2640 if (check_bit(op->fft.cs_bitmap, 1 << i)) 2641 num_cs++; 2642 output_size = (num_cs * op->fft.output_sequence_size * 4) << op->fft.num_antennas_log2; 2643 return output_size; 2644 } 2645 2646 static int 2647 init_test_op_params(struct test_op_params *op_params, 2648 enum rte_bbdev_op_type op_type, const int expected_status, 2649 const int vector_mask, struct rte_mempool *ops_mp, 2650 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2651 { 2652 int ret = 0; 2653 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2654 op_type == RTE_BBDEV_OP_LDPC_DEC) 2655 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2656 &op_params->ref_dec_op, 1); 2657 else if (op_type == RTE_BBDEV_OP_FFT) 2658 ret = rte_bbdev_fft_op_alloc_bulk(ops_mp, 2659 &op_params->ref_fft_op, 1); 2660 else 2661 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2662 &op_params->ref_enc_op, 1); 2663 2664 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2665 2666 op_params->mp = ops_mp; 2667 op_params->burst_sz = burst_sz; 2668 op_params->num_to_process = num_to_process; 2669 op_params->num_lcores = num_lcores; 2670 op_params->vector_mask = vector_mask; 2671 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2672 op_type == RTE_BBDEV_OP_LDPC_DEC) 2673 op_params->ref_dec_op->status = expected_status; 2674 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2675 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2676 op_params->ref_enc_op->status = expected_status; 2677 else if (op_type == RTE_BBDEV_OP_FFT) 2678 op_params->ref_fft_op->status = expected_status; 2679 return 0; 2680 } 2681 2682 static int 2683 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2684 struct test_op_params *op_params) 2685 { 2686 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2687 unsigned int i; 2688 struct active_device *ad; 2689 unsigned int burst_sz = get_burst_sz(); 2690 enum rte_bbdev_op_type op_type = test_vector.op_type; 2691 const struct rte_bbdev_op_cap *capabilities = NULL; 2692 2693 ad = &active_devs[dev_id]; 2694 2695 /* Check if device supports op_type */ 2696 if (!is_avail_op(ad, test_vector.op_type)) 2697 return TEST_SUCCESS; 2698 2699 struct rte_bbdev_info info; 2700 rte_bbdev_info_get(ad->dev_id, &info); 2701 socket_id = GET_SOCKET(info.socket_id); 2702 2703 f_ret = create_mempools(ad, socket_id, op_type, 2704 get_num_ops()); 2705 if (f_ret != TEST_SUCCESS) { 2706 printf("Couldn't create mempools"); 2707 goto fail; 2708 } 2709 if (op_type == RTE_BBDEV_OP_NONE) 2710 op_type = RTE_BBDEV_OP_TURBO_ENC; 2711 2712 f_ret = init_test_op_params(op_params, test_vector.op_type, 2713 test_vector.expected_status, 2714 test_vector.mask, 2715 ad->ops_mempool, 2716 burst_sz, 2717 get_num_ops(), 2718 get_num_lcores()); 2719 if (f_ret != TEST_SUCCESS) { 2720 printf("Couldn't init test op params"); 2721 goto fail; 2722 } 2723 2724 2725 /* Find capabilities */ 2726 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2727 do { 2728 if (cap->type == test_vector.op_type) { 2729 capabilities = cap; 2730 break; 2731 } 2732 cap++; 2733 } while (cap->type != RTE_BBDEV_OP_NONE); 2734 TEST_ASSERT_NOT_NULL(capabilities, 2735 "Couldn't find capabilities"); 2736 2737 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2738 create_reference_dec_op(op_params->ref_dec_op); 2739 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2740 create_reference_enc_op(op_params->ref_enc_op); 2741 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2742 create_reference_ldpc_enc_op(op_params->ref_enc_op); 2743 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2744 create_reference_ldpc_dec_op(op_params->ref_dec_op); 2745 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 2746 create_reference_fft_op(op_params->ref_fft_op); 2747 2748 for (i = 0; i < ad->nb_queues; ++i) { 2749 f_ret = fill_queue_buffers(op_params, 2750 ad->in_mbuf_pool, 2751 ad->hard_out_mbuf_pool, 2752 ad->soft_out_mbuf_pool, 2753 ad->harq_in_mbuf_pool, 2754 ad->harq_out_mbuf_pool, 2755 ad->queue_ids[i], 2756 capabilities, 2757 info.drv.min_alignment, 2758 socket_id); 2759 if (f_ret != TEST_SUCCESS) { 2760 printf("Couldn't init queue buffers"); 2761 goto fail; 2762 } 2763 } 2764 2765 /* Run test case function */ 2766 t_ret = test_case_func(ad, op_params); 2767 2768 /* Free active device resources and return */ 2769 free_buffers(ad, op_params); 2770 return t_ret; 2771 2772 fail: 2773 free_buffers(ad, op_params); 2774 return TEST_FAILED; 2775 } 2776 2777 /* Run given test function per active device per supported op type 2778 * per burst size. 2779 */ 2780 static int 2781 run_test_case(test_case_function *test_case_func) 2782 { 2783 int ret = 0; 2784 uint8_t dev; 2785 2786 /* Alloc op_params */ 2787 struct test_op_params *op_params = rte_zmalloc(NULL, 2788 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 2789 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 2790 RTE_ALIGN(sizeof(struct test_op_params), 2791 RTE_CACHE_LINE_SIZE)); 2792 2793 /* For each device run test case function */ 2794 for (dev = 0; dev < nb_active_devs; ++dev) 2795 ret |= run_test_case_on_device(test_case_func, dev, op_params); 2796 2797 rte_free(op_params); 2798 2799 return ret; 2800 } 2801 2802 2803 /* Push back the HARQ output from DDR to host */ 2804 static void 2805 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2806 struct rte_bbdev_dec_op **ops, 2807 const uint16_t n) 2808 { 2809 uint16_t j; 2810 int save_status, ret; 2811 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2812 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 2813 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2814 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 2815 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2816 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2817 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2818 for (j = 0; j < n; ++j) { 2819 if ((loopback && mem_out) || hc_out) { 2820 save_status = ops[j]->status; 2821 ops[j]->ldpc_dec.op_flags = 2822 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2823 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2824 if (h_comp) 2825 ops[j]->ldpc_dec.op_flags += 2826 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2827 ops[j]->ldpc_dec.harq_combined_input.offset = 2828 harq_offset; 2829 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 2830 harq_offset += HARQ_INCR; 2831 if (!loopback) 2832 ops[j]->ldpc_dec.harq_combined_input.length = 2833 ops[j]->ldpc_dec.harq_combined_output.length; 2834 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2835 &ops[j], 1); 2836 ret = 0; 2837 while (ret == 0) 2838 ret = rte_bbdev_dequeue_ldpc_dec_ops( 2839 dev_id, queue_id, 2840 &ops_deq[j], 1); 2841 ops[j]->ldpc_dec.op_flags = flags; 2842 ops[j]->status = save_status; 2843 } 2844 } 2845 } 2846 2847 /* 2848 * Push back the HARQ output from HW DDR to Host 2849 * Preload HARQ memory input and adjust HARQ offset 2850 */ 2851 static void 2852 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2853 struct rte_bbdev_dec_op **ops, const uint16_t n, 2854 bool preload) 2855 { 2856 uint16_t j; 2857 int deq; 2858 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2859 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 2860 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 2861 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2862 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2863 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 2864 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2865 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2866 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2867 if ((mem_in || hc_in) && preload) { 2868 for (j = 0; j < n; ++j) { 2869 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 2870 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 2871 ops[j]->ldpc_dec.op_flags = 2872 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2873 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2874 if (h_comp) 2875 ops[j]->ldpc_dec.op_flags += 2876 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2877 ops[j]->ldpc_dec.harq_combined_output.offset = 2878 harq_offset; 2879 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 2880 harq_offset += HARQ_INCR; 2881 } 2882 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 2883 deq = 0; 2884 while (deq != n) 2885 deq += rte_bbdev_dequeue_ldpc_dec_ops( 2886 dev_id, queue_id, &ops_deq[deq], 2887 n - deq); 2888 /* Restore the operations */ 2889 for (j = 0; j < n; ++j) { 2890 ops[j]->ldpc_dec.op_flags = flags; 2891 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 2892 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 2893 } 2894 } 2895 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2896 for (j = 0; j < n; ++j) { 2897 /* Adjust HARQ offset when we reach external DDR */ 2898 if (mem_in || hc_in) 2899 ops[j]->ldpc_dec.harq_combined_input.offset 2900 = harq_offset; 2901 if (mem_out || hc_out) 2902 ops[j]->ldpc_dec.harq_combined_output.offset 2903 = harq_offset; 2904 harq_offset += HARQ_INCR; 2905 } 2906 } 2907 2908 static void 2909 dequeue_event_callback(uint16_t dev_id, 2910 enum rte_bbdev_event_type event, void *cb_arg, 2911 void *ret_param) 2912 { 2913 int ret; 2914 uint16_t i; 2915 uint64_t total_time; 2916 uint16_t deq, burst_sz, num_ops; 2917 uint16_t queue_id = *(uint16_t *) ret_param; 2918 struct rte_bbdev_info info; 2919 double tb_len_bits; 2920 struct thread_params *tp = cb_arg; 2921 2922 /* Find matching thread params using queue_id */ 2923 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 2924 if (tp->queue_id == queue_id) 2925 break; 2926 2927 if (i == MAX_QUEUES) { 2928 printf("%s: Queue_id from interrupt details was not found!\n", 2929 __func__); 2930 return; 2931 } 2932 2933 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 2934 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2935 printf( 2936 "Dequeue interrupt handler called for incorrect event!\n"); 2937 return; 2938 } 2939 2940 burst_sz = __atomic_load_n(&tp->burst_sz, __ATOMIC_RELAXED); 2941 num_ops = tp->op_params->num_to_process; 2942 2943 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2944 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2945 &tp->dec_ops[ 2946 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2947 burst_sz); 2948 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2949 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2950 &tp->dec_ops[ 2951 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2952 burst_sz); 2953 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2954 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2955 &tp->enc_ops[ 2956 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2957 burst_sz); 2958 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 2959 deq = rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 2960 &tp->fft_ops[ 2961 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2962 burst_sz); 2963 else /*RTE_BBDEV_OP_TURBO_ENC*/ 2964 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2965 &tp->enc_ops[ 2966 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2967 burst_sz); 2968 2969 if (deq < burst_sz) { 2970 printf( 2971 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 2972 burst_sz, deq); 2973 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2974 return; 2975 } 2976 2977 if (__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) + deq < num_ops) { 2978 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 2979 return; 2980 } 2981 2982 total_time = rte_rdtsc_precise() - tp->start_time; 2983 2984 rte_bbdev_info_get(dev_id, &info); 2985 2986 ret = TEST_SUCCESS; 2987 2988 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2989 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2990 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 2991 tp->op_params->vector_mask); 2992 /* get the max of iter_count for all dequeued ops */ 2993 for (i = 0; i < num_ops; ++i) 2994 tp->iter_count = RTE_MAX( 2995 tp->dec_ops[i]->turbo_dec.iter_count, 2996 tp->iter_count); 2997 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2998 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 2999 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3000 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 3001 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3002 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 3003 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3004 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 3005 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 3006 } else if (test_vector.op_type == RTE_BBDEV_OP_FFT) { 3007 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 3008 ret = validate_fft_op(tp->fft_ops, num_ops, ref_op); 3009 rte_bbdev_fft_op_free_bulk(tp->fft_ops, deq); 3010 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 3011 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3012 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 3013 tp->op_params->vector_mask); 3014 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 3015 } 3016 3017 if (ret) { 3018 printf("Buffers validation failed\n"); 3019 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3020 } 3021 3022 switch (test_vector.op_type) { 3023 case RTE_BBDEV_OP_TURBO_DEC: 3024 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 3025 break; 3026 case RTE_BBDEV_OP_TURBO_ENC: 3027 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 3028 break; 3029 case RTE_BBDEV_OP_LDPC_DEC: 3030 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 3031 break; 3032 case RTE_BBDEV_OP_FFT: 3033 tb_len_bits = calc_fft_size(tp->op_params->ref_fft_op); 3034 break; 3035 case RTE_BBDEV_OP_LDPC_ENC: 3036 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 3037 break; 3038 case RTE_BBDEV_OP_NONE: 3039 tb_len_bits = 0.0; 3040 break; 3041 default: 3042 printf("Unknown op type: %d\n", test_vector.op_type); 3043 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 3044 return; 3045 } 3046 3047 tp->ops_per_sec += ((double)num_ops) / 3048 ((double)total_time / (double)rte_get_tsc_hz()); 3049 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 3050 ((double)total_time / (double)rte_get_tsc_hz()); 3051 3052 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 3053 } 3054 3055 static int 3056 throughput_intr_lcore_ldpc_dec(void *arg) 3057 { 3058 struct thread_params *tp = arg; 3059 unsigned int enqueued; 3060 const uint16_t queue_id = tp->queue_id; 3061 const uint16_t burst_sz = tp->op_params->burst_sz; 3062 const uint16_t num_to_process = tp->op_params->num_to_process; 3063 struct rte_bbdev_dec_op *ops[num_to_process]; 3064 struct test_buffers *bufs = NULL; 3065 struct rte_bbdev_info info; 3066 int ret, i, j; 3067 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3068 uint16_t num_to_enq, enq; 3069 3070 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3071 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3072 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3073 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3074 3075 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3076 "BURST_SIZE should be <= %u", MAX_BURST); 3077 3078 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3079 "Failed to enable interrupts for dev: %u, queue_id: %u", 3080 tp->dev_id, queue_id); 3081 3082 rte_bbdev_info_get(tp->dev_id, &info); 3083 3084 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3085 "NUM_OPS cannot exceed %u for this device", 3086 info.drv.queue_size_lim); 3087 3088 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3089 3090 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3091 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3092 3093 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3094 3095 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3096 num_to_process); 3097 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3098 num_to_process); 3099 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3100 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 3101 bufs->hard_outputs, bufs->soft_outputs, 3102 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3103 3104 /* Set counter to validate the ordering */ 3105 for (j = 0; j < num_to_process; ++j) 3106 ops[j]->opaque_data = (void *)(uintptr_t)j; 3107 3108 for (j = 0; j < TEST_REPETITIONS; ++j) { 3109 for (i = 0; i < num_to_process; ++i) { 3110 if (!loopback) 3111 rte_pktmbuf_reset( 3112 ops[i]->ldpc_dec.hard_output.data); 3113 if (hc_out || loopback) 3114 mbuf_reset( 3115 ops[i]->ldpc_dec.harq_combined_output.data); 3116 } 3117 3118 tp->start_time = rte_rdtsc_precise(); 3119 for (enqueued = 0; enqueued < num_to_process;) { 3120 num_to_enq = burst_sz; 3121 3122 if (unlikely(num_to_process - enqueued < num_to_enq)) 3123 num_to_enq = num_to_process - enqueued; 3124 3125 enq = 0; 3126 do { 3127 enq += rte_bbdev_enqueue_ldpc_dec_ops( 3128 tp->dev_id, 3129 queue_id, &ops[enqueued], 3130 num_to_enq); 3131 } while (unlikely(num_to_enq != enq)); 3132 enqueued += enq; 3133 3134 /* Write to thread burst_sz current number of enqueued 3135 * descriptors. It ensures that proper number of 3136 * descriptors will be dequeued in callback 3137 * function - needed for last batch in case where 3138 * the number of operations is not a multiple of 3139 * burst size. 3140 */ 3141 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3142 3143 /* Wait until processing of previous batch is 3144 * completed 3145 */ 3146 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3147 } 3148 if (j != TEST_REPETITIONS - 1) 3149 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3150 } 3151 3152 return TEST_SUCCESS; 3153 } 3154 3155 static int 3156 throughput_intr_lcore_dec(void *arg) 3157 { 3158 struct thread_params *tp = arg; 3159 unsigned int enqueued; 3160 const uint16_t queue_id = tp->queue_id; 3161 const uint16_t burst_sz = tp->op_params->burst_sz; 3162 const uint16_t num_to_process = tp->op_params->num_to_process; 3163 struct rte_bbdev_dec_op *ops[num_to_process]; 3164 struct test_buffers *bufs = NULL; 3165 struct rte_bbdev_info info; 3166 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3167 int ret, i, j; 3168 uint16_t num_to_enq, enq; 3169 3170 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3171 "BURST_SIZE should be <= %u", MAX_BURST); 3172 3173 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3174 "Failed to enable interrupts for dev: %u, queue_id: %u", 3175 tp->dev_id, queue_id); 3176 3177 rte_bbdev_info_get(tp->dev_id, &info); 3178 3179 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3180 "NUM_OPS cannot exceed %u for this device", 3181 info.drv.queue_size_lim); 3182 3183 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3184 3185 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3186 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3187 3188 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3189 3190 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 3191 num_to_process); 3192 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_to_process); 3193 ref_op->turbo_dec.iter_max = get_iter_max(); 3194 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3195 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 3196 bufs->hard_outputs, bufs->soft_outputs, 3197 tp->op_params->ref_dec_op); 3198 3199 /* Set counter to validate the ordering. */ 3200 for (j = 0; j < num_to_process; ++j) 3201 ops[j]->opaque_data = (void *)(uintptr_t)j; 3202 3203 for (j = 0; j < TEST_REPETITIONS; ++j) { 3204 for (i = 0; i < num_to_process; ++i) { 3205 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 3206 if (ops[i]->turbo_dec.soft_output.data != NULL) 3207 rte_pktmbuf_reset(ops[i]->turbo_dec.soft_output.data); 3208 } 3209 3210 3211 tp->start_time = rte_rdtsc_precise(); 3212 for (enqueued = 0; enqueued < num_to_process;) { 3213 num_to_enq = burst_sz; 3214 3215 if (unlikely(num_to_process - enqueued < num_to_enq)) 3216 num_to_enq = num_to_process - enqueued; 3217 3218 enq = 0; 3219 do { 3220 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3221 queue_id, &ops[enqueued], 3222 num_to_enq); 3223 } while (unlikely(num_to_enq != enq)); 3224 enqueued += enq; 3225 3226 /* Write to thread burst_sz current number of enqueued 3227 * descriptors. It ensures that proper number of 3228 * descriptors will be dequeued in callback 3229 * function - needed for last batch in case where 3230 * the number of operations is not a multiple of 3231 * burst size. 3232 */ 3233 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3234 3235 /* Wait until processing of previous batch is 3236 * completed 3237 */ 3238 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3239 } 3240 if (j != TEST_REPETITIONS - 1) 3241 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3242 } 3243 3244 return TEST_SUCCESS; 3245 } 3246 3247 static int 3248 throughput_intr_lcore_enc(void *arg) 3249 { 3250 struct thread_params *tp = arg; 3251 unsigned int enqueued; 3252 const uint16_t queue_id = tp->queue_id; 3253 const uint16_t burst_sz = tp->op_params->burst_sz; 3254 const uint16_t num_to_process = tp->op_params->num_to_process; 3255 struct rte_bbdev_enc_op *ops[num_to_process]; 3256 struct test_buffers *bufs = NULL; 3257 struct rte_bbdev_info info; 3258 int ret, i, j; 3259 uint16_t num_to_enq, enq; 3260 3261 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3262 "BURST_SIZE should be <= %u", MAX_BURST); 3263 3264 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3265 "Failed to enable interrupts for dev: %u, queue_id: %u", 3266 tp->dev_id, queue_id); 3267 3268 rte_bbdev_info_get(tp->dev_id, &info); 3269 3270 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3271 "NUM_OPS cannot exceed %u for this device", 3272 info.drv.queue_size_lim); 3273 3274 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3275 3276 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3277 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3278 3279 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3280 3281 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3282 num_to_process); 3283 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3284 num_to_process); 3285 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3286 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 3287 bufs->hard_outputs, tp->op_params->ref_enc_op); 3288 3289 /* Set counter to validate the ordering */ 3290 for (j = 0; j < num_to_process; ++j) 3291 ops[j]->opaque_data = (void *)(uintptr_t)j; 3292 3293 for (j = 0; j < TEST_REPETITIONS; ++j) { 3294 for (i = 0; i < num_to_process; ++i) 3295 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 3296 3297 tp->start_time = rte_rdtsc_precise(); 3298 for (enqueued = 0; enqueued < num_to_process;) { 3299 num_to_enq = burst_sz; 3300 3301 if (unlikely(num_to_process - enqueued < num_to_enq)) 3302 num_to_enq = num_to_process - enqueued; 3303 3304 enq = 0; 3305 do { 3306 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3307 queue_id, &ops[enqueued], 3308 num_to_enq); 3309 } while (unlikely(enq != num_to_enq)); 3310 enqueued += enq; 3311 3312 /* Write to thread burst_sz current number of enqueued 3313 * descriptors. It ensures that proper number of 3314 * descriptors will be dequeued in callback 3315 * function - needed for last batch in case where 3316 * the number of operations is not a multiple of 3317 * burst size. 3318 */ 3319 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3320 3321 /* Wait until processing of previous batch is 3322 * completed 3323 */ 3324 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3325 } 3326 if (j != TEST_REPETITIONS - 1) 3327 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3328 } 3329 3330 return TEST_SUCCESS; 3331 } 3332 3333 3334 static int 3335 throughput_intr_lcore_ldpc_enc(void *arg) 3336 { 3337 struct thread_params *tp = arg; 3338 unsigned int enqueued; 3339 const uint16_t queue_id = tp->queue_id; 3340 const uint16_t burst_sz = tp->op_params->burst_sz; 3341 const uint16_t num_to_process = tp->op_params->num_to_process; 3342 struct rte_bbdev_enc_op *ops[num_to_process]; 3343 struct test_buffers *bufs = NULL; 3344 struct rte_bbdev_info info; 3345 int ret, i, j; 3346 uint16_t num_to_enq, enq; 3347 3348 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3349 "BURST_SIZE should be <= %u", MAX_BURST); 3350 3351 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3352 "Failed to enable interrupts for dev: %u, queue_id: %u", 3353 tp->dev_id, queue_id); 3354 3355 rte_bbdev_info_get(tp->dev_id, &info); 3356 3357 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3358 "NUM_OPS cannot exceed %u for this device", 3359 info.drv.queue_size_lim); 3360 3361 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3362 3363 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3364 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3365 3366 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3367 3368 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3369 num_to_process); 3370 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3371 num_to_process); 3372 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3373 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3374 bufs->inputs, bufs->hard_outputs, 3375 tp->op_params->ref_enc_op); 3376 3377 /* Set counter to validate the ordering */ 3378 for (j = 0; j < num_to_process; ++j) 3379 ops[j]->opaque_data = (void *)(uintptr_t)j; 3380 3381 for (j = 0; j < TEST_REPETITIONS; ++j) { 3382 for (i = 0; i < num_to_process; ++i) 3383 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 3384 3385 tp->start_time = rte_rdtsc_precise(); 3386 for (enqueued = 0; enqueued < num_to_process;) { 3387 num_to_enq = burst_sz; 3388 3389 if (unlikely(num_to_process - enqueued < num_to_enq)) 3390 num_to_enq = num_to_process - enqueued; 3391 3392 enq = 0; 3393 do { 3394 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3395 tp->dev_id, 3396 queue_id, &ops[enqueued], 3397 num_to_enq); 3398 } while (unlikely(enq != num_to_enq)); 3399 enqueued += enq; 3400 3401 /* Write to thread burst_sz current number of enqueued 3402 * descriptors. It ensures that proper number of 3403 * descriptors will be dequeued in callback 3404 * function - needed for last batch in case where 3405 * the number of operations is not a multiple of 3406 * burst size. 3407 */ 3408 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3409 3410 /* Wait until processing of previous batch is 3411 * completed 3412 */ 3413 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3414 } 3415 if (j != TEST_REPETITIONS - 1) 3416 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3417 } 3418 3419 return TEST_SUCCESS; 3420 } 3421 3422 3423 static int 3424 throughput_intr_lcore_fft(void *arg) 3425 { 3426 struct thread_params *tp = arg; 3427 unsigned int enqueued; 3428 const uint16_t queue_id = tp->queue_id; 3429 const uint16_t burst_sz = tp->op_params->burst_sz; 3430 const uint16_t num_to_process = tp->op_params->num_to_process; 3431 struct rte_bbdev_fft_op *ops[num_to_process]; 3432 struct test_buffers *bufs = NULL; 3433 struct rte_bbdev_info info; 3434 int ret, i, j; 3435 uint16_t num_to_enq, enq; 3436 3437 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3438 "BURST_SIZE should be <= %u", MAX_BURST); 3439 3440 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3441 "Failed to enable interrupts for dev: %u, queue_id: %u", 3442 tp->dev_id, queue_id); 3443 3444 rte_bbdev_info_get(tp->dev_id, &info); 3445 3446 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3447 "NUM_OPS cannot exceed %u for this device", 3448 info.drv.queue_size_lim); 3449 3450 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3451 3452 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3453 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3454 3455 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3456 3457 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops, 3458 num_to_process); 3459 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3460 num_to_process); 3461 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3462 copy_reference_fft_op(ops, num_to_process, 0, bufs->inputs, 3463 bufs->hard_outputs, bufs->soft_outputs, tp->op_params->ref_fft_op); 3464 3465 /* Set counter to validate the ordering */ 3466 for (j = 0; j < num_to_process; ++j) 3467 ops[j]->opaque_data = (void *)(uintptr_t)j; 3468 3469 for (j = 0; j < TEST_REPETITIONS; ++j) { 3470 for (i = 0; i < num_to_process; ++i) 3471 rte_pktmbuf_reset(ops[i]->fft.base_output.data); 3472 3473 tp->start_time = rte_rdtsc_precise(); 3474 for (enqueued = 0; enqueued < num_to_process;) { 3475 num_to_enq = burst_sz; 3476 3477 if (unlikely(num_to_process - enqueued < num_to_enq)) 3478 num_to_enq = num_to_process - enqueued; 3479 3480 enq = 0; 3481 do { 3482 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 3483 queue_id, &ops[enqueued], 3484 num_to_enq); 3485 } while (unlikely(enq != num_to_enq)); 3486 enqueued += enq; 3487 3488 /* Write to thread burst_sz current number of enqueued 3489 * descriptors. It ensures that proper number of 3490 * descriptors will be dequeued in callback 3491 * function - needed for last batch in case where 3492 * the number of operations is not a multiple of 3493 * burst size. 3494 */ 3495 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3496 3497 /* Wait until processing of previous batch is 3498 * completed 3499 */ 3500 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3501 } 3502 if (j != TEST_REPETITIONS - 1) 3503 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3504 } 3505 3506 return TEST_SUCCESS; 3507 } 3508 3509 static int 3510 throughput_pmd_lcore_dec(void *arg) 3511 { 3512 struct thread_params *tp = arg; 3513 uint16_t enq, deq; 3514 uint64_t total_time = 0, start_time; 3515 const uint16_t queue_id = tp->queue_id; 3516 const uint16_t burst_sz = tp->op_params->burst_sz; 3517 const uint16_t num_ops = tp->op_params->num_to_process; 3518 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3519 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3520 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3521 struct test_buffers *bufs = NULL; 3522 int i, j, ret; 3523 struct rte_bbdev_info info; 3524 uint16_t num_to_enq; 3525 bool so_enable; 3526 3527 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3528 "BURST_SIZE should be <= %u", MAX_BURST); 3529 3530 rte_bbdev_info_get(tp->dev_id, &info); 3531 3532 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3533 "NUM_OPS cannot exceed %u for this device", 3534 info.drv.queue_size_lim); 3535 3536 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3537 3538 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3539 3540 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3541 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3542 ref_op->turbo_dec.iter_max = get_iter_max(); 3543 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3544 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3545 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3546 3547 so_enable = check_bit(ops_enq[0]->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT); 3548 3549 /* Set counter to validate the ordering */ 3550 for (j = 0; j < num_ops; ++j) 3551 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3552 3553 for (i = 0; i < TEST_REPETITIONS; ++i) { 3554 3555 for (j = 0; j < num_ops; ++j) 3556 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3557 if (so_enable) 3558 for (j = 0; j < num_ops; ++j) 3559 mbuf_reset(ops_enq[j]->turbo_dec.soft_output.data); 3560 3561 start_time = rte_rdtsc_precise(); 3562 3563 for (enq = 0, deq = 0; enq < num_ops;) { 3564 num_to_enq = burst_sz; 3565 3566 if (unlikely(num_ops - enq < num_to_enq)) 3567 num_to_enq = num_ops - enq; 3568 3569 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3570 queue_id, &ops_enq[enq], num_to_enq); 3571 3572 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3573 queue_id, &ops_deq[deq], enq - deq); 3574 } 3575 3576 /* dequeue the remaining */ 3577 while (deq < enq) { 3578 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3579 queue_id, &ops_deq[deq], enq - deq); 3580 } 3581 3582 total_time += rte_rdtsc_precise() - start_time; 3583 } 3584 3585 tp->iter_count = 0; 3586 /* get the max of iter_count for all dequeued ops */ 3587 for (i = 0; i < num_ops; ++i) { 3588 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3589 tp->iter_count); 3590 } 3591 3592 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3593 ret = validate_dec_op(ops_deq, num_ops, ref_op, 3594 tp->op_params->vector_mask); 3595 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3596 } 3597 3598 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3599 3600 double tb_len_bits = calc_dec_TB_size(ref_op); 3601 3602 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3603 ((double)total_time / (double)rte_get_tsc_hz()); 3604 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3605 1000000.0) / ((double)total_time / 3606 (double)rte_get_tsc_hz()); 3607 3608 return TEST_SUCCESS; 3609 } 3610 3611 static int 3612 bler_pmd_lcore_ldpc_dec(void *arg) 3613 { 3614 struct thread_params *tp = arg; 3615 uint16_t enq, deq; 3616 uint64_t total_time = 0, start_time; 3617 const uint16_t queue_id = tp->queue_id; 3618 const uint16_t burst_sz = tp->op_params->burst_sz; 3619 const uint16_t num_ops = tp->op_params->num_to_process; 3620 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3621 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3622 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3623 struct test_buffers *bufs = NULL; 3624 int i, j, ret; 3625 float parity_bler = 0; 3626 struct rte_bbdev_info info; 3627 uint16_t num_to_enq; 3628 bool extDdr = check_bit(ldpc_cap_flags, 3629 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3630 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3631 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3632 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3633 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3634 3635 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3636 "BURST_SIZE should be <= %u", MAX_BURST); 3637 3638 rte_bbdev_info_get(tp->dev_id, &info); 3639 3640 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3641 "NUM_OPS cannot exceed %u for this device", 3642 info.drv.queue_size_lim); 3643 3644 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3645 3646 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3647 3648 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3649 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3650 3651 /* For BLER tests we need to enable early termination */ 3652 if (!check_bit(ref_op->ldpc_dec.op_flags, 3653 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3654 ref_op->ldpc_dec.op_flags += 3655 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3656 ref_op->ldpc_dec.iter_max = get_iter_max(); 3657 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3658 3659 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3660 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3661 bufs->hard_outputs, bufs->soft_outputs, 3662 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3663 generate_llr_input(num_ops, bufs->inputs, ref_op); 3664 3665 /* Set counter to validate the ordering */ 3666 for (j = 0; j < num_ops; ++j) 3667 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3668 3669 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 3670 for (j = 0; j < num_ops; ++j) { 3671 if (!loopback) 3672 mbuf_reset( 3673 ops_enq[j]->ldpc_dec.hard_output.data); 3674 if (hc_out || loopback) 3675 mbuf_reset(ops_enq[j]->ldpc_dec.harq_combined_output.data); 3676 } 3677 if (extDdr) 3678 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3679 num_ops, true); 3680 start_time = rte_rdtsc_precise(); 3681 3682 for (enq = 0, deq = 0; enq < num_ops;) { 3683 num_to_enq = burst_sz; 3684 3685 if (unlikely(num_ops - enq < num_to_enq)) 3686 num_to_enq = num_ops - enq; 3687 3688 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3689 queue_id, &ops_enq[enq], num_to_enq); 3690 3691 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3692 queue_id, &ops_deq[deq], enq - deq); 3693 } 3694 3695 /* dequeue the remaining */ 3696 while (deq < enq) { 3697 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3698 queue_id, &ops_deq[deq], enq - deq); 3699 } 3700 3701 total_time += rte_rdtsc_precise() - start_time; 3702 } 3703 3704 tp->iter_count = 0; 3705 tp->iter_average = 0; 3706 /* get the max of iter_count for all dequeued ops */ 3707 for (i = 0; i < num_ops; ++i) { 3708 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3709 tp->iter_count); 3710 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 3711 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 3712 parity_bler += 1.0; 3713 } 3714 3715 parity_bler /= num_ops; /* This one is based on SYND */ 3716 tp->iter_average /= num_ops; 3717 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 3718 3719 if (test_vector.op_type != RTE_BBDEV_OP_NONE 3720 && tp->bler == 0 3721 && parity_bler == 0 3722 && !hc_out) { 3723 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3724 tp->op_params->vector_mask); 3725 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3726 } 3727 3728 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3729 3730 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3731 tp->ops_per_sec = ((double)num_ops * 1) / 3732 ((double)total_time / (double)rte_get_tsc_hz()); 3733 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 3734 1000000.0) / ((double)total_time / 3735 (double)rte_get_tsc_hz()); 3736 3737 return TEST_SUCCESS; 3738 } 3739 3740 static int 3741 throughput_pmd_lcore_ldpc_dec(void *arg) 3742 { 3743 struct thread_params *tp = arg; 3744 uint16_t enq, deq; 3745 uint64_t total_time = 0, start_time; 3746 const uint16_t queue_id = tp->queue_id; 3747 const uint16_t burst_sz = tp->op_params->burst_sz; 3748 const uint16_t num_ops = tp->op_params->num_to_process; 3749 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3750 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3751 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3752 struct test_buffers *bufs = NULL; 3753 int i, j, ret; 3754 struct rte_bbdev_info info; 3755 uint16_t num_to_enq; 3756 bool extDdr = check_bit(ldpc_cap_flags, 3757 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3758 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3759 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3760 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3761 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3762 3763 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3764 "BURST_SIZE should be <= %u", MAX_BURST); 3765 3766 rte_bbdev_info_get(tp->dev_id, &info); 3767 3768 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3769 "NUM_OPS cannot exceed %u for this device", 3770 info.drv.queue_size_lim); 3771 3772 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3773 3774 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3775 3776 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3777 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3778 3779 /* For throughput tests we need to disable early termination */ 3780 if (check_bit(ref_op->ldpc_dec.op_flags, 3781 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3782 ref_op->ldpc_dec.op_flags -= 3783 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3784 ref_op->ldpc_dec.iter_max = get_iter_max(); 3785 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3786 3787 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3788 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3789 bufs->hard_outputs, bufs->soft_outputs, 3790 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3791 3792 /* Set counter to validate the ordering */ 3793 for (j = 0; j < num_ops; ++j) 3794 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3795 3796 for (i = 0; i < TEST_REPETITIONS; ++i) { 3797 for (j = 0; j < num_ops; ++j) { 3798 if (!loopback) 3799 mbuf_reset( 3800 ops_enq[j]->ldpc_dec.hard_output.data); 3801 if (hc_out || loopback) 3802 mbuf_reset( 3803 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3804 } 3805 if (extDdr) 3806 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3807 num_ops, true); 3808 start_time = rte_rdtsc_precise(); 3809 3810 for (enq = 0, deq = 0; enq < num_ops;) { 3811 num_to_enq = burst_sz; 3812 3813 if (unlikely(num_ops - enq < num_to_enq)) 3814 num_to_enq = num_ops - enq; 3815 3816 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3817 queue_id, &ops_enq[enq], num_to_enq); 3818 3819 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3820 queue_id, &ops_deq[deq], enq - deq); 3821 } 3822 3823 /* dequeue the remaining */ 3824 while (deq < enq) { 3825 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3826 queue_id, &ops_deq[deq], enq - deq); 3827 } 3828 3829 total_time += rte_rdtsc_precise() - start_time; 3830 } 3831 3832 tp->iter_count = 0; 3833 /* get the max of iter_count for all dequeued ops */ 3834 for (i = 0; i < num_ops; ++i) { 3835 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3836 tp->iter_count); 3837 } 3838 if (extDdr) { 3839 /* Read loopback is not thread safe */ 3840 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 3841 } 3842 3843 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3844 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3845 tp->op_params->vector_mask); 3846 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3847 } 3848 3849 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3850 3851 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3852 3853 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3854 ((double)total_time / (double)rte_get_tsc_hz()); 3855 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3856 1000000.0) / ((double)total_time / 3857 (double)rte_get_tsc_hz()); 3858 3859 return TEST_SUCCESS; 3860 } 3861 3862 static int 3863 throughput_pmd_lcore_enc(void *arg) 3864 { 3865 struct thread_params *tp = arg; 3866 uint16_t enq, deq; 3867 uint64_t total_time = 0, start_time; 3868 const uint16_t queue_id = tp->queue_id; 3869 const uint16_t burst_sz = tp->op_params->burst_sz; 3870 const uint16_t num_ops = tp->op_params->num_to_process; 3871 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3872 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3873 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3874 struct test_buffers *bufs = NULL; 3875 int i, j, ret; 3876 struct rte_bbdev_info info; 3877 uint16_t num_to_enq; 3878 3879 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3880 "BURST_SIZE should be <= %u", MAX_BURST); 3881 3882 rte_bbdev_info_get(tp->dev_id, &info); 3883 3884 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3885 "NUM_OPS cannot exceed %u for this device", 3886 info.drv.queue_size_lim); 3887 3888 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3889 3890 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3891 3892 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3893 num_ops); 3894 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3895 num_ops); 3896 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3897 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3898 bufs->hard_outputs, ref_op); 3899 3900 /* Set counter to validate the ordering */ 3901 for (j = 0; j < num_ops; ++j) 3902 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3903 3904 for (i = 0; i < TEST_REPETITIONS; ++i) { 3905 3906 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3907 for (j = 0; j < num_ops; ++j) 3908 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3909 3910 start_time = rte_rdtsc_precise(); 3911 3912 for (enq = 0, deq = 0; enq < num_ops;) { 3913 num_to_enq = burst_sz; 3914 3915 if (unlikely(num_ops - enq < num_to_enq)) 3916 num_to_enq = num_ops - enq; 3917 3918 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3919 queue_id, &ops_enq[enq], num_to_enq); 3920 3921 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3922 queue_id, &ops_deq[deq], enq - deq); 3923 } 3924 3925 /* dequeue the remaining */ 3926 while (deq < enq) { 3927 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3928 queue_id, &ops_deq[deq], enq - deq); 3929 } 3930 3931 total_time += rte_rdtsc_precise() - start_time; 3932 } 3933 3934 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3935 ret = validate_enc_op(ops_deq, num_ops, ref_op); 3936 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3937 } 3938 3939 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3940 3941 double tb_len_bits = calc_enc_TB_size(ref_op); 3942 3943 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3944 ((double)total_time / (double)rte_get_tsc_hz()); 3945 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3946 / 1000000.0) / ((double)total_time / 3947 (double)rte_get_tsc_hz()); 3948 3949 return TEST_SUCCESS; 3950 } 3951 3952 static int 3953 throughput_pmd_lcore_ldpc_enc(void *arg) 3954 { 3955 struct thread_params *tp = arg; 3956 uint16_t enq, deq; 3957 uint64_t total_time = 0, start_time; 3958 const uint16_t queue_id = tp->queue_id; 3959 const uint16_t burst_sz = tp->op_params->burst_sz; 3960 const uint16_t num_ops = tp->op_params->num_to_process; 3961 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3962 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3963 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3964 struct test_buffers *bufs = NULL; 3965 int i, j, ret; 3966 struct rte_bbdev_info info; 3967 uint16_t num_to_enq; 3968 3969 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3970 "BURST_SIZE should be <= %u", MAX_BURST); 3971 3972 rte_bbdev_info_get(tp->dev_id, &info); 3973 3974 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3975 "NUM_OPS cannot exceed %u for this device", 3976 info.drv.queue_size_lim); 3977 3978 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3979 3980 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3981 3982 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3983 num_ops); 3984 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3985 num_ops); 3986 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3987 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3988 bufs->hard_outputs, ref_op); 3989 3990 /* Set counter to validate the ordering */ 3991 for (j = 0; j < num_ops; ++j) 3992 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3993 3994 for (i = 0; i < TEST_REPETITIONS; ++i) { 3995 3996 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3997 for (j = 0; j < num_ops; ++j) 3998 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3999 4000 start_time = rte_rdtsc_precise(); 4001 4002 for (enq = 0, deq = 0; enq < num_ops;) { 4003 num_to_enq = burst_sz; 4004 4005 if (unlikely(num_ops - enq < num_to_enq)) 4006 num_to_enq = num_ops - enq; 4007 4008 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 4009 queue_id, &ops_enq[enq], num_to_enq); 4010 4011 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4012 queue_id, &ops_deq[deq], enq - deq); 4013 } 4014 4015 /* dequeue the remaining */ 4016 while (deq < enq) { 4017 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 4018 queue_id, &ops_deq[deq], enq - deq); 4019 } 4020 4021 total_time += rte_rdtsc_precise() - start_time; 4022 } 4023 4024 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4025 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 4026 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4027 } 4028 4029 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 4030 4031 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 4032 4033 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4034 ((double)total_time / (double)rte_get_tsc_hz()); 4035 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 4036 / 1000000.0) / ((double)total_time / 4037 (double)rte_get_tsc_hz()); 4038 4039 return TEST_SUCCESS; 4040 } 4041 4042 static int 4043 throughput_pmd_lcore_fft(void *arg) 4044 { 4045 struct thread_params *tp = arg; 4046 uint16_t enq, deq; 4047 uint64_t total_time = 0, start_time; 4048 const uint16_t queue_id = tp->queue_id; 4049 const uint16_t burst_sz = tp->op_params->burst_sz; 4050 const uint16_t num_ops = tp->op_params->num_to_process; 4051 struct rte_bbdev_fft_op *ops_enq[num_ops]; 4052 struct rte_bbdev_fft_op *ops_deq[num_ops]; 4053 struct rte_bbdev_fft_op *ref_op = tp->op_params->ref_fft_op; 4054 struct test_buffers *bufs = NULL; 4055 int i, j, ret; 4056 struct rte_bbdev_info info; 4057 uint16_t num_to_enq; 4058 4059 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4060 "BURST_SIZE should be <= %u", MAX_BURST); 4061 4062 rte_bbdev_info_get(tp->dev_id, &info); 4063 4064 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 4065 "NUM_OPS cannot exceed %u for this device", 4066 info.drv.queue_size_lim); 4067 4068 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4069 4070 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4071 4072 ret = rte_bbdev_fft_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 4073 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 4074 4075 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4076 copy_reference_fft_op(ops_enq, num_ops, 0, bufs->inputs, 4077 bufs->hard_outputs, bufs->soft_outputs, ref_op); 4078 4079 /* Set counter to validate the ordering */ 4080 for (j = 0; j < num_ops; ++j) 4081 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4082 4083 for (i = 0; i < TEST_REPETITIONS; ++i) { 4084 4085 for (j = 0; j < num_ops; ++j) 4086 mbuf_reset(ops_enq[j]->fft.base_output.data); 4087 4088 start_time = rte_rdtsc_precise(); 4089 4090 for (enq = 0, deq = 0; enq < num_ops;) { 4091 num_to_enq = burst_sz; 4092 4093 if (unlikely(num_ops - enq < num_to_enq)) 4094 num_to_enq = num_ops - enq; 4095 4096 enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, 4097 queue_id, &ops_enq[enq], num_to_enq); 4098 4099 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4100 queue_id, &ops_deq[deq], enq - deq); 4101 } 4102 4103 /* dequeue the remaining */ 4104 while (deq < enq) { 4105 deq += rte_bbdev_dequeue_fft_ops(tp->dev_id, 4106 queue_id, &ops_deq[deq], enq - deq); 4107 } 4108 4109 total_time += rte_rdtsc_precise() - start_time; 4110 } 4111 4112 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4113 ret = validate_fft_op(ops_deq, num_ops, ref_op); 4114 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4115 } 4116 4117 rte_bbdev_fft_op_free_bulk(ops_enq, num_ops); 4118 4119 double tb_len_bits = calc_fft_size(ref_op); 4120 4121 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 4122 ((double)total_time / (double)rte_get_tsc_hz()); 4123 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 4124 1000000.0) / ((double)total_time / 4125 (double)rte_get_tsc_hz()); 4126 4127 return TEST_SUCCESS; 4128 } 4129 4130 static void 4131 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 4132 { 4133 unsigned int iter = 0; 4134 double total_mops = 0, total_mbps = 0; 4135 4136 for (iter = 0; iter < used_cores; iter++) { 4137 printf( 4138 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 4139 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 4140 t_params[iter].mbps); 4141 total_mops += t_params[iter].ops_per_sec; 4142 total_mbps += t_params[iter].mbps; 4143 } 4144 printf( 4145 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 4146 used_cores, total_mops, total_mbps); 4147 } 4148 4149 /* Aggregate the performance results over the number of cores used */ 4150 static void 4151 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 4152 { 4153 unsigned int core_idx = 0; 4154 double total_mops = 0, total_mbps = 0; 4155 uint8_t iter_count = 0; 4156 4157 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4158 printf( 4159 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 4160 t_params[core_idx].lcore_id, 4161 t_params[core_idx].ops_per_sec, 4162 t_params[core_idx].mbps, 4163 t_params[core_idx].iter_count); 4164 total_mops += t_params[core_idx].ops_per_sec; 4165 total_mbps += t_params[core_idx].mbps; 4166 iter_count = RTE_MAX(iter_count, 4167 t_params[core_idx].iter_count); 4168 } 4169 printf( 4170 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 4171 used_cores, total_mops, total_mbps, iter_count); 4172 } 4173 4174 /* Aggregate the performance results over the number of cores used */ 4175 static void 4176 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 4177 { 4178 unsigned int core_idx = 0; 4179 double total_mbps = 0, total_bler = 0, total_iter = 0; 4180 double snr = get_snr(); 4181 4182 for (core_idx = 0; core_idx < used_cores; core_idx++) { 4183 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 4184 t_params[core_idx].lcore_id, 4185 t_params[core_idx].bler * 100, 4186 t_params[core_idx].iter_average, 4187 t_params[core_idx].mbps, 4188 get_vector_filename()); 4189 total_mbps += t_params[core_idx].mbps; 4190 total_bler += t_params[core_idx].bler; 4191 total_iter += t_params[core_idx].iter_average; 4192 } 4193 total_bler /= used_cores; 4194 total_iter /= used_cores; 4195 4196 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n", 4197 snr, total_bler * 100, total_iter, get_iter_max(), 4198 total_mbps, get_vector_filename()); 4199 } 4200 4201 /* 4202 * Test function that determines BLER wireless performance 4203 */ 4204 static int 4205 bler_test(struct active_device *ad, 4206 struct test_op_params *op_params) 4207 { 4208 int ret; 4209 unsigned int lcore_id, used_cores = 0; 4210 struct thread_params *t_params; 4211 struct rte_bbdev_info info; 4212 lcore_function_t *bler_function; 4213 uint16_t num_lcores; 4214 const char *op_type_str; 4215 4216 rte_bbdev_info_get(ad->dev_id, &info); 4217 4218 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4219 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4220 test_vector.op_type); 4221 4222 printf("+ ------------------------------------------------------- +\n"); 4223 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4224 info.dev_name, ad->nb_queues, op_params->burst_sz, 4225 op_params->num_to_process, op_params->num_lcores, 4226 op_type_str, 4227 intr_enabled ? "Interrupt mode" : "PMD mode", 4228 (double)rte_get_tsc_hz() / 1000000000.0); 4229 4230 /* Set number of lcores */ 4231 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4232 ? ad->nb_queues 4233 : op_params->num_lcores; 4234 4235 /* Allocate memory for thread parameters structure */ 4236 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4237 RTE_CACHE_LINE_SIZE); 4238 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4239 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4240 RTE_CACHE_LINE_SIZE)); 4241 4242 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 4243 !check_bit(test_vector.ldpc_dec.op_flags, 4244 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 4245 && !check_bit(test_vector.ldpc_dec.op_flags, 4246 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 4247 bler_function = bler_pmd_lcore_ldpc_dec; 4248 else 4249 return TEST_SKIPPED; 4250 4251 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 4252 4253 /* Main core is set at first entry */ 4254 t_params[0].dev_id = ad->dev_id; 4255 t_params[0].lcore_id = rte_lcore_id(); 4256 t_params[0].op_params = op_params; 4257 t_params[0].queue_id = ad->queue_ids[used_cores++]; 4258 t_params[0].iter_count = 0; 4259 4260 RTE_LCORE_FOREACH_WORKER(lcore_id) { 4261 if (used_cores >= num_lcores) 4262 break; 4263 4264 t_params[used_cores].dev_id = ad->dev_id; 4265 t_params[used_cores].lcore_id = lcore_id; 4266 t_params[used_cores].op_params = op_params; 4267 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 4268 t_params[used_cores].iter_count = 0; 4269 4270 rte_eal_remote_launch(bler_function, 4271 &t_params[used_cores++], lcore_id); 4272 } 4273 4274 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4275 ret = bler_function(&t_params[0]); 4276 4277 /* Main core is always used */ 4278 for (used_cores = 1; used_cores < num_lcores; used_cores++) 4279 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 4280 4281 print_dec_bler(t_params, num_lcores); 4282 4283 /* Return if test failed */ 4284 if (ret) { 4285 rte_free(t_params); 4286 return ret; 4287 } 4288 4289 /* Function to print something here*/ 4290 rte_free(t_params); 4291 return ret; 4292 } 4293 4294 /* 4295 * Test function that determines how long an enqueue + dequeue of a burst 4296 * takes on available lcores. 4297 */ 4298 static int 4299 throughput_test(struct active_device *ad, 4300 struct test_op_params *op_params) 4301 { 4302 int ret; 4303 unsigned int lcore_id, used_cores = 0; 4304 struct thread_params *t_params, *tp; 4305 struct rte_bbdev_info info; 4306 lcore_function_t *throughput_function; 4307 uint16_t num_lcores; 4308 const char *op_type_str; 4309 4310 rte_bbdev_info_get(ad->dev_id, &info); 4311 4312 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 4313 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 4314 test_vector.op_type); 4315 4316 printf("+ ------------------------------------------------------- +\n"); 4317 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 4318 info.dev_name, ad->nb_queues, op_params->burst_sz, 4319 op_params->num_to_process, op_params->num_lcores, 4320 op_type_str, 4321 intr_enabled ? "Interrupt mode" : "PMD mode", 4322 (double)rte_get_tsc_hz() / 1000000000.0); 4323 4324 /* Set number of lcores */ 4325 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 4326 ? ad->nb_queues 4327 : op_params->num_lcores; 4328 4329 /* Allocate memory for thread parameters structure */ 4330 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 4331 RTE_CACHE_LINE_SIZE); 4332 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 4333 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 4334 RTE_CACHE_LINE_SIZE)); 4335 4336 if (intr_enabled) { 4337 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 4338 throughput_function = throughput_intr_lcore_dec; 4339 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 4340 throughput_function = throughput_intr_lcore_ldpc_dec; 4341 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 4342 throughput_function = throughput_intr_lcore_enc; 4343 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 4344 throughput_function = throughput_intr_lcore_ldpc_enc; 4345 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 4346 throughput_function = throughput_intr_lcore_fft; 4347 else 4348 throughput_function = throughput_intr_lcore_enc; 4349 4350 /* Dequeue interrupt callback registration */ 4351 ret = rte_bbdev_callback_register(ad->dev_id, 4352 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 4353 t_params); 4354 if (ret < 0) { 4355 rte_free(t_params); 4356 return ret; 4357 } 4358 } else { 4359 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 4360 throughput_function = throughput_pmd_lcore_dec; 4361 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 4362 throughput_function = throughput_pmd_lcore_ldpc_dec; 4363 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 4364 throughput_function = throughput_pmd_lcore_enc; 4365 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 4366 throughput_function = throughput_pmd_lcore_ldpc_enc; 4367 else if (test_vector.op_type == RTE_BBDEV_OP_FFT) 4368 throughput_function = throughput_pmd_lcore_fft; 4369 else 4370 throughput_function = throughput_pmd_lcore_enc; 4371 } 4372 4373 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 4374 4375 /* Main core is set at first entry */ 4376 t_params[0].dev_id = ad->dev_id; 4377 t_params[0].lcore_id = rte_lcore_id(); 4378 t_params[0].op_params = op_params; 4379 t_params[0].queue_id = ad->queue_ids[used_cores++]; 4380 t_params[0].iter_count = 0; 4381 4382 RTE_LCORE_FOREACH_WORKER(lcore_id) { 4383 if (used_cores >= num_lcores) 4384 break; 4385 4386 t_params[used_cores].dev_id = ad->dev_id; 4387 t_params[used_cores].lcore_id = lcore_id; 4388 t_params[used_cores].op_params = op_params; 4389 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 4390 t_params[used_cores].iter_count = 0; 4391 4392 rte_eal_remote_launch(throughput_function, 4393 &t_params[used_cores++], lcore_id); 4394 } 4395 4396 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 4397 ret = throughput_function(&t_params[0]); 4398 4399 /* Main core is always used */ 4400 for (used_cores = 1; used_cores < num_lcores; used_cores++) 4401 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 4402 4403 /* Return if test failed */ 4404 if (ret) { 4405 rte_free(t_params); 4406 return ret; 4407 } 4408 4409 /* Print throughput if interrupts are disabled and test passed */ 4410 if (!intr_enabled) { 4411 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 4412 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 4413 print_dec_throughput(t_params, num_lcores); 4414 else 4415 print_enc_throughput(t_params, num_lcores); 4416 rte_free(t_params); 4417 return ret; 4418 } 4419 4420 /* In interrupt TC we need to wait for the interrupt callback to deqeue 4421 * all pending operations. Skip waiting for queues which reported an 4422 * error using processing_status variable. 4423 * Wait for main lcore operations. 4424 */ 4425 tp = &t_params[0]; 4426 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 4427 op_params->num_to_process) && 4428 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 4429 TEST_FAILED)) 4430 rte_pause(); 4431 4432 tp->ops_per_sec /= TEST_REPETITIONS; 4433 tp->mbps /= TEST_REPETITIONS; 4434 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 4435 4436 /* Wait for worker lcores operations */ 4437 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 4438 tp = &t_params[used_cores]; 4439 4440 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 4441 op_params->num_to_process) && 4442 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 4443 TEST_FAILED)) 4444 rte_pause(); 4445 4446 tp->ops_per_sec /= TEST_REPETITIONS; 4447 tp->mbps /= TEST_REPETITIONS; 4448 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 4449 } 4450 4451 /* Print throughput if test passed */ 4452 if (!ret) { 4453 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 4454 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 4455 print_dec_throughput(t_params, num_lcores); 4456 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 4457 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 4458 print_enc_throughput(t_params, num_lcores); 4459 } 4460 4461 rte_free(t_params); 4462 return ret; 4463 } 4464 4465 static int 4466 latency_test_dec(struct rte_mempool *mempool, 4467 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 4468 int vector_mask, uint16_t dev_id, uint16_t queue_id, 4469 const uint16_t num_to_process, uint16_t burst_sz, 4470 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4471 { 4472 int ret = TEST_SUCCESS; 4473 uint16_t i, j, dequeued; 4474 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4475 uint64_t start_time = 0, last_time = 0; 4476 4477 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4478 uint16_t enq = 0, deq = 0; 4479 bool first_time = true; 4480 last_time = 0; 4481 4482 if (unlikely(num_to_process - dequeued < burst_sz)) 4483 burst_sz = num_to_process - dequeued; 4484 4485 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4486 TEST_ASSERT_SUCCESS(ret, 4487 "rte_bbdev_dec_op_alloc_bulk() failed"); 4488 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4489 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 4490 bufs->inputs, 4491 bufs->hard_outputs, 4492 bufs->soft_outputs, 4493 ref_op); 4494 4495 /* Set counter to validate the ordering */ 4496 for (j = 0; j < burst_sz; ++j) 4497 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4498 4499 start_time = rte_rdtsc_precise(); 4500 4501 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 4502 burst_sz); 4503 TEST_ASSERT(enq == burst_sz, 4504 "Error enqueueing burst, expected %u, got %u", 4505 burst_sz, enq); 4506 4507 /* Dequeue */ 4508 do { 4509 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4510 &ops_deq[deq], burst_sz - deq); 4511 if (likely(first_time && (deq > 0))) { 4512 last_time = rte_rdtsc_precise() - start_time; 4513 first_time = false; 4514 } 4515 } while (unlikely(burst_sz != deq)); 4516 4517 *max_time = RTE_MAX(*max_time, last_time); 4518 *min_time = RTE_MIN(*min_time, last_time); 4519 *total_time += last_time; 4520 4521 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4522 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 4523 vector_mask); 4524 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4525 } 4526 4527 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4528 dequeued += deq; 4529 } 4530 4531 return i; 4532 } 4533 4534 /* Test case for latency/validation for LDPC Decoder */ 4535 static int 4536 latency_test_ldpc_dec(struct rte_mempool *mempool, 4537 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 4538 int vector_mask, uint16_t dev_id, uint16_t queue_id, 4539 const uint16_t num_to_process, uint16_t burst_sz, 4540 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 4541 bool disable_et) 4542 { 4543 int ret = TEST_SUCCESS; 4544 uint16_t i, j, dequeued; 4545 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4546 uint64_t start_time = 0, last_time = 0; 4547 bool extDdr = ldpc_cap_flags & 4548 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4549 4550 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4551 uint16_t enq = 0, deq = 0; 4552 bool first_time = true; 4553 last_time = 0; 4554 4555 if (unlikely(num_to_process - dequeued < burst_sz)) 4556 burst_sz = num_to_process - dequeued; 4557 4558 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4559 TEST_ASSERT_SUCCESS(ret, 4560 "rte_bbdev_dec_op_alloc_bulk() failed"); 4561 4562 /* For latency tests we need to disable early termination */ 4563 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 4564 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4565 ref_op->ldpc_dec.op_flags -= 4566 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4567 ref_op->ldpc_dec.iter_max = get_iter_max(); 4568 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4569 4570 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4571 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4572 bufs->inputs, 4573 bufs->hard_outputs, 4574 bufs->soft_outputs, 4575 bufs->harq_inputs, 4576 bufs->harq_outputs, 4577 ref_op); 4578 4579 if (extDdr) 4580 preload_harq_ddr(dev_id, queue_id, ops_enq, 4581 burst_sz, true); 4582 4583 /* Set counter to validate the ordering */ 4584 for (j = 0; j < burst_sz; ++j) 4585 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4586 4587 start_time = rte_rdtsc_precise(); 4588 4589 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4590 &ops_enq[enq], burst_sz); 4591 TEST_ASSERT(enq == burst_sz, 4592 "Error enqueueing burst, expected %u, got %u", 4593 burst_sz, enq); 4594 4595 /* Dequeue */ 4596 do { 4597 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4598 &ops_deq[deq], burst_sz - deq); 4599 if (likely(first_time && (deq > 0))) { 4600 last_time = rte_rdtsc_precise() - start_time; 4601 first_time = false; 4602 } 4603 } while (unlikely(burst_sz != deq)); 4604 4605 *max_time = RTE_MAX(*max_time, last_time); 4606 *min_time = RTE_MIN(*min_time, last_time); 4607 *total_time += last_time; 4608 4609 if (extDdr) 4610 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4611 4612 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4613 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 4614 vector_mask); 4615 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4616 } 4617 4618 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4619 dequeued += deq; 4620 } 4621 return i; 4622 } 4623 4624 static int 4625 latency_test_enc(struct rte_mempool *mempool, 4626 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4627 uint16_t dev_id, uint16_t queue_id, 4628 const uint16_t num_to_process, uint16_t burst_sz, 4629 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4630 { 4631 int ret = TEST_SUCCESS; 4632 uint16_t i, j, dequeued; 4633 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4634 uint64_t start_time = 0, last_time = 0; 4635 4636 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4637 uint16_t enq = 0, deq = 0; 4638 bool first_time = true; 4639 last_time = 0; 4640 4641 if (unlikely(num_to_process - dequeued < burst_sz)) 4642 burst_sz = num_to_process - dequeued; 4643 4644 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4645 TEST_ASSERT_SUCCESS(ret, 4646 "rte_bbdev_enc_op_alloc_bulk() failed"); 4647 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4648 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4649 bufs->inputs, 4650 bufs->hard_outputs, 4651 ref_op); 4652 4653 /* Set counter to validate the ordering */ 4654 for (j = 0; j < burst_sz; ++j) 4655 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4656 4657 start_time = rte_rdtsc_precise(); 4658 4659 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 4660 burst_sz); 4661 TEST_ASSERT(enq == burst_sz, 4662 "Error enqueueing burst, expected %u, got %u", 4663 burst_sz, enq); 4664 4665 /* Dequeue */ 4666 do { 4667 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4668 &ops_deq[deq], burst_sz - deq); 4669 if (likely(first_time && (deq > 0))) { 4670 last_time += rte_rdtsc_precise() - start_time; 4671 first_time = false; 4672 } 4673 } while (unlikely(burst_sz != deq)); 4674 4675 *max_time = RTE_MAX(*max_time, last_time); 4676 *min_time = RTE_MIN(*min_time, last_time); 4677 *total_time += last_time; 4678 4679 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4680 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4681 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4682 } 4683 4684 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4685 dequeued += deq; 4686 } 4687 4688 return i; 4689 } 4690 4691 static int 4692 latency_test_ldpc_enc(struct rte_mempool *mempool, 4693 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4694 uint16_t dev_id, uint16_t queue_id, 4695 const uint16_t num_to_process, uint16_t burst_sz, 4696 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4697 { 4698 int ret = TEST_SUCCESS; 4699 uint16_t i, j, dequeued; 4700 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4701 uint64_t start_time = 0, last_time = 0; 4702 4703 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4704 uint16_t enq = 0, deq = 0; 4705 bool first_time = true; 4706 last_time = 0; 4707 4708 if (unlikely(num_to_process - dequeued < burst_sz)) 4709 burst_sz = num_to_process - dequeued; 4710 4711 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4712 TEST_ASSERT_SUCCESS(ret, 4713 "rte_bbdev_enc_op_alloc_bulk() failed"); 4714 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4715 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4716 bufs->inputs, 4717 bufs->hard_outputs, 4718 ref_op); 4719 4720 /* Set counter to validate the ordering */ 4721 for (j = 0; j < burst_sz; ++j) 4722 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4723 4724 start_time = rte_rdtsc_precise(); 4725 4726 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4727 &ops_enq[enq], burst_sz); 4728 TEST_ASSERT(enq == burst_sz, 4729 "Error enqueueing burst, expected %u, got %u", 4730 burst_sz, enq); 4731 4732 /* Dequeue */ 4733 do { 4734 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4735 &ops_deq[deq], burst_sz - deq); 4736 if (likely(first_time && (deq > 0))) { 4737 last_time += rte_rdtsc_precise() - start_time; 4738 first_time = false; 4739 } 4740 } while (unlikely(burst_sz != deq)); 4741 4742 *max_time = RTE_MAX(*max_time, last_time); 4743 *min_time = RTE_MIN(*min_time, last_time); 4744 *total_time += last_time; 4745 4746 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4747 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4748 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4749 } 4750 4751 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4752 dequeued += deq; 4753 } 4754 4755 return i; 4756 } 4757 4758 4759 static int 4760 latency_test_fft(struct rte_mempool *mempool, 4761 struct test_buffers *bufs, struct rte_bbdev_fft_op *ref_op, 4762 uint16_t dev_id, uint16_t queue_id, 4763 const uint16_t num_to_process, uint16_t burst_sz, 4764 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4765 { 4766 int ret = TEST_SUCCESS; 4767 uint16_t i, j, dequeued; 4768 struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4769 uint64_t start_time = 0, last_time = 0; 4770 4771 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4772 uint16_t enq = 0, deq = 0; 4773 bool first_time = true; 4774 last_time = 0; 4775 4776 if (unlikely(num_to_process - dequeued < burst_sz)) 4777 burst_sz = num_to_process - dequeued; 4778 4779 ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); 4780 TEST_ASSERT_SUCCESS(ret, 4781 "rte_bbdev_fft_op_alloc_bulk() failed"); 4782 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4783 copy_reference_fft_op(ops_enq, burst_sz, dequeued, 4784 bufs->inputs, 4785 bufs->hard_outputs, bufs->soft_outputs, 4786 ref_op); 4787 4788 /* Set counter to validate the ordering */ 4789 for (j = 0; j < burst_sz; ++j) 4790 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4791 4792 start_time = rte_rdtsc_precise(); 4793 4794 enq = rte_bbdev_enqueue_fft_ops(dev_id, queue_id, 4795 &ops_enq[enq], burst_sz); 4796 TEST_ASSERT(enq == burst_sz, 4797 "Error enqueueing burst, expected %u, got %u", 4798 burst_sz, enq); 4799 4800 /* Dequeue */ 4801 do { 4802 deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, 4803 &ops_deq[deq], burst_sz - deq); 4804 if (likely(first_time && (deq > 0))) { 4805 last_time += rte_rdtsc_precise() - start_time; 4806 first_time = false; 4807 } 4808 } while (unlikely(burst_sz != deq)); 4809 4810 *max_time = RTE_MAX(*max_time, last_time); 4811 *min_time = RTE_MIN(*min_time, last_time); 4812 *total_time += last_time; 4813 4814 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4815 ret = validate_fft_op(ops_deq, burst_sz, ref_op); 4816 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4817 } 4818 4819 rte_bbdev_fft_op_free_bulk(ops_enq, deq); 4820 dequeued += deq; 4821 } 4822 4823 return i; 4824 } 4825 4826 /* Common function for running validation and latency test cases */ 4827 static int 4828 validation_latency_test(struct active_device *ad, 4829 struct test_op_params *op_params, bool latency_flag) 4830 { 4831 int iter; 4832 uint16_t burst_sz = op_params->burst_sz; 4833 const uint16_t num_to_process = op_params->num_to_process; 4834 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4835 const uint16_t queue_id = ad->queue_ids[0]; 4836 struct test_buffers *bufs = NULL; 4837 struct rte_bbdev_info info; 4838 uint64_t total_time, min_time, max_time; 4839 const char *op_type_str; 4840 4841 total_time = max_time = 0; 4842 min_time = UINT64_MAX; 4843 4844 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4845 "BURST_SIZE should be <= %u", MAX_BURST); 4846 4847 rte_bbdev_info_get(ad->dev_id, &info); 4848 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4849 4850 op_type_str = rte_bbdev_op_type_str(op_type); 4851 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4852 4853 printf("+ ------------------------------------------------------- +\n"); 4854 if (latency_flag) 4855 printf("== test: latency\ndev:"); 4856 else 4857 printf("== test: validation\ndev:"); 4858 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 4859 info.dev_name, burst_sz, num_to_process, op_type_str); 4860 4861 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4862 iter = latency_test_dec(op_params->mp, bufs, 4863 op_params->ref_dec_op, op_params->vector_mask, 4864 ad->dev_id, queue_id, num_to_process, 4865 burst_sz, &total_time, &min_time, &max_time); 4866 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4867 iter = latency_test_ldpc_enc(op_params->mp, bufs, 4868 op_params->ref_enc_op, ad->dev_id, queue_id, 4869 num_to_process, burst_sz, &total_time, 4870 &min_time, &max_time); 4871 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4872 iter = latency_test_ldpc_dec(op_params->mp, bufs, 4873 op_params->ref_dec_op, op_params->vector_mask, 4874 ad->dev_id, queue_id, num_to_process, 4875 burst_sz, &total_time, &min_time, &max_time, 4876 latency_flag); 4877 else if (op_type == RTE_BBDEV_OP_FFT) 4878 iter = latency_test_fft(op_params->mp, bufs, 4879 op_params->ref_fft_op, 4880 ad->dev_id, queue_id, 4881 num_to_process, burst_sz, &total_time, 4882 &min_time, &max_time); 4883 else /* RTE_BBDEV_OP_TURBO_ENC */ 4884 iter = latency_test_enc(op_params->mp, bufs, 4885 op_params->ref_enc_op, 4886 ad->dev_id, queue_id, 4887 num_to_process, burst_sz, &total_time, 4888 &min_time, &max_time); 4889 4890 if (iter <= 0) 4891 return TEST_FAILED; 4892 4893 printf("Operation latency:\n" 4894 "\tavg: %lg cycles, %lg us\n" 4895 "\tmin: %lg cycles, %lg us\n" 4896 "\tmax: %lg cycles, %lg us\n", 4897 (double)total_time / (double)iter, 4898 (double)(total_time * 1000000) / (double)iter / 4899 (double)rte_get_tsc_hz(), (double)min_time, 4900 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 4901 (double)max_time, (double)(max_time * 1000000) / 4902 (double)rte_get_tsc_hz()); 4903 4904 return TEST_SUCCESS; 4905 } 4906 4907 static int 4908 latency_test(struct active_device *ad, struct test_op_params *op_params) 4909 { 4910 return validation_latency_test(ad, op_params, true); 4911 } 4912 4913 static int 4914 validation_test(struct active_device *ad, struct test_op_params *op_params) 4915 { 4916 return validation_latency_test(ad, op_params, false); 4917 } 4918 4919 #ifdef RTE_BBDEV_OFFLOAD_COST 4920 static int 4921 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 4922 struct rte_bbdev_stats *stats) 4923 { 4924 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 4925 struct rte_bbdev_stats *q_stats; 4926 4927 if (queue_id >= dev->data->num_queues) 4928 return -1; 4929 4930 q_stats = &dev->data->queues[queue_id].queue_stats; 4931 4932 stats->enqueued_count = q_stats->enqueued_count; 4933 stats->dequeued_count = q_stats->dequeued_count; 4934 stats->enqueue_err_count = q_stats->enqueue_err_count; 4935 stats->dequeue_err_count = q_stats->dequeue_err_count; 4936 stats->enqueue_warning_count = q_stats->enqueue_warning_count; 4937 stats->dequeue_warning_count = q_stats->dequeue_warning_count; 4938 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 4939 4940 return 0; 4941 } 4942 4943 static int 4944 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 4945 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4946 uint16_t queue_id, const uint16_t num_to_process, 4947 uint16_t burst_sz, struct test_time_stats *time_st) 4948 { 4949 int i, dequeued, ret; 4950 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4951 uint64_t enq_start_time, deq_start_time; 4952 uint64_t enq_sw_last_time, deq_last_time; 4953 struct rte_bbdev_stats stats; 4954 4955 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4956 uint16_t enq = 0, deq = 0; 4957 4958 if (unlikely(num_to_process - dequeued < burst_sz)) 4959 burst_sz = num_to_process - dequeued; 4960 4961 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4962 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4963 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 4964 bufs->inputs, 4965 bufs->hard_outputs, 4966 bufs->soft_outputs, 4967 ref_op); 4968 4969 /* Start time meas for enqueue function offload latency */ 4970 enq_start_time = rte_rdtsc_precise(); 4971 do { 4972 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 4973 &ops_enq[enq], burst_sz - enq); 4974 } while (unlikely(burst_sz != enq)); 4975 4976 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4977 TEST_ASSERT_SUCCESS(ret, 4978 "Failed to get stats for queue (%u) of device (%u)", 4979 queue_id, dev_id); 4980 4981 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 4982 stats.acc_offload_cycles; 4983 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4984 enq_sw_last_time); 4985 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4986 enq_sw_last_time); 4987 time_st->enq_sw_total_time += enq_sw_last_time; 4988 4989 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4990 stats.acc_offload_cycles); 4991 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4992 stats.acc_offload_cycles); 4993 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4994 4995 /* give time for device to process ops */ 4996 rte_delay_us(WAIT_OFFLOAD_US); 4997 4998 /* Start time meas for dequeue function offload latency */ 4999 deq_start_time = rte_rdtsc_precise(); 5000 /* Dequeue one operation */ 5001 do { 5002 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5003 &ops_deq[deq], enq); 5004 } while (unlikely(deq == 0)); 5005 5006 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5007 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5008 deq_last_time); 5009 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5010 deq_last_time); 5011 time_st->deq_total_time += deq_last_time; 5012 5013 /* Dequeue remaining operations if needed*/ 5014 while (burst_sz != deq) 5015 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 5016 &ops_deq[deq], burst_sz - deq); 5017 5018 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5019 dequeued += deq; 5020 } 5021 5022 return i; 5023 } 5024 5025 static int 5026 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 5027 struct test_buffers *bufs, 5028 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 5029 uint16_t queue_id, const uint16_t num_to_process, 5030 uint16_t burst_sz, struct test_time_stats *time_st) 5031 { 5032 int i, dequeued, ret; 5033 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5034 uint64_t enq_start_time, deq_start_time; 5035 uint64_t enq_sw_last_time, deq_last_time; 5036 struct rte_bbdev_stats stats; 5037 bool extDdr = ldpc_cap_flags & 5038 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 5039 5040 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5041 uint16_t enq = 0, deq = 0; 5042 5043 if (unlikely(num_to_process - dequeued < burst_sz)) 5044 burst_sz = num_to_process - dequeued; 5045 5046 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 5047 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5048 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 5049 bufs->inputs, 5050 bufs->hard_outputs, 5051 bufs->soft_outputs, 5052 bufs->harq_inputs, 5053 bufs->harq_outputs, 5054 ref_op); 5055 5056 if (extDdr) 5057 preload_harq_ddr(dev_id, queue_id, ops_enq, 5058 burst_sz, true); 5059 5060 /* Start time meas for enqueue function offload latency */ 5061 enq_start_time = rte_rdtsc_precise(); 5062 do { 5063 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 5064 &ops_enq[enq], burst_sz - enq); 5065 } while (unlikely(burst_sz != enq)); 5066 5067 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 5068 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5069 TEST_ASSERT_SUCCESS(ret, 5070 "Failed to get stats for queue (%u) of device (%u)", 5071 queue_id, dev_id); 5072 5073 enq_sw_last_time -= stats.acc_offload_cycles; 5074 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5075 enq_sw_last_time); 5076 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5077 enq_sw_last_time); 5078 time_st->enq_sw_total_time += enq_sw_last_time; 5079 5080 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5081 stats.acc_offload_cycles); 5082 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5083 stats.acc_offload_cycles); 5084 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5085 5086 /* give time for device to process ops */ 5087 rte_delay_us(WAIT_OFFLOAD_US); 5088 5089 /* Start time meas for dequeue function offload latency */ 5090 deq_start_time = rte_rdtsc_precise(); 5091 /* Dequeue one operation */ 5092 do { 5093 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5094 &ops_deq[deq], enq); 5095 } while (unlikely(deq == 0)); 5096 5097 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5098 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5099 deq_last_time); 5100 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5101 deq_last_time); 5102 time_st->deq_total_time += deq_last_time; 5103 5104 /* Dequeue remaining operations if needed*/ 5105 while (burst_sz != deq) 5106 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 5107 &ops_deq[deq], burst_sz - deq); 5108 5109 if (extDdr) { 5110 /* Read loopback is not thread safe */ 5111 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 5112 } 5113 5114 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 5115 dequeued += deq; 5116 } 5117 5118 return i; 5119 } 5120 5121 static int 5122 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 5123 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 5124 uint16_t queue_id, const uint16_t num_to_process, 5125 uint16_t burst_sz, struct test_time_stats *time_st) 5126 { 5127 int i, dequeued, ret; 5128 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5129 uint64_t enq_start_time, deq_start_time; 5130 uint64_t enq_sw_last_time, deq_last_time; 5131 struct rte_bbdev_stats stats; 5132 5133 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5134 uint16_t enq = 0, deq = 0; 5135 5136 if (unlikely(num_to_process - dequeued < burst_sz)) 5137 burst_sz = num_to_process - dequeued; 5138 5139 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5140 TEST_ASSERT_SUCCESS(ret, 5141 "rte_bbdev_enc_op_alloc_bulk() failed"); 5142 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5143 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 5144 bufs->inputs, 5145 bufs->hard_outputs, 5146 ref_op); 5147 5148 /* Start time meas for enqueue function offload latency */ 5149 enq_start_time = rte_rdtsc_precise(); 5150 do { 5151 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 5152 &ops_enq[enq], burst_sz - enq); 5153 } while (unlikely(burst_sz != enq)); 5154 5155 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 5156 5157 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5158 TEST_ASSERT_SUCCESS(ret, 5159 "Failed to get stats for queue (%u) of device (%u)", 5160 queue_id, dev_id); 5161 enq_sw_last_time -= stats.acc_offload_cycles; 5162 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5163 enq_sw_last_time); 5164 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5165 enq_sw_last_time); 5166 time_st->enq_sw_total_time += enq_sw_last_time; 5167 5168 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5169 stats.acc_offload_cycles); 5170 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5171 stats.acc_offload_cycles); 5172 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5173 5174 /* give time for device to process ops */ 5175 rte_delay_us(WAIT_OFFLOAD_US); 5176 5177 /* Start time meas for dequeue function offload latency */ 5178 deq_start_time = rte_rdtsc_precise(); 5179 /* Dequeue one operation */ 5180 do { 5181 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5182 &ops_deq[deq], enq); 5183 } while (unlikely(deq == 0)); 5184 5185 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5186 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5187 deq_last_time); 5188 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5189 deq_last_time); 5190 time_st->deq_total_time += deq_last_time; 5191 5192 while (burst_sz != deq) 5193 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 5194 &ops_deq[deq], burst_sz - deq); 5195 5196 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5197 dequeued += deq; 5198 } 5199 5200 return i; 5201 } 5202 5203 static int 5204 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 5205 struct test_buffers *bufs, 5206 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 5207 uint16_t queue_id, const uint16_t num_to_process, 5208 uint16_t burst_sz, struct test_time_stats *time_st) 5209 { 5210 int i, dequeued, ret; 5211 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 5212 uint64_t enq_start_time, deq_start_time; 5213 uint64_t enq_sw_last_time, deq_last_time; 5214 struct rte_bbdev_stats stats; 5215 5216 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 5217 uint16_t enq = 0, deq = 0; 5218 5219 if (unlikely(num_to_process - dequeued < burst_sz)) 5220 burst_sz = num_to_process - dequeued; 5221 5222 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 5223 TEST_ASSERT_SUCCESS(ret, 5224 "rte_bbdev_enc_op_alloc_bulk() failed"); 5225 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 5226 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 5227 bufs->inputs, 5228 bufs->hard_outputs, 5229 ref_op); 5230 5231 /* Start time meas for enqueue function offload latency */ 5232 enq_start_time = rte_rdtsc_precise(); 5233 do { 5234 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 5235 &ops_enq[enq], burst_sz - enq); 5236 } while (unlikely(burst_sz != enq)); 5237 5238 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 5239 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 5240 TEST_ASSERT_SUCCESS(ret, 5241 "Failed to get stats for queue (%u) of device (%u)", 5242 queue_id, dev_id); 5243 5244 enq_sw_last_time -= stats.acc_offload_cycles; 5245 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 5246 enq_sw_last_time); 5247 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 5248 enq_sw_last_time); 5249 time_st->enq_sw_total_time += enq_sw_last_time; 5250 5251 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 5252 stats.acc_offload_cycles); 5253 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 5254 stats.acc_offload_cycles); 5255 time_st->enq_acc_total_time += stats.acc_offload_cycles; 5256 5257 /* give time for device to process ops */ 5258 rte_delay_us(WAIT_OFFLOAD_US); 5259 5260 /* Start time meas for dequeue function offload latency */ 5261 deq_start_time = rte_rdtsc_precise(); 5262 /* Dequeue one operation */ 5263 do { 5264 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5265 &ops_deq[deq], enq); 5266 } while (unlikely(deq == 0)); 5267 5268 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5269 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 5270 deq_last_time); 5271 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 5272 deq_last_time); 5273 time_st->deq_total_time += deq_last_time; 5274 5275 while (burst_sz != deq) 5276 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 5277 &ops_deq[deq], burst_sz - deq); 5278 5279 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 5280 dequeued += deq; 5281 } 5282 5283 return i; 5284 } 5285 #endif 5286 5287 static int 5288 offload_cost_test(struct active_device *ad, 5289 struct test_op_params *op_params) 5290 { 5291 #ifndef RTE_BBDEV_OFFLOAD_COST 5292 RTE_SET_USED(ad); 5293 RTE_SET_USED(op_params); 5294 printf("Offload latency test is disabled.\n"); 5295 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 5296 return TEST_SKIPPED; 5297 #else 5298 int iter; 5299 uint16_t burst_sz = op_params->burst_sz; 5300 const uint16_t num_to_process = op_params->num_to_process; 5301 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5302 const uint16_t queue_id = ad->queue_ids[0]; 5303 struct test_buffers *bufs = NULL; 5304 struct rte_bbdev_info info; 5305 const char *op_type_str; 5306 struct test_time_stats time_st; 5307 5308 memset(&time_st, 0, sizeof(struct test_time_stats)); 5309 time_st.enq_sw_min_time = UINT64_MAX; 5310 time_st.enq_acc_min_time = UINT64_MAX; 5311 time_st.deq_min_time = UINT64_MAX; 5312 5313 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5314 "BURST_SIZE should be <= %u", MAX_BURST); 5315 5316 rte_bbdev_info_get(ad->dev_id, &info); 5317 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 5318 5319 op_type_str = rte_bbdev_op_type_str(op_type); 5320 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5321 5322 printf("+ ------------------------------------------------------- +\n"); 5323 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 5324 info.dev_name, burst_sz, num_to_process, op_type_str); 5325 5326 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 5327 iter = offload_latency_test_dec(op_params->mp, bufs, 5328 op_params->ref_dec_op, ad->dev_id, queue_id, 5329 num_to_process, burst_sz, &time_st); 5330 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 5331 iter = offload_latency_test_enc(op_params->mp, bufs, 5332 op_params->ref_enc_op, ad->dev_id, queue_id, 5333 num_to_process, burst_sz, &time_st); 5334 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5335 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 5336 op_params->ref_enc_op, ad->dev_id, queue_id, 5337 num_to_process, burst_sz, &time_st); 5338 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5339 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 5340 op_params->ref_dec_op, ad->dev_id, queue_id, 5341 num_to_process, burst_sz, &time_st); 5342 else if (op_type == RTE_BBDEV_OP_FFT) 5343 iter = offload_latency_test_fft(op_params->mp, bufs, 5344 op_params->ref_fft_op, ad->dev_id, queue_id, 5345 num_to_process, burst_sz, &time_st); 5346 else 5347 iter = offload_latency_test_enc(op_params->mp, bufs, 5348 op_params->ref_enc_op, ad->dev_id, queue_id, 5349 num_to_process, burst_sz, &time_st); 5350 5351 if (iter <= 0) 5352 return TEST_FAILED; 5353 5354 printf("Enqueue driver offload cost latency:\n" 5355 "\tavg: %lg cycles, %lg us\n" 5356 "\tmin: %lg cycles, %lg us\n" 5357 "\tmax: %lg cycles, %lg us\n" 5358 "Enqueue accelerator offload cost latency:\n" 5359 "\tavg: %lg cycles, %lg us\n" 5360 "\tmin: %lg cycles, %lg us\n" 5361 "\tmax: %lg cycles, %lg us\n", 5362 (double)time_st.enq_sw_total_time / (double)iter, 5363 (double)(time_st.enq_sw_total_time * 1000000) / 5364 (double)iter / (double)rte_get_tsc_hz(), 5365 (double)time_st.enq_sw_min_time, 5366 (double)(time_st.enq_sw_min_time * 1000000) / 5367 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 5368 (double)(time_st.enq_sw_max_time * 1000000) / 5369 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 5370 (double)iter, 5371 (double)(time_st.enq_acc_total_time * 1000000) / 5372 (double)iter / (double)rte_get_tsc_hz(), 5373 (double)time_st.enq_acc_min_time, 5374 (double)(time_st.enq_acc_min_time * 1000000) / 5375 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 5376 (double)(time_st.enq_acc_max_time * 1000000) / 5377 rte_get_tsc_hz()); 5378 5379 printf("Dequeue offload cost latency - one op:\n" 5380 "\tavg: %lg cycles, %lg us\n" 5381 "\tmin: %lg cycles, %lg us\n" 5382 "\tmax: %lg cycles, %lg us\n", 5383 (double)time_st.deq_total_time / (double)iter, 5384 (double)(time_st.deq_total_time * 1000000) / 5385 (double)iter / (double)rte_get_tsc_hz(), 5386 (double)time_st.deq_min_time, 5387 (double)(time_st.deq_min_time * 1000000) / 5388 rte_get_tsc_hz(), (double)time_st.deq_max_time, 5389 (double)(time_st.deq_max_time * 1000000) / 5390 rte_get_tsc_hz()); 5391 5392 struct rte_bbdev_stats stats = {0}; 5393 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 5394 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 5395 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 5396 "Mismatch in enqueue count %10"PRIu64" %d", 5397 stats.enqueued_count, num_to_process); 5398 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 5399 "Mismatch in dequeue count %10"PRIu64" %d", 5400 stats.dequeued_count, num_to_process); 5401 } 5402 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 5403 "Enqueue count Error %10"PRIu64"", 5404 stats.enqueue_err_count); 5405 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 5406 "Dequeue count Error (%10"PRIu64"", 5407 stats.dequeue_err_count); 5408 5409 return TEST_SUCCESS; 5410 #endif 5411 } 5412 5413 #ifdef RTE_BBDEV_OFFLOAD_COST 5414 static int 5415 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 5416 const uint16_t num_to_process, uint16_t burst_sz, 5417 uint64_t *deq_total_time, uint64_t *deq_min_time, 5418 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 5419 { 5420 int i, deq_total; 5421 struct rte_bbdev_dec_op *ops[MAX_BURST]; 5422 uint64_t deq_start_time, deq_last_time; 5423 5424 /* Test deq offload latency from an empty queue */ 5425 5426 for (i = 0, deq_total = 0; deq_total < num_to_process; 5427 ++i, deq_total += burst_sz) { 5428 deq_start_time = rte_rdtsc_precise(); 5429 5430 if (unlikely(num_to_process - deq_total < burst_sz)) 5431 burst_sz = num_to_process - deq_total; 5432 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 5433 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 5434 burst_sz); 5435 else 5436 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 5437 burst_sz); 5438 5439 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5440 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 5441 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 5442 *deq_total_time += deq_last_time; 5443 } 5444 5445 return i; 5446 } 5447 5448 static int 5449 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 5450 const uint16_t num_to_process, uint16_t burst_sz, 5451 uint64_t *deq_total_time, uint64_t *deq_min_time, 5452 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 5453 { 5454 int i, deq_total; 5455 struct rte_bbdev_enc_op *ops[MAX_BURST]; 5456 uint64_t deq_start_time, deq_last_time; 5457 5458 /* Test deq offload latency from an empty queue */ 5459 for (i = 0, deq_total = 0; deq_total < num_to_process; 5460 ++i, deq_total += burst_sz) { 5461 deq_start_time = rte_rdtsc_precise(); 5462 5463 if (unlikely(num_to_process - deq_total < burst_sz)) 5464 burst_sz = num_to_process - deq_total; 5465 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 5466 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 5467 burst_sz); 5468 else 5469 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 5470 burst_sz); 5471 5472 deq_last_time = rte_rdtsc_precise() - deq_start_time; 5473 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 5474 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 5475 *deq_total_time += deq_last_time; 5476 } 5477 5478 return i; 5479 } 5480 5481 #endif 5482 5483 static int 5484 offload_latency_empty_q_test(struct active_device *ad, 5485 struct test_op_params *op_params) 5486 { 5487 #ifndef RTE_BBDEV_OFFLOAD_COST 5488 RTE_SET_USED(ad); 5489 RTE_SET_USED(op_params); 5490 printf("Offload latency empty dequeue test is disabled.\n"); 5491 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 5492 return TEST_SKIPPED; 5493 #else 5494 int iter; 5495 uint64_t deq_total_time, deq_min_time, deq_max_time; 5496 uint16_t burst_sz = op_params->burst_sz; 5497 const uint16_t num_to_process = op_params->num_to_process; 5498 const enum rte_bbdev_op_type op_type = test_vector.op_type; 5499 const uint16_t queue_id = ad->queue_ids[0]; 5500 struct rte_bbdev_info info; 5501 const char *op_type_str; 5502 5503 deq_total_time = deq_max_time = 0; 5504 deq_min_time = UINT64_MAX; 5505 5506 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 5507 "BURST_SIZE should be <= %u", MAX_BURST); 5508 5509 rte_bbdev_info_get(ad->dev_id, &info); 5510 5511 op_type_str = rte_bbdev_op_type_str(op_type); 5512 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 5513 5514 printf("+ ------------------------------------------------------- +\n"); 5515 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 5516 info.dev_name, burst_sz, num_to_process, op_type_str); 5517 5518 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 5519 op_type == RTE_BBDEV_OP_LDPC_DEC) 5520 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 5521 num_to_process, burst_sz, &deq_total_time, 5522 &deq_min_time, &deq_max_time, op_type); 5523 else 5524 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 5525 num_to_process, burst_sz, &deq_total_time, 5526 &deq_min_time, &deq_max_time, op_type); 5527 5528 if (iter <= 0) 5529 return TEST_FAILED; 5530 5531 printf("Empty dequeue offload:\n" 5532 "\tavg: %lg cycles, %lg us\n" 5533 "\tmin: %lg cycles, %lg us\n" 5534 "\tmax: %lg cycles, %lg us\n", 5535 (double)deq_total_time / (double)iter, 5536 (double)(deq_total_time * 1000000) / (double)iter / 5537 (double)rte_get_tsc_hz(), (double)deq_min_time, 5538 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 5539 (double)deq_max_time, (double)(deq_max_time * 1000000) / 5540 rte_get_tsc_hz()); 5541 5542 return TEST_SUCCESS; 5543 #endif 5544 } 5545 5546 static int 5547 bler_tc(void) 5548 { 5549 return run_test_case(bler_test); 5550 } 5551 5552 static int 5553 throughput_tc(void) 5554 { 5555 return run_test_case(throughput_test); 5556 } 5557 5558 static int 5559 offload_cost_tc(void) 5560 { 5561 return run_test_case(offload_cost_test); 5562 } 5563 5564 static int 5565 offload_latency_empty_q_tc(void) 5566 { 5567 return run_test_case(offload_latency_empty_q_test); 5568 } 5569 5570 static int 5571 latency_tc(void) 5572 { 5573 return run_test_case(latency_test); 5574 } 5575 5576 static int 5577 validation_tc(void) 5578 { 5579 return run_test_case(validation_test); 5580 } 5581 5582 static int 5583 interrupt_tc(void) 5584 { 5585 return run_test_case(throughput_test); 5586 } 5587 5588 static struct unit_test_suite bbdev_bler_testsuite = { 5589 .suite_name = "BBdev BLER Tests", 5590 .setup = testsuite_setup, 5591 .teardown = testsuite_teardown, 5592 .unit_test_cases = { 5593 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 5594 TEST_CASES_END() /**< NULL terminate unit test array */ 5595 } 5596 }; 5597 5598 static struct unit_test_suite bbdev_throughput_testsuite = { 5599 .suite_name = "BBdev Throughput Tests", 5600 .setup = testsuite_setup, 5601 .teardown = testsuite_teardown, 5602 .unit_test_cases = { 5603 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 5604 TEST_CASES_END() /**< NULL terminate unit test array */ 5605 } 5606 }; 5607 5608 static struct unit_test_suite bbdev_validation_testsuite = { 5609 .suite_name = "BBdev Validation Tests", 5610 .setup = testsuite_setup, 5611 .teardown = testsuite_teardown, 5612 .unit_test_cases = { 5613 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 5614 TEST_CASES_END() /**< NULL terminate unit test array */ 5615 } 5616 }; 5617 5618 static struct unit_test_suite bbdev_latency_testsuite = { 5619 .suite_name = "BBdev Latency Tests", 5620 .setup = testsuite_setup, 5621 .teardown = testsuite_teardown, 5622 .unit_test_cases = { 5623 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 5624 TEST_CASES_END() /**< NULL terminate unit test array */ 5625 } 5626 }; 5627 5628 static struct unit_test_suite bbdev_offload_cost_testsuite = { 5629 .suite_name = "BBdev Offload Cost Tests", 5630 .setup = testsuite_setup, 5631 .teardown = testsuite_teardown, 5632 .unit_test_cases = { 5633 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 5634 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 5635 TEST_CASES_END() /**< NULL terminate unit test array */ 5636 } 5637 }; 5638 5639 static struct unit_test_suite bbdev_interrupt_testsuite = { 5640 .suite_name = "BBdev Interrupt Tests", 5641 .setup = interrupt_testsuite_setup, 5642 .teardown = testsuite_teardown, 5643 .unit_test_cases = { 5644 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 5645 TEST_CASES_END() /**< NULL terminate unit test array */ 5646 } 5647 }; 5648 5649 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 5650 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 5651 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 5652 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 5653 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 5654 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 5655