1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #include "main.h" 22 #include "test_bbdev_vector.h" 23 24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 25 26 #define MAX_QUEUES RTE_MAX_LCORE 27 #define TEST_REPETITIONS 100 28 #define WAIT_OFFLOAD_US 1000 29 30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 31 #include <fpga_lte_fec.h> 32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 34 #define VF_UL_4G_QUEUE_VALUE 4 35 #define VF_DL_4G_QUEUE_VALUE 4 36 #define UL_4G_BANDWIDTH 3 37 #define DL_4G_BANDWIDTH 3 38 #define UL_4G_LOAD_BALANCE 128 39 #define DL_4G_LOAD_BALANCE 128 40 #define FLR_4G_TIMEOUT 610 41 #endif 42 43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 44 #include <rte_pmd_fpga_5gnr_fec.h> 45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 47 #define VF_UL_5G_QUEUE_VALUE 4 48 #define VF_DL_5G_QUEUE_VALUE 4 49 #define UL_5G_BANDWIDTH 3 50 #define DL_5G_BANDWIDTH 3 51 #define UL_5G_LOAD_BALANCE 128 52 #define DL_5G_LOAD_BALANCE 128 53 #define FLR_5G_TIMEOUT 610 54 #endif 55 56 #ifdef RTE_BASEBAND_ACC100 57 #include <rte_acc100_cfg.h> 58 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 59 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 60 #define ACC100_QMGR_NUM_AQS 16 61 #define ACC100_QMGR_NUM_QGS 2 62 #define ACC100_QMGR_AQ_DEPTH 5 63 #define ACC100_QMGR_INVALID_IDX -1 64 #define ACC100_QMGR_RR 1 65 #define ACC100_QOS_GBR 0 66 #endif 67 68 #define OPS_CACHE_SIZE 256U 69 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 70 71 #define SYNC_WAIT 0 72 #define SYNC_START 1 73 #define INVALID_OPAQUE -1 74 75 #define INVALID_QUEUE_ID -1 76 /* Increment for next code block in external HARQ memory */ 77 #define HARQ_INCR 32768 78 /* Headroom for filler LLRs insertion in HARQ buffer */ 79 #define FILLER_HEADROOM 1024 80 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 81 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 82 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 83 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 84 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 85 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 86 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 87 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 88 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 89 90 static struct test_bbdev_vector test_vector; 91 92 /* Switch between PMD and Interrupt for throughput TC */ 93 static bool intr_enabled; 94 95 /* LLR arithmetic representation for numerical conversion */ 96 static int ldpc_llr_decimals; 97 static int ldpc_llr_size; 98 /* Keep track of the LDPC decoder device capability flag */ 99 static uint32_t ldpc_cap_flags; 100 101 /* Represents tested active devices */ 102 static struct active_device { 103 const char *driver_name; 104 uint8_t dev_id; 105 uint16_t supported_ops; 106 uint16_t queue_ids[MAX_QUEUES]; 107 uint16_t nb_queues; 108 struct rte_mempool *ops_mempool; 109 struct rte_mempool *in_mbuf_pool; 110 struct rte_mempool *hard_out_mbuf_pool; 111 struct rte_mempool *soft_out_mbuf_pool; 112 struct rte_mempool *harq_in_mbuf_pool; 113 struct rte_mempool *harq_out_mbuf_pool; 114 } active_devs[RTE_BBDEV_MAX_DEVS]; 115 116 static uint8_t nb_active_devs; 117 118 /* Data buffers used by BBDEV ops */ 119 struct test_buffers { 120 struct rte_bbdev_op_data *inputs; 121 struct rte_bbdev_op_data *hard_outputs; 122 struct rte_bbdev_op_data *soft_outputs; 123 struct rte_bbdev_op_data *harq_inputs; 124 struct rte_bbdev_op_data *harq_outputs; 125 }; 126 127 /* Operation parameters specific for given test case */ 128 struct test_op_params { 129 struct rte_mempool *mp; 130 struct rte_bbdev_dec_op *ref_dec_op; 131 struct rte_bbdev_enc_op *ref_enc_op; 132 uint16_t burst_sz; 133 uint16_t num_to_process; 134 uint16_t num_lcores; 135 int vector_mask; 136 uint16_t sync; 137 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 138 }; 139 140 /* Contains per lcore params */ 141 struct thread_params { 142 uint8_t dev_id; 143 uint16_t queue_id; 144 uint32_t lcore_id; 145 uint64_t start_time; 146 double ops_per_sec; 147 double mbps; 148 uint8_t iter_count; 149 double iter_average; 150 double bler; 151 uint16_t nb_dequeued; 152 int16_t processing_status; 153 uint16_t burst_sz; 154 struct test_op_params *op_params; 155 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 156 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 157 }; 158 159 #ifdef RTE_BBDEV_OFFLOAD_COST 160 /* Stores time statistics */ 161 struct test_time_stats { 162 /* Stores software enqueue total working time */ 163 uint64_t enq_sw_total_time; 164 /* Stores minimum value of software enqueue working time */ 165 uint64_t enq_sw_min_time; 166 /* Stores maximum value of software enqueue working time */ 167 uint64_t enq_sw_max_time; 168 /* Stores turbo enqueue total working time */ 169 uint64_t enq_acc_total_time; 170 /* Stores minimum value of accelerator enqueue working time */ 171 uint64_t enq_acc_min_time; 172 /* Stores maximum value of accelerator enqueue working time */ 173 uint64_t enq_acc_max_time; 174 /* Stores dequeue total working time */ 175 uint64_t deq_total_time; 176 /* Stores minimum value of dequeue working time */ 177 uint64_t deq_min_time; 178 /* Stores maximum value of dequeue working time */ 179 uint64_t deq_max_time; 180 }; 181 #endif 182 183 typedef int (test_case_function)(struct active_device *ad, 184 struct test_op_params *op_params); 185 186 static inline void 187 mbuf_reset(struct rte_mbuf *m) 188 { 189 m->pkt_len = 0; 190 191 do { 192 m->data_len = 0; 193 m = m->next; 194 } while (m != NULL); 195 } 196 197 /* Read flag value 0/1 from bitmap */ 198 static inline bool 199 check_bit(uint32_t bitmap, uint32_t bitmask) 200 { 201 return bitmap & bitmask; 202 } 203 204 static inline void 205 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 206 { 207 ad->supported_ops |= (1 << op_type); 208 } 209 210 static inline bool 211 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 212 { 213 return ad->supported_ops & (1 << op_type); 214 } 215 216 static inline bool 217 flags_match(uint32_t flags_req, uint32_t flags_present) 218 { 219 return (flags_req & flags_present) == flags_req; 220 } 221 222 static void 223 clear_soft_out_cap(uint32_t *op_flags) 224 { 225 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 226 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 227 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 228 } 229 230 /* This API is to convert all the test vector op data entries 231 * to big endian format. It is used when the device supports 232 * the input in the big endian format. 233 */ 234 static inline void 235 convert_op_data_to_be(void) 236 { 237 struct op_data_entries *op; 238 enum op_data_type type; 239 uint8_t nb_segs, *rem_data, temp; 240 uint32_t *data, len; 241 int complete, rem, i, j; 242 243 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 244 nb_segs = test_vector.entries[type].nb_segments; 245 op = &test_vector.entries[type]; 246 247 /* Invert byte endianness for all the segments */ 248 for (i = 0; i < nb_segs; ++i) { 249 len = op->segments[i].length; 250 data = op->segments[i].addr; 251 252 /* Swap complete u32 bytes */ 253 complete = len / 4; 254 for (j = 0; j < complete; j++) 255 data[j] = rte_bswap32(data[j]); 256 257 /* Swap any remaining bytes */ 258 rem = len % 4; 259 rem_data = (uint8_t *)&data[j]; 260 for (j = 0; j < rem/2; j++) { 261 temp = rem_data[j]; 262 rem_data[j] = rem_data[rem - j - 1]; 263 rem_data[rem - j - 1] = temp; 264 } 265 } 266 } 267 } 268 269 static int 270 check_dev_cap(const struct rte_bbdev_info *dev_info) 271 { 272 unsigned int i; 273 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 274 nb_harq_inputs, nb_harq_outputs; 275 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 276 uint8_t dev_data_endianness = dev_info->drv.data_endianness; 277 278 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 279 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 280 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 281 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 282 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 283 284 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 285 if (op_cap->type != test_vector.op_type) 286 continue; 287 288 if (dev_data_endianness == RTE_BIG_ENDIAN) 289 convert_op_data_to_be(); 290 291 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 292 const struct rte_bbdev_op_cap_turbo_dec *cap = 293 &op_cap->cap.turbo_dec; 294 /* Ignore lack of soft output capability, just skip 295 * checking if soft output is valid. 296 */ 297 if ((test_vector.turbo_dec.op_flags & 298 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 299 !(cap->capability_flags & 300 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 301 printf( 302 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 303 dev_info->dev_name); 304 clear_soft_out_cap( 305 &test_vector.turbo_dec.op_flags); 306 } 307 308 if (!flags_match(test_vector.turbo_dec.op_flags, 309 cap->capability_flags)) 310 return TEST_FAILED; 311 if (nb_inputs > cap->num_buffers_src) { 312 printf("Too many inputs defined: %u, max: %u\n", 313 nb_inputs, cap->num_buffers_src); 314 return TEST_FAILED; 315 } 316 if (nb_soft_outputs > cap->num_buffers_soft_out && 317 (test_vector.turbo_dec.op_flags & 318 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 319 printf( 320 "Too many soft outputs defined: %u, max: %u\n", 321 nb_soft_outputs, 322 cap->num_buffers_soft_out); 323 return TEST_FAILED; 324 } 325 if (nb_hard_outputs > cap->num_buffers_hard_out) { 326 printf( 327 "Too many hard outputs defined: %u, max: %u\n", 328 nb_hard_outputs, 329 cap->num_buffers_hard_out); 330 return TEST_FAILED; 331 } 332 if (intr_enabled && !(cap->capability_flags & 333 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 334 printf( 335 "Dequeue interrupts are not supported!\n"); 336 return TEST_FAILED; 337 } 338 339 return TEST_SUCCESS; 340 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 341 const struct rte_bbdev_op_cap_turbo_enc *cap = 342 &op_cap->cap.turbo_enc; 343 344 if (!flags_match(test_vector.turbo_enc.op_flags, 345 cap->capability_flags)) 346 return TEST_FAILED; 347 if (nb_inputs > cap->num_buffers_src) { 348 printf("Too many inputs defined: %u, max: %u\n", 349 nb_inputs, cap->num_buffers_src); 350 return TEST_FAILED; 351 } 352 if (nb_hard_outputs > cap->num_buffers_dst) { 353 printf( 354 "Too many hard outputs defined: %u, max: %u\n", 355 nb_hard_outputs, cap->num_buffers_dst); 356 return TEST_FAILED; 357 } 358 if (intr_enabled && !(cap->capability_flags & 359 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 360 printf( 361 "Dequeue interrupts are not supported!\n"); 362 return TEST_FAILED; 363 } 364 365 return TEST_SUCCESS; 366 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 367 const struct rte_bbdev_op_cap_ldpc_enc *cap = 368 &op_cap->cap.ldpc_enc; 369 370 if (!flags_match(test_vector.ldpc_enc.op_flags, 371 cap->capability_flags)){ 372 printf("Flag Mismatch\n"); 373 return TEST_FAILED; 374 } 375 if (nb_inputs > cap->num_buffers_src) { 376 printf("Too many inputs defined: %u, max: %u\n", 377 nb_inputs, cap->num_buffers_src); 378 return TEST_FAILED; 379 } 380 if (nb_hard_outputs > cap->num_buffers_dst) { 381 printf( 382 "Too many hard outputs defined: %u, max: %u\n", 383 nb_hard_outputs, cap->num_buffers_dst); 384 return TEST_FAILED; 385 } 386 if (intr_enabled && !(cap->capability_flags & 387 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 388 printf( 389 "Dequeue interrupts are not supported!\n"); 390 return TEST_FAILED; 391 } 392 393 return TEST_SUCCESS; 394 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 395 const struct rte_bbdev_op_cap_ldpc_dec *cap = 396 &op_cap->cap.ldpc_dec; 397 398 if (!flags_match(test_vector.ldpc_dec.op_flags, 399 cap->capability_flags)){ 400 printf("Flag Mismatch\n"); 401 return TEST_FAILED; 402 } 403 if (nb_inputs > cap->num_buffers_src) { 404 printf("Too many inputs defined: %u, max: %u\n", 405 nb_inputs, cap->num_buffers_src); 406 return TEST_FAILED; 407 } 408 if (nb_hard_outputs > cap->num_buffers_hard_out) { 409 printf( 410 "Too many hard outputs defined: %u, max: %u\n", 411 nb_hard_outputs, 412 cap->num_buffers_hard_out); 413 return TEST_FAILED; 414 } 415 if (nb_harq_inputs > cap->num_buffers_hard_out) { 416 printf( 417 "Too many HARQ inputs defined: %u, max: %u\n", 418 nb_harq_inputs, 419 cap->num_buffers_hard_out); 420 return TEST_FAILED; 421 } 422 if (nb_harq_outputs > cap->num_buffers_hard_out) { 423 printf( 424 "Too many HARQ outputs defined: %u, max: %u\n", 425 nb_harq_outputs, 426 cap->num_buffers_hard_out); 427 return TEST_FAILED; 428 } 429 if (intr_enabled && !(cap->capability_flags & 430 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 431 printf( 432 "Dequeue interrupts are not supported!\n"); 433 return TEST_FAILED; 434 } 435 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 436 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 437 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 438 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 439 ))) { 440 printf("Skip loop-back with interrupt\n"); 441 return TEST_FAILED; 442 } 443 return TEST_SUCCESS; 444 } 445 } 446 447 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 448 return TEST_SUCCESS; /* Special case for NULL device */ 449 450 return TEST_FAILED; 451 } 452 453 /* calculates optimal mempool size not smaller than the val */ 454 static unsigned int 455 optimal_mempool_size(unsigned int val) 456 { 457 return rte_align32pow2(val + 1) - 1; 458 } 459 460 /* allocates mbuf mempool for inputs and outputs */ 461 static struct rte_mempool * 462 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 463 int socket_id, unsigned int mbuf_pool_size, 464 const char *op_type_str) 465 { 466 unsigned int i; 467 uint32_t max_seg_sz = 0; 468 char pool_name[RTE_MEMPOOL_NAMESIZE]; 469 470 /* find max input segment size */ 471 for (i = 0; i < entries->nb_segments; ++i) 472 if (entries->segments[i].length > max_seg_sz) 473 max_seg_sz = entries->segments[i].length; 474 475 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 476 dev_id); 477 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 478 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 479 + FILLER_HEADROOM, 480 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 481 } 482 483 static int 484 create_mempools(struct active_device *ad, int socket_id, 485 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 486 { 487 struct rte_mempool *mp; 488 unsigned int ops_pool_size, mbuf_pool_size = 0; 489 char pool_name[RTE_MEMPOOL_NAMESIZE]; 490 const char *op_type_str; 491 enum rte_bbdev_op_type op_type = org_op_type; 492 493 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 494 struct op_data_entries *hard_out = 495 &test_vector.entries[DATA_HARD_OUTPUT]; 496 struct op_data_entries *soft_out = 497 &test_vector.entries[DATA_SOFT_OUTPUT]; 498 struct op_data_entries *harq_in = 499 &test_vector.entries[DATA_HARQ_INPUT]; 500 struct op_data_entries *harq_out = 501 &test_vector.entries[DATA_HARQ_OUTPUT]; 502 503 /* allocate ops mempool */ 504 ops_pool_size = optimal_mempool_size(RTE_MAX( 505 /* Ops used plus 1 reference op */ 506 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 507 /* Minimal cache size plus 1 reference op */ 508 (unsigned int)(1.5 * rte_lcore_count() * 509 OPS_CACHE_SIZE + 1)), 510 OPS_POOL_SIZE_MIN)); 511 512 if (org_op_type == RTE_BBDEV_OP_NONE) 513 op_type = RTE_BBDEV_OP_TURBO_ENC; 514 515 op_type_str = rte_bbdev_op_type_str(op_type); 516 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 517 518 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 519 ad->dev_id); 520 mp = rte_bbdev_op_pool_create(pool_name, op_type, 521 ops_pool_size, OPS_CACHE_SIZE, socket_id); 522 TEST_ASSERT_NOT_NULL(mp, 523 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 524 ops_pool_size, 525 ad->dev_id, 526 socket_id); 527 ad->ops_mempool = mp; 528 529 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 530 if (org_op_type == RTE_BBDEV_OP_NONE) 531 return TEST_SUCCESS; 532 533 /* Inputs */ 534 if (in->nb_segments > 0) { 535 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 536 in->nb_segments); 537 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 538 mbuf_pool_size, "in"); 539 TEST_ASSERT_NOT_NULL(mp, 540 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 541 mbuf_pool_size, 542 ad->dev_id, 543 socket_id); 544 ad->in_mbuf_pool = mp; 545 } 546 547 /* Hard outputs */ 548 if (hard_out->nb_segments > 0) { 549 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 550 hard_out->nb_segments); 551 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 552 mbuf_pool_size, 553 "hard_out"); 554 TEST_ASSERT_NOT_NULL(mp, 555 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 556 mbuf_pool_size, 557 ad->dev_id, 558 socket_id); 559 ad->hard_out_mbuf_pool = mp; 560 } 561 562 /* Soft outputs */ 563 if (soft_out->nb_segments > 0) { 564 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 565 soft_out->nb_segments); 566 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 567 mbuf_pool_size, 568 "soft_out"); 569 TEST_ASSERT_NOT_NULL(mp, 570 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 571 mbuf_pool_size, 572 ad->dev_id, 573 socket_id); 574 ad->soft_out_mbuf_pool = mp; 575 } 576 577 /* HARQ inputs */ 578 if (harq_in->nb_segments > 0) { 579 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 580 harq_in->nb_segments); 581 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 582 mbuf_pool_size, 583 "harq_in"); 584 TEST_ASSERT_NOT_NULL(mp, 585 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 586 mbuf_pool_size, 587 ad->dev_id, 588 socket_id); 589 ad->harq_in_mbuf_pool = mp; 590 } 591 592 /* HARQ outputs */ 593 if (harq_out->nb_segments > 0) { 594 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 595 harq_out->nb_segments); 596 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 597 mbuf_pool_size, 598 "harq_out"); 599 TEST_ASSERT_NOT_NULL(mp, 600 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 601 mbuf_pool_size, 602 ad->dev_id, 603 socket_id); 604 ad->harq_out_mbuf_pool = mp; 605 } 606 607 return TEST_SUCCESS; 608 } 609 610 static int 611 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 612 struct test_bbdev_vector *vector) 613 { 614 int ret; 615 unsigned int queue_id; 616 struct rte_bbdev_queue_conf qconf; 617 struct active_device *ad = &active_devs[nb_active_devs]; 618 unsigned int nb_queues; 619 enum rte_bbdev_op_type op_type = vector->op_type; 620 621 /* Configure fpga lte fec with PF & VF values 622 * if '-i' flag is set and using fpga device 623 */ 624 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 625 if ((get_init_device() == true) && 626 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 627 struct rte_fpga_lte_fec_conf conf; 628 unsigned int i; 629 630 printf("Configure FPGA LTE FEC Driver %s with default values\n", 631 info->drv.driver_name); 632 633 /* clear default configuration before initialization */ 634 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 635 636 /* Set PF mode : 637 * true if PF is used for data plane 638 * false for VFs 639 */ 640 conf.pf_mode_en = true; 641 642 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 643 /* Number of UL queues per VF (fpga supports 8 VFs) */ 644 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 645 /* Number of DL queues per VF (fpga supports 8 VFs) */ 646 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 647 } 648 649 /* UL bandwidth. Needed for schedule algorithm */ 650 conf.ul_bandwidth = UL_4G_BANDWIDTH; 651 /* DL bandwidth */ 652 conf.dl_bandwidth = DL_4G_BANDWIDTH; 653 654 /* UL & DL load Balance Factor to 64 */ 655 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 656 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 657 658 /**< FLR timeout value */ 659 conf.flr_time_out = FLR_4G_TIMEOUT; 660 661 /* setup FPGA PF with configuration information */ 662 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 663 TEST_ASSERT_SUCCESS(ret, 664 "Failed to configure 4G FPGA PF for bbdev %s", 665 info->dev_name); 666 } 667 #endif 668 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 669 if ((get_init_device() == true) && 670 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 671 struct rte_fpga_5gnr_fec_conf conf; 672 unsigned int i; 673 674 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 675 info->drv.driver_name); 676 677 /* clear default configuration before initialization */ 678 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 679 680 /* Set PF mode : 681 * true if PF is used for data plane 682 * false for VFs 683 */ 684 conf.pf_mode_en = true; 685 686 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 687 /* Number of UL queues per VF (fpga supports 8 VFs) */ 688 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 689 /* Number of DL queues per VF (fpga supports 8 VFs) */ 690 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 691 } 692 693 /* UL bandwidth. Needed for schedule algorithm */ 694 conf.ul_bandwidth = UL_5G_BANDWIDTH; 695 /* DL bandwidth */ 696 conf.dl_bandwidth = DL_5G_BANDWIDTH; 697 698 /* UL & DL load Balance Factor to 64 */ 699 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 700 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 701 702 /**< FLR timeout value */ 703 conf.flr_time_out = FLR_5G_TIMEOUT; 704 705 /* setup FPGA PF with configuration information */ 706 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 707 TEST_ASSERT_SUCCESS(ret, 708 "Failed to configure 5G FPGA PF for bbdev %s", 709 info->dev_name); 710 } 711 #endif 712 #ifdef RTE_BASEBAND_ACC100 713 if ((get_init_device() == true) && 714 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 715 struct rte_acc100_conf conf; 716 unsigned int i; 717 718 printf("Configure ACC100 FEC Driver %s with default values\n", 719 info->drv.driver_name); 720 721 /* clear default configuration before initialization */ 722 memset(&conf, 0, sizeof(struct rte_acc100_conf)); 723 724 /* Always set in PF mode for built-in configuration */ 725 conf.pf_mode_en = true; 726 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) { 727 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 728 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 729 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 730 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 731 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 732 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 733 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 734 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 735 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 736 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 737 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 738 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 739 } 740 741 conf.input_pos_llr_1_bit = true; 742 conf.output_pos_llr_1_bit = true; 743 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 744 745 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 746 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 747 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 748 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 749 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 750 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 751 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 752 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 753 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 754 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 755 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 756 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 757 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 758 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 759 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 760 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 761 762 /* setup PF with configuration information */ 763 ret = rte_acc100_configure(info->dev_name, &conf); 764 TEST_ASSERT_SUCCESS(ret, 765 "Failed to configure ACC100 PF for bbdev %s", 766 info->dev_name); 767 } 768 #endif 769 /* Let's refresh this now this is configured */ 770 rte_bbdev_info_get(dev_id, info); 771 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 772 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 773 774 /* setup device */ 775 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 776 if (ret < 0) { 777 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 778 dev_id, nb_queues, info->socket_id, ret); 779 return TEST_FAILED; 780 } 781 782 /* configure interrupts if needed */ 783 if (intr_enabled) { 784 ret = rte_bbdev_intr_enable(dev_id); 785 if (ret < 0) { 786 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 787 ret); 788 return TEST_FAILED; 789 } 790 } 791 792 /* setup device queues */ 793 qconf.socket = info->socket_id; 794 qconf.queue_size = info->drv.default_queue_conf.queue_size; 795 qconf.priority = 0; 796 qconf.deferred_start = 0; 797 qconf.op_type = op_type; 798 799 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 800 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 801 if (ret != 0) { 802 printf( 803 "Allocated all queues (id=%u) at prio%u on dev%u\n", 804 queue_id, qconf.priority, dev_id); 805 qconf.priority++; 806 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 807 &qconf); 808 } 809 if (ret != 0) { 810 printf("All queues on dev %u allocated: %u\n", 811 dev_id, queue_id); 812 break; 813 } 814 ad->queue_ids[queue_id] = queue_id; 815 } 816 TEST_ASSERT(queue_id != 0, 817 "ERROR Failed to configure any queues on dev %u", 818 dev_id); 819 ad->nb_queues = queue_id; 820 821 set_avail_op(ad, op_type); 822 823 return TEST_SUCCESS; 824 } 825 826 static int 827 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 828 struct test_bbdev_vector *vector) 829 { 830 int ret; 831 832 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 833 active_devs[nb_active_devs].dev_id = dev_id; 834 835 ret = add_bbdev_dev(dev_id, info, vector); 836 if (ret == TEST_SUCCESS) 837 ++nb_active_devs; 838 return ret; 839 } 840 841 static uint8_t 842 populate_active_devices(void) 843 { 844 int ret; 845 uint8_t dev_id; 846 uint8_t nb_devs_added = 0; 847 struct rte_bbdev_info info; 848 849 RTE_BBDEV_FOREACH(dev_id) { 850 rte_bbdev_info_get(dev_id, &info); 851 852 if (check_dev_cap(&info)) { 853 printf( 854 "Device %d (%s) does not support specified capabilities\n", 855 dev_id, info.dev_name); 856 continue; 857 } 858 859 ret = add_active_device(dev_id, &info, &test_vector); 860 if (ret != 0) { 861 printf("Adding active bbdev %s skipped\n", 862 info.dev_name); 863 continue; 864 } 865 nb_devs_added++; 866 } 867 868 return nb_devs_added; 869 } 870 871 static int 872 read_test_vector(void) 873 { 874 int ret; 875 876 memset(&test_vector, 0, sizeof(test_vector)); 877 printf("Test vector file = %s\n", get_vector_filename()); 878 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 879 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 880 get_vector_filename()); 881 882 return TEST_SUCCESS; 883 } 884 885 static int 886 testsuite_setup(void) 887 { 888 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 889 890 if (populate_active_devices() == 0) { 891 printf("No suitable devices found!\n"); 892 return TEST_SKIPPED; 893 } 894 895 return TEST_SUCCESS; 896 } 897 898 static int 899 interrupt_testsuite_setup(void) 900 { 901 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 902 903 /* Enable interrupts */ 904 intr_enabled = true; 905 906 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 907 if (populate_active_devices() == 0 || 908 test_vector.op_type == RTE_BBDEV_OP_NONE) { 909 intr_enabled = false; 910 printf("No suitable devices found!\n"); 911 return TEST_SKIPPED; 912 } 913 914 return TEST_SUCCESS; 915 } 916 917 static void 918 testsuite_teardown(void) 919 { 920 uint8_t dev_id; 921 922 /* Unconfigure devices */ 923 RTE_BBDEV_FOREACH(dev_id) 924 rte_bbdev_close(dev_id); 925 926 /* Clear active devices structs. */ 927 memset(active_devs, 0, sizeof(active_devs)); 928 nb_active_devs = 0; 929 930 /* Disable interrupts */ 931 intr_enabled = false; 932 } 933 934 static int 935 ut_setup(void) 936 { 937 uint8_t i, dev_id; 938 939 for (i = 0; i < nb_active_devs; i++) { 940 dev_id = active_devs[i].dev_id; 941 /* reset bbdev stats */ 942 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 943 "Failed to reset stats of bbdev %u", dev_id); 944 /* start the device */ 945 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 946 "Failed to start bbdev %u", dev_id); 947 } 948 949 return TEST_SUCCESS; 950 } 951 952 static void 953 ut_teardown(void) 954 { 955 uint8_t i, dev_id; 956 struct rte_bbdev_stats stats; 957 958 for (i = 0; i < nb_active_devs; i++) { 959 dev_id = active_devs[i].dev_id; 960 /* read stats and print */ 961 rte_bbdev_stats_get(dev_id, &stats); 962 /* Stop the device */ 963 rte_bbdev_stop(dev_id); 964 } 965 } 966 967 static int 968 init_op_data_objs(struct rte_bbdev_op_data *bufs, 969 struct op_data_entries *ref_entries, 970 struct rte_mempool *mbuf_pool, const uint16_t n, 971 enum op_data_type op_type, uint16_t min_alignment) 972 { 973 int ret; 974 unsigned int i, j; 975 bool large_input = false; 976 977 for (i = 0; i < n; ++i) { 978 char *data; 979 struct op_data_buf *seg = &ref_entries->segments[0]; 980 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 981 TEST_ASSERT_NOT_NULL(m_head, 982 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 983 op_type, n * ref_entries->nb_segments, 984 mbuf_pool->size); 985 986 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) { 987 /* 988 * Special case when DPDK mbuf cannot handle 989 * the required input size 990 */ 991 printf("Warning: Larger input size than DPDK mbuf %d\n", 992 seg->length); 993 large_input = true; 994 } 995 bufs[i].data = m_head; 996 bufs[i].offset = 0; 997 bufs[i].length = 0; 998 999 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 1000 if ((op_type == DATA_INPUT) && large_input) { 1001 /* Allocate a fake overused mbuf */ 1002 data = rte_malloc(NULL, seg->length, 0); 1003 TEST_ASSERT_NOT_NULL(data, 1004 "rte malloc failed with %u bytes", 1005 seg->length); 1006 memcpy(data, seg->addr, seg->length); 1007 m_head->buf_addr = data; 1008 m_head->buf_iova = rte_malloc_virt2iova(data); 1009 m_head->data_off = 0; 1010 m_head->data_len = seg->length; 1011 } else { 1012 data = rte_pktmbuf_append(m_head, seg->length); 1013 TEST_ASSERT_NOT_NULL(data, 1014 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1015 seg->length, op_type); 1016 1017 TEST_ASSERT(data == RTE_PTR_ALIGN( 1018 data, min_alignment), 1019 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1020 data, min_alignment); 1021 rte_memcpy(data, seg->addr, seg->length); 1022 } 1023 1024 bufs[i].length += seg->length; 1025 1026 for (j = 1; j < ref_entries->nb_segments; ++j) { 1027 struct rte_mbuf *m_tail = 1028 rte_pktmbuf_alloc(mbuf_pool); 1029 TEST_ASSERT_NOT_NULL(m_tail, 1030 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1031 op_type, 1032 n * ref_entries->nb_segments, 1033 mbuf_pool->size); 1034 seg += 1; 1035 1036 data = rte_pktmbuf_append(m_tail, seg->length); 1037 TEST_ASSERT_NOT_NULL(data, 1038 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 1039 seg->length, op_type); 1040 1041 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 1042 min_alignment), 1043 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 1044 data, min_alignment); 1045 rte_memcpy(data, seg->addr, seg->length); 1046 bufs[i].length += seg->length; 1047 1048 ret = rte_pktmbuf_chain(m_head, m_tail); 1049 TEST_ASSERT_SUCCESS(ret, 1050 "Couldn't chain mbufs from %d data type mbuf pool", 1051 op_type); 1052 } 1053 } else { 1054 1055 /* allocate chained-mbuf for output buffer */ 1056 for (j = 1; j < ref_entries->nb_segments; ++j) { 1057 struct rte_mbuf *m_tail = 1058 rte_pktmbuf_alloc(mbuf_pool); 1059 TEST_ASSERT_NOT_NULL(m_tail, 1060 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1061 op_type, 1062 n * ref_entries->nb_segments, 1063 mbuf_pool->size); 1064 1065 ret = rte_pktmbuf_chain(m_head, m_tail); 1066 TEST_ASSERT_SUCCESS(ret, 1067 "Couldn't chain mbufs from %d data type mbuf pool", 1068 op_type); 1069 } 1070 } 1071 } 1072 1073 return 0; 1074 } 1075 1076 static int 1077 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1078 const int socket) 1079 { 1080 int i; 1081 1082 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1083 if (*buffers == NULL) { 1084 printf("WARNING: Failed to allocate op_data on socket %d\n", 1085 socket); 1086 /* try to allocate memory on other detected sockets */ 1087 for (i = 0; i < socket; i++) { 1088 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1089 if (*buffers != NULL) 1090 break; 1091 } 1092 } 1093 1094 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1095 } 1096 1097 static void 1098 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1099 const uint16_t n, const int8_t max_llr_modulus) 1100 { 1101 uint16_t i, byte_idx; 1102 1103 for (i = 0; i < n; ++i) { 1104 struct rte_mbuf *m = input_ops[i].data; 1105 while (m != NULL) { 1106 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1107 input_ops[i].offset); 1108 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1109 ++byte_idx) 1110 llr[byte_idx] = round((double)max_llr_modulus * 1111 llr[byte_idx] / INT8_MAX); 1112 1113 m = m->next; 1114 } 1115 } 1116 } 1117 1118 /* 1119 * We may have to insert filler bits 1120 * when they are required by the HARQ assumption 1121 */ 1122 static void 1123 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1124 const uint16_t n, struct test_op_params *op_params) 1125 { 1126 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1127 1128 if (input_ops == NULL) 1129 return; 1130 /* No need to add filler if not required by device */ 1131 if (!(ldpc_cap_flags & 1132 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1133 return; 1134 /* No need to add filler for loopback operation */ 1135 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1136 return; 1137 1138 uint16_t i, j, parity_offset; 1139 for (i = 0; i < n; ++i) { 1140 struct rte_mbuf *m = input_ops[i].data; 1141 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1142 input_ops[i].offset); 1143 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1144 * dec.z_c - dec.n_filler; 1145 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1146 m->data_len = new_hin_size; 1147 input_ops[i].length = new_hin_size; 1148 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1149 j--) 1150 llr[j] = llr[j - dec.n_filler]; 1151 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1152 for (j = 0; j < dec.n_filler; j++) 1153 llr[parity_offset + j] = llr_max_pre_scaling; 1154 } 1155 } 1156 1157 static void 1158 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1159 const uint16_t n, const int8_t llr_size, 1160 const int8_t llr_decimals) 1161 { 1162 if (input_ops == NULL) 1163 return; 1164 1165 uint16_t i, byte_idx; 1166 1167 int16_t llr_max, llr_min, llr_tmp; 1168 llr_max = (1 << (llr_size - 1)) - 1; 1169 llr_min = -llr_max; 1170 for (i = 0; i < n; ++i) { 1171 struct rte_mbuf *m = input_ops[i].data; 1172 while (m != NULL) { 1173 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1174 input_ops[i].offset); 1175 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1176 ++byte_idx) { 1177 1178 llr_tmp = llr[byte_idx]; 1179 if (llr_decimals == 4) 1180 llr_tmp *= 8; 1181 else if (llr_decimals == 2) 1182 llr_tmp *= 2; 1183 else if (llr_decimals == 0) 1184 llr_tmp /= 2; 1185 llr_tmp = RTE_MIN(llr_max, 1186 RTE_MAX(llr_min, llr_tmp)); 1187 llr[byte_idx] = (int8_t) llr_tmp; 1188 } 1189 1190 m = m->next; 1191 } 1192 } 1193 } 1194 1195 1196 1197 static int 1198 fill_queue_buffers(struct test_op_params *op_params, 1199 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1200 struct rte_mempool *soft_out_mp, 1201 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1202 uint16_t queue_id, 1203 const struct rte_bbdev_op_cap *capabilities, 1204 uint16_t min_alignment, const int socket_id) 1205 { 1206 int ret; 1207 enum op_data_type type; 1208 const uint16_t n = op_params->num_to_process; 1209 1210 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1211 in_mp, 1212 soft_out_mp, 1213 hard_out_mp, 1214 harq_in_mp, 1215 harq_out_mp, 1216 }; 1217 1218 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1219 &op_params->q_bufs[socket_id][queue_id].inputs, 1220 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1221 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1222 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1223 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1224 }; 1225 1226 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1227 struct op_data_entries *ref_entries = 1228 &test_vector.entries[type]; 1229 if (ref_entries->nb_segments == 0) 1230 continue; 1231 1232 ret = allocate_buffers_on_socket(queue_ops[type], 1233 n * sizeof(struct rte_bbdev_op_data), 1234 socket_id); 1235 TEST_ASSERT_SUCCESS(ret, 1236 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1237 1238 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1239 mbuf_pools[type], n, type, min_alignment); 1240 TEST_ASSERT_SUCCESS(ret, 1241 "Couldn't init rte_bbdev_op_data structs"); 1242 } 1243 1244 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1245 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1246 capabilities->cap.turbo_dec.max_llr_modulus); 1247 1248 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1249 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1250 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1251 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1252 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1253 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1254 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1255 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1256 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1257 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1258 if (!loopback && !llr_comp) 1259 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1260 ldpc_llr_size, ldpc_llr_decimals); 1261 if (!loopback && !harq_comp) 1262 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1263 ldpc_llr_size, ldpc_llr_decimals); 1264 if (!loopback) 1265 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1266 op_params); 1267 } 1268 1269 return 0; 1270 } 1271 1272 static void 1273 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1274 { 1275 unsigned int i, j; 1276 1277 rte_mempool_free(ad->ops_mempool); 1278 rte_mempool_free(ad->in_mbuf_pool); 1279 rte_mempool_free(ad->hard_out_mbuf_pool); 1280 rte_mempool_free(ad->soft_out_mbuf_pool); 1281 rte_mempool_free(ad->harq_in_mbuf_pool); 1282 rte_mempool_free(ad->harq_out_mbuf_pool); 1283 1284 for (i = 0; i < rte_lcore_count(); ++i) { 1285 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1286 rte_free(op_params->q_bufs[j][i].inputs); 1287 rte_free(op_params->q_bufs[j][i].hard_outputs); 1288 rte_free(op_params->q_bufs[j][i].soft_outputs); 1289 rte_free(op_params->q_bufs[j][i].harq_inputs); 1290 rte_free(op_params->q_bufs[j][i].harq_outputs); 1291 } 1292 } 1293 } 1294 1295 static void 1296 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1297 unsigned int start_idx, 1298 struct rte_bbdev_op_data *inputs, 1299 struct rte_bbdev_op_data *hard_outputs, 1300 struct rte_bbdev_op_data *soft_outputs, 1301 struct rte_bbdev_dec_op *ref_op) 1302 { 1303 unsigned int i; 1304 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1305 1306 for (i = 0; i < n; ++i) { 1307 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1308 ops[i]->turbo_dec.tb_params.ea = 1309 turbo_dec->tb_params.ea; 1310 ops[i]->turbo_dec.tb_params.eb = 1311 turbo_dec->tb_params.eb; 1312 ops[i]->turbo_dec.tb_params.k_pos = 1313 turbo_dec->tb_params.k_pos; 1314 ops[i]->turbo_dec.tb_params.k_neg = 1315 turbo_dec->tb_params.k_neg; 1316 ops[i]->turbo_dec.tb_params.c = 1317 turbo_dec->tb_params.c; 1318 ops[i]->turbo_dec.tb_params.c_neg = 1319 turbo_dec->tb_params.c_neg; 1320 ops[i]->turbo_dec.tb_params.cab = 1321 turbo_dec->tb_params.cab; 1322 ops[i]->turbo_dec.tb_params.r = 1323 turbo_dec->tb_params.r; 1324 } else { 1325 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1326 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1327 } 1328 1329 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1330 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1331 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1332 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1333 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1334 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1335 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1336 1337 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1338 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1339 if (soft_outputs != NULL) 1340 ops[i]->turbo_dec.soft_output = 1341 soft_outputs[start_idx + i]; 1342 } 1343 } 1344 1345 static void 1346 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1347 unsigned int start_idx, 1348 struct rte_bbdev_op_data *inputs, 1349 struct rte_bbdev_op_data *outputs, 1350 struct rte_bbdev_enc_op *ref_op) 1351 { 1352 unsigned int i; 1353 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1354 for (i = 0; i < n; ++i) { 1355 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1356 ops[i]->turbo_enc.tb_params.ea = 1357 turbo_enc->tb_params.ea; 1358 ops[i]->turbo_enc.tb_params.eb = 1359 turbo_enc->tb_params.eb; 1360 ops[i]->turbo_enc.tb_params.k_pos = 1361 turbo_enc->tb_params.k_pos; 1362 ops[i]->turbo_enc.tb_params.k_neg = 1363 turbo_enc->tb_params.k_neg; 1364 ops[i]->turbo_enc.tb_params.c = 1365 turbo_enc->tb_params.c; 1366 ops[i]->turbo_enc.tb_params.c_neg = 1367 turbo_enc->tb_params.c_neg; 1368 ops[i]->turbo_enc.tb_params.cab = 1369 turbo_enc->tb_params.cab; 1370 ops[i]->turbo_enc.tb_params.ncb_pos = 1371 turbo_enc->tb_params.ncb_pos; 1372 ops[i]->turbo_enc.tb_params.ncb_neg = 1373 turbo_enc->tb_params.ncb_neg; 1374 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1375 } else { 1376 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1377 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1378 ops[i]->turbo_enc.cb_params.ncb = 1379 turbo_enc->cb_params.ncb; 1380 } 1381 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1382 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1383 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1384 1385 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1386 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1387 } 1388 } 1389 1390 1391 /* Returns a random number drawn from a normal distribution 1392 * with mean of 0 and variance of 1 1393 * Marsaglia algorithm 1394 */ 1395 static double 1396 randn(int n) 1397 { 1398 double S, Z, U1, U2, u, v, fac; 1399 1400 do { 1401 U1 = (double)rand() / RAND_MAX; 1402 U2 = (double)rand() / RAND_MAX; 1403 u = 2. * U1 - 1.; 1404 v = 2. * U2 - 1.; 1405 S = u * u + v * v; 1406 } while (S >= 1 || S == 0); 1407 fac = sqrt(-2. * log(S) / S); 1408 Z = (n % 2) ? u * fac : v * fac; 1409 return Z; 1410 } 1411 1412 static inline double 1413 maxstar(double A, double B) 1414 { 1415 if (fabs(A - B) > 5) 1416 return RTE_MAX(A, B); 1417 else 1418 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1419 } 1420 1421 /* 1422 * Generate Qm LLRS for Qm==8 1423 * Modulation, AWGN and LLR estimation from max log development 1424 */ 1425 static void 1426 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1427 { 1428 int qm = 8; 1429 int qam = 256; 1430 int m, k; 1431 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1432 /* 5.1.4 of TS38.211 */ 1433 const double symbols_I[256] = { 1434 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1435 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1436 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1437 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1438 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1439 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1440 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1441 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1442 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1443 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1444 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1445 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1446 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1447 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1448 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1449 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1450 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1451 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1452 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1453 -13, -13, -15, -15, -13, -13, -15, -15}; 1454 const double symbols_Q[256] = { 1455 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1456 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1457 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1458 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1459 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1460 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1461 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1462 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1463 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1464 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1465 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1466 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1467 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1468 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1469 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1470 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1471 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1472 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1473 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1474 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1475 /* Average constellation point energy */ 1476 N0 *= 170.0; 1477 for (k = 0; k < qm; k++) 1478 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1479 /* 5.1.4 of TS38.211 */ 1480 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1481 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1482 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1483 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1484 /* AWGN channel */ 1485 I += sqrt(N0 / 2) * randn(0); 1486 Q += sqrt(N0 / 2) * randn(1); 1487 /* 1488 * Calculate the log of the probability that each of 1489 * the constellation points was transmitted 1490 */ 1491 for (m = 0; m < qam; m++) 1492 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1493 + pow(Q - symbols_Q[m], 2.0)) / N0; 1494 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1495 for (k = 0; k < qm; k++) { 1496 p0 = -999999; 1497 p1 = -999999; 1498 /* For each constellation point */ 1499 for (m = 0; m < qam; m++) { 1500 if ((m >> (qm - k - 1)) & 1) 1501 p1 = maxstar(p1, log_syml_prob[m]); 1502 else 1503 p0 = maxstar(p0, log_syml_prob[m]); 1504 } 1505 /* Calculate the LLR */ 1506 llr_ = p0 - p1; 1507 llr_ *= (1 << ldpc_llr_decimals); 1508 llr_ = round(llr_); 1509 if (llr_ > llr_max) 1510 llr_ = llr_max; 1511 if (llr_ < -llr_max) 1512 llr_ = -llr_max; 1513 llrs[qm * i + k] = (int8_t) llr_; 1514 } 1515 } 1516 1517 1518 /* 1519 * Generate Qm LLRS for Qm==6 1520 * Modulation, AWGN and LLR estimation from max log development 1521 */ 1522 static void 1523 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1524 { 1525 int qm = 6; 1526 int qam = 64; 1527 int m, k; 1528 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1529 /* 5.1.4 of TS38.211 */ 1530 const double symbols_I[64] = { 1531 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1532 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1533 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1534 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1535 -5, -5, -7, -7, -5, -5, -7, -7}; 1536 const double symbols_Q[64] = { 1537 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1538 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1539 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1540 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1541 -3, -1, -3, -1, -5, -7, -5, -7}; 1542 /* Average constellation point energy */ 1543 N0 *= 42.0; 1544 for (k = 0; k < qm; k++) 1545 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1546 /* 5.1.4 of TS38.211 */ 1547 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1548 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1549 /* AWGN channel */ 1550 I += sqrt(N0 / 2) * randn(0); 1551 Q += sqrt(N0 / 2) * randn(1); 1552 /* 1553 * Calculate the log of the probability that each of 1554 * the constellation points was transmitted 1555 */ 1556 for (m = 0; m < qam; m++) 1557 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1558 + pow(Q - symbols_Q[m], 2.0)) / N0; 1559 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1560 for (k = 0; k < qm; k++) { 1561 p0 = -999999; 1562 p1 = -999999; 1563 /* For each constellation point */ 1564 for (m = 0; m < qam; m++) { 1565 if ((m >> (qm - k - 1)) & 1) 1566 p1 = maxstar(p1, log_syml_prob[m]); 1567 else 1568 p0 = maxstar(p0, log_syml_prob[m]); 1569 } 1570 /* Calculate the LLR */ 1571 llr_ = p0 - p1; 1572 llr_ *= (1 << ldpc_llr_decimals); 1573 llr_ = round(llr_); 1574 if (llr_ > llr_max) 1575 llr_ = llr_max; 1576 if (llr_ < -llr_max) 1577 llr_ = -llr_max; 1578 llrs[qm * i + k] = (int8_t) llr_; 1579 } 1580 } 1581 1582 /* 1583 * Generate Qm LLRS for Qm==4 1584 * Modulation, AWGN and LLR estimation from max log development 1585 */ 1586 static void 1587 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1588 { 1589 int qm = 4; 1590 int qam = 16; 1591 int m, k; 1592 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1593 /* 5.1.4 of TS38.211 */ 1594 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1595 -1, -1, -3, -3, -1, -1, -3, -3}; 1596 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1597 1, 3, 1, 3, -1, -3, -1, -3}; 1598 /* Average constellation point energy */ 1599 N0 *= 10.0; 1600 for (k = 0; k < qm; k++) 1601 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1602 /* 5.1.4 of TS38.211 */ 1603 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1604 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1605 /* AWGN channel */ 1606 I += sqrt(N0 / 2) * randn(0); 1607 Q += sqrt(N0 / 2) * randn(1); 1608 /* 1609 * Calculate the log of the probability that each of 1610 * the constellation points was transmitted 1611 */ 1612 for (m = 0; m < qam; m++) 1613 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1614 + pow(Q - symbols_Q[m], 2.0)) / N0; 1615 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1616 for (k = 0; k < qm; k++) { 1617 p0 = -999999; 1618 p1 = -999999; 1619 /* For each constellation point */ 1620 for (m = 0; m < qam; m++) { 1621 if ((m >> (qm - k - 1)) & 1) 1622 p1 = maxstar(p1, log_syml_prob[m]); 1623 else 1624 p0 = maxstar(p0, log_syml_prob[m]); 1625 } 1626 /* Calculate the LLR */ 1627 llr_ = p0 - p1; 1628 llr_ *= (1 << ldpc_llr_decimals); 1629 llr_ = round(llr_); 1630 if (llr_ > llr_max) 1631 llr_ = llr_max; 1632 if (llr_ < -llr_max) 1633 llr_ = -llr_max; 1634 llrs[qm * i + k] = (int8_t) llr_; 1635 } 1636 } 1637 1638 static void 1639 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1640 { 1641 double b, b1, n; 1642 double coeff = 2.0 * sqrt(N0); 1643 1644 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1645 if (llrs[j] < 8 && llrs[j] > -8) 1646 return; 1647 1648 /* Note don't change sign here */ 1649 n = randn(j % 2); 1650 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1651 + coeff * n) / N0; 1652 b = b1 * (1 << ldpc_llr_decimals); 1653 b = round(b); 1654 if (b > llr_max) 1655 b = llr_max; 1656 if (b < -llr_max) 1657 b = -llr_max; 1658 llrs[j] = (int8_t) b; 1659 } 1660 1661 /* Generate LLR for a given SNR */ 1662 static void 1663 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1664 struct rte_bbdev_dec_op *ref_op) 1665 { 1666 struct rte_mbuf *m; 1667 uint16_t qm; 1668 uint32_t i, j, e, range; 1669 double N0, llr_max; 1670 1671 e = ref_op->ldpc_dec.cb_params.e; 1672 qm = ref_op->ldpc_dec.q_m; 1673 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1674 range = e / qm; 1675 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1676 1677 for (i = 0; i < n; ++i) { 1678 m = inputs[i].data; 1679 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1680 if (qm == 8) { 1681 for (j = 0; j < range; ++j) 1682 gen_qm8_llr(llrs, j, N0, llr_max); 1683 } else if (qm == 6) { 1684 for (j = 0; j < range; ++j) 1685 gen_qm6_llr(llrs, j, N0, llr_max); 1686 } else if (qm == 4) { 1687 for (j = 0; j < range; ++j) 1688 gen_qm4_llr(llrs, j, N0, llr_max); 1689 } else { 1690 for (j = 0; j < e; ++j) 1691 gen_qm2_llr(llrs, j, N0, llr_max); 1692 } 1693 } 1694 } 1695 1696 static void 1697 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1698 unsigned int start_idx, 1699 struct rte_bbdev_op_data *inputs, 1700 struct rte_bbdev_op_data *hard_outputs, 1701 struct rte_bbdev_op_data *soft_outputs, 1702 struct rte_bbdev_op_data *harq_inputs, 1703 struct rte_bbdev_op_data *harq_outputs, 1704 struct rte_bbdev_dec_op *ref_op) 1705 { 1706 unsigned int i; 1707 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1708 1709 for (i = 0; i < n; ++i) { 1710 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1711 ops[i]->ldpc_dec.tb_params.ea = 1712 ldpc_dec->tb_params.ea; 1713 ops[i]->ldpc_dec.tb_params.eb = 1714 ldpc_dec->tb_params.eb; 1715 ops[i]->ldpc_dec.tb_params.c = 1716 ldpc_dec->tb_params.c; 1717 ops[i]->ldpc_dec.tb_params.cab = 1718 ldpc_dec->tb_params.cab; 1719 ops[i]->ldpc_dec.tb_params.r = 1720 ldpc_dec->tb_params.r; 1721 } else { 1722 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1723 } 1724 1725 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1726 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1727 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1728 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1729 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1730 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1731 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1732 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1733 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1734 1735 if (hard_outputs != NULL) 1736 ops[i]->ldpc_dec.hard_output = 1737 hard_outputs[start_idx + i]; 1738 if (inputs != NULL) 1739 ops[i]->ldpc_dec.input = 1740 inputs[start_idx + i]; 1741 if (soft_outputs != NULL) 1742 ops[i]->ldpc_dec.soft_output = 1743 soft_outputs[start_idx + i]; 1744 if (harq_inputs != NULL) 1745 ops[i]->ldpc_dec.harq_combined_input = 1746 harq_inputs[start_idx + i]; 1747 if (harq_outputs != NULL) 1748 ops[i]->ldpc_dec.harq_combined_output = 1749 harq_outputs[start_idx + i]; 1750 } 1751 } 1752 1753 1754 static void 1755 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1756 unsigned int start_idx, 1757 struct rte_bbdev_op_data *inputs, 1758 struct rte_bbdev_op_data *outputs, 1759 struct rte_bbdev_enc_op *ref_op) 1760 { 1761 unsigned int i; 1762 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1763 for (i = 0; i < n; ++i) { 1764 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1765 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1766 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1767 ops[i]->ldpc_enc.tb_params.cab = 1768 ldpc_enc->tb_params.cab; 1769 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1770 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1771 } else { 1772 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1773 } 1774 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1775 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1776 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1777 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1778 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1779 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1780 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1781 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1782 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1783 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1784 } 1785 } 1786 1787 static int 1788 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1789 unsigned int order_idx, const int expected_status) 1790 { 1791 int status = op->status; 1792 /* ignore parity mismatch false alarms for long iterations */ 1793 if (get_iter_max() >= 10) { 1794 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1795 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1796 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1797 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 1798 } 1799 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1800 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1801 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1802 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 1803 } 1804 } 1805 1806 TEST_ASSERT(status == expected_status, 1807 "op_status (%d) != expected_status (%d)", 1808 op->status, expected_status); 1809 1810 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1811 "Ordering error, expected %p, got %p", 1812 (void *)(uintptr_t)order_idx, op->opaque_data); 1813 1814 return TEST_SUCCESS; 1815 } 1816 1817 static int 1818 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1819 unsigned int order_idx, const int expected_status) 1820 { 1821 TEST_ASSERT(op->status == expected_status, 1822 "op_status (%d) != expected_status (%d)", 1823 op->status, expected_status); 1824 1825 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) 1826 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1827 "Ordering error, expected %p, got %p", 1828 (void *)(uintptr_t)order_idx, op->opaque_data); 1829 1830 return TEST_SUCCESS; 1831 } 1832 1833 static inline int 1834 validate_op_chain(struct rte_bbdev_op_data *op, 1835 struct op_data_entries *orig_op) 1836 { 1837 uint8_t i; 1838 struct rte_mbuf *m = op->data; 1839 uint8_t nb_dst_segments = orig_op->nb_segments; 1840 uint32_t total_data_size = 0; 1841 1842 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1843 "Number of segments differ in original (%u) and filled (%u) op", 1844 nb_dst_segments, m->nb_segs); 1845 1846 /* Validate each mbuf segment length */ 1847 for (i = 0; i < nb_dst_segments; ++i) { 1848 /* Apply offset to the first mbuf segment */ 1849 uint16_t offset = (i == 0) ? op->offset : 0; 1850 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1851 total_data_size += orig_op->segments[i].length; 1852 1853 TEST_ASSERT(orig_op->segments[i].length == data_len, 1854 "Length of segment differ in original (%u) and filled (%u) op", 1855 orig_op->segments[i].length, data_len); 1856 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1857 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1858 data_len, 1859 "Output buffers (CB=%u) are not equal", i); 1860 m = m->next; 1861 } 1862 1863 /* Validate total mbuf pkt length */ 1864 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1865 TEST_ASSERT(total_data_size == pkt_len, 1866 "Length of data differ in original (%u) and filled (%u) op", 1867 total_data_size, pkt_len); 1868 1869 return TEST_SUCCESS; 1870 } 1871 1872 /* 1873 * Compute K0 for a given configuration for HARQ output length computation 1874 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 1875 */ 1876 static inline uint16_t 1877 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 1878 { 1879 if (rv_index == 0) 1880 return 0; 1881 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1882 if (n_cb == n) { 1883 if (rv_index == 1) 1884 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 1885 else if (rv_index == 2) 1886 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 1887 else 1888 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 1889 } 1890 /* LBRM case - includes a division by N */ 1891 if (rv_index == 1) 1892 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 1893 / n) * z_c; 1894 else if (rv_index == 2) 1895 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 1896 / n) * z_c; 1897 else 1898 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 1899 / n) * z_c; 1900 } 1901 1902 /* HARQ output length including the Filler bits */ 1903 static inline uint16_t 1904 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 1905 { 1906 uint16_t k0 = 0; 1907 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 1908 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 1909 /* Compute RM out size and number of rows */ 1910 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1911 * ops_ld->z_c - ops_ld->n_filler; 1912 uint16_t deRmOutSize = RTE_MIN( 1913 k0 + ops_ld->cb_params.e + 1914 ((k0 > parity_offset) ? 1915 0 : ops_ld->n_filler), 1916 ops_ld->n_cb); 1917 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 1918 / ops_ld->z_c); 1919 uint16_t harq_output_len = numRows * ops_ld->z_c; 1920 return harq_output_len; 1921 } 1922 1923 static inline int 1924 validate_op_harq_chain(struct rte_bbdev_op_data *op, 1925 struct op_data_entries *orig_op, 1926 struct rte_bbdev_op_ldpc_dec *ops_ld) 1927 { 1928 uint8_t i; 1929 uint32_t j, jj, k; 1930 struct rte_mbuf *m = op->data; 1931 uint8_t nb_dst_segments = orig_op->nb_segments; 1932 uint32_t total_data_size = 0; 1933 int8_t *harq_orig, *harq_out, abs_harq_origin; 1934 uint32_t byte_error = 0, cum_error = 0, error; 1935 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 1936 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1937 uint16_t parity_offset; 1938 1939 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1940 "Number of segments differ in original (%u) and filled (%u) op", 1941 nb_dst_segments, m->nb_segs); 1942 1943 /* Validate each mbuf segment length */ 1944 for (i = 0; i < nb_dst_segments; ++i) { 1945 /* Apply offset to the first mbuf segment */ 1946 uint16_t offset = (i == 0) ? op->offset : 0; 1947 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1948 total_data_size += orig_op->segments[i].length; 1949 1950 TEST_ASSERT(orig_op->segments[i].length < 1951 (uint32_t)(data_len + 64), 1952 "Length of segment differ in original (%u) and filled (%u) op", 1953 orig_op->segments[i].length, data_len); 1954 harq_orig = (int8_t *) orig_op->segments[i].addr; 1955 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 1956 1957 if (!(ldpc_cap_flags & 1958 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 1959 ) || (ops_ld->op_flags & 1960 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1961 data_len -= ops_ld->z_c; 1962 parity_offset = data_len; 1963 } else { 1964 /* Compute RM out size and number of rows */ 1965 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1966 * ops_ld->z_c - ops_ld->n_filler; 1967 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 1968 ops_ld->n_filler; 1969 if (data_len > deRmOutSize) 1970 data_len = deRmOutSize; 1971 if (data_len > orig_op->segments[i].length) 1972 data_len = orig_op->segments[i].length; 1973 } 1974 /* 1975 * HARQ output can have minor differences 1976 * due to integer representation and related scaling 1977 */ 1978 for (j = 0, jj = 0; j < data_len; j++, jj++) { 1979 if (j == parity_offset) { 1980 /* Special Handling of the filler bits */ 1981 for (k = 0; k < ops_ld->n_filler; k++) { 1982 if (harq_out[jj] != 1983 llr_max_pre_scaling) { 1984 printf("HARQ Filler issue %d: %d %d\n", 1985 jj, harq_out[jj], 1986 llr_max); 1987 byte_error++; 1988 } 1989 jj++; 1990 } 1991 } 1992 if (!(ops_ld->op_flags & 1993 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1994 if (ldpc_llr_decimals > 1) 1995 harq_out[jj] = (harq_out[jj] + 1) 1996 >> (ldpc_llr_decimals - 1); 1997 /* Saturated to S7 */ 1998 if (harq_orig[j] > llr_max) 1999 harq_orig[j] = llr_max; 2000 if (harq_orig[j] < -llr_max) 2001 harq_orig[j] = -llr_max; 2002 } 2003 if (harq_orig[j] != harq_out[jj]) { 2004 error = (harq_orig[j] > harq_out[jj]) ? 2005 harq_orig[j] - harq_out[jj] : 2006 harq_out[jj] - harq_orig[j]; 2007 abs_harq_origin = harq_orig[j] > 0 ? 2008 harq_orig[j] : 2009 -harq_orig[j]; 2010 /* Residual quantization error */ 2011 if ((error > 8 && (abs_harq_origin < 2012 (llr_max - 16))) || 2013 (error > 16)) { 2014 printf("HARQ mismatch %d: exp %d act %d => %d\n", 2015 j, harq_orig[j], 2016 harq_out[jj], error); 2017 byte_error++; 2018 cum_error += error; 2019 } 2020 } 2021 } 2022 m = m->next; 2023 } 2024 2025 if (byte_error) 2026 TEST_ASSERT(byte_error <= 1, 2027 "HARQ output mismatch (%d) %d", 2028 byte_error, cum_error); 2029 2030 /* Validate total mbuf pkt length */ 2031 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 2032 TEST_ASSERT(total_data_size < pkt_len + 64, 2033 "Length of data differ in original (%u) and filled (%u) op", 2034 total_data_size, pkt_len); 2035 2036 return TEST_SUCCESS; 2037 } 2038 2039 static int 2040 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2041 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2042 { 2043 unsigned int i; 2044 int ret; 2045 struct op_data_entries *hard_data_orig = 2046 &test_vector.entries[DATA_HARD_OUTPUT]; 2047 struct op_data_entries *soft_data_orig = 2048 &test_vector.entries[DATA_SOFT_OUTPUT]; 2049 struct rte_bbdev_op_turbo_dec *ops_td; 2050 struct rte_bbdev_op_data *hard_output; 2051 struct rte_bbdev_op_data *soft_output; 2052 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 2053 2054 for (i = 0; i < n; ++i) { 2055 ops_td = &ops[i]->turbo_dec; 2056 hard_output = &ops_td->hard_output; 2057 soft_output = &ops_td->soft_output; 2058 2059 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2060 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2061 "Returned iter_count (%d) > expected iter_count (%d)", 2062 ops_td->iter_count, ref_td->iter_count); 2063 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2064 TEST_ASSERT_SUCCESS(ret, 2065 "Checking status and ordering for decoder failed"); 2066 2067 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2068 hard_data_orig), 2069 "Hard output buffers (CB=%u) are not equal", 2070 i); 2071 2072 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2073 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2074 soft_data_orig), 2075 "Soft output buffers (CB=%u) are not equal", 2076 i); 2077 } 2078 2079 return TEST_SUCCESS; 2080 } 2081 2082 /* Check Number of code blocks errors */ 2083 static int 2084 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2085 { 2086 unsigned int i; 2087 struct op_data_entries *hard_data_orig = 2088 &test_vector.entries[DATA_HARD_OUTPUT]; 2089 struct rte_bbdev_op_ldpc_dec *ops_td; 2090 struct rte_bbdev_op_data *hard_output; 2091 int errors = 0; 2092 struct rte_mbuf *m; 2093 2094 for (i = 0; i < n; ++i) { 2095 ops_td = &ops[i]->ldpc_dec; 2096 hard_output = &ops_td->hard_output; 2097 m = hard_output->data; 2098 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2099 hard_data_orig->segments[0].addr, 2100 hard_data_orig->segments[0].length)) 2101 errors++; 2102 } 2103 return errors; 2104 } 2105 2106 static int 2107 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2108 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2109 { 2110 unsigned int i; 2111 int ret; 2112 struct op_data_entries *hard_data_orig = 2113 &test_vector.entries[DATA_HARD_OUTPUT]; 2114 struct op_data_entries *soft_data_orig = 2115 &test_vector.entries[DATA_SOFT_OUTPUT]; 2116 struct op_data_entries *harq_data_orig = 2117 &test_vector.entries[DATA_HARQ_OUTPUT]; 2118 struct rte_bbdev_op_ldpc_dec *ops_td; 2119 struct rte_bbdev_op_data *hard_output; 2120 struct rte_bbdev_op_data *harq_output; 2121 struct rte_bbdev_op_data *soft_output; 2122 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2123 2124 for (i = 0; i < n; ++i) { 2125 ops_td = &ops[i]->ldpc_dec; 2126 hard_output = &ops_td->hard_output; 2127 harq_output = &ops_td->harq_combined_output; 2128 soft_output = &ops_td->soft_output; 2129 2130 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2131 TEST_ASSERT_SUCCESS(ret, 2132 "Checking status and ordering for decoder failed"); 2133 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2134 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2135 "Returned iter_count (%d) > expected iter_count (%d)", 2136 ops_td->iter_count, ref_td->iter_count); 2137 /* 2138 * We can ignore output data when the decoding failed to 2139 * converge or for loop-back cases 2140 */ 2141 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2142 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2143 ) && ( 2144 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2145 )) == 0) 2146 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2147 hard_data_orig), 2148 "Hard output buffers (CB=%u) are not equal", 2149 i); 2150 2151 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2152 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2153 soft_data_orig), 2154 "Soft output buffers (CB=%u) are not equal", 2155 i); 2156 if (ref_op->ldpc_dec.op_flags & 2157 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2158 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2159 harq_data_orig, ops_td), 2160 "HARQ output buffers (CB=%u) are not equal", 2161 i); 2162 } 2163 if (ref_op->ldpc_dec.op_flags & 2164 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2165 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2166 harq_data_orig, ops_td), 2167 "HARQ output buffers (CB=%u) are not equal", 2168 i); 2169 2170 } 2171 2172 return TEST_SUCCESS; 2173 } 2174 2175 2176 static int 2177 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2178 struct rte_bbdev_enc_op *ref_op) 2179 { 2180 unsigned int i; 2181 int ret; 2182 struct op_data_entries *hard_data_orig = 2183 &test_vector.entries[DATA_HARD_OUTPUT]; 2184 2185 for (i = 0; i < n; ++i) { 2186 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2187 TEST_ASSERT_SUCCESS(ret, 2188 "Checking status and ordering for encoder failed"); 2189 TEST_ASSERT_SUCCESS(validate_op_chain( 2190 &ops[i]->turbo_enc.output, 2191 hard_data_orig), 2192 "Output buffers (CB=%u) are not equal", 2193 i); 2194 } 2195 2196 return TEST_SUCCESS; 2197 } 2198 2199 static int 2200 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2201 struct rte_bbdev_enc_op *ref_op) 2202 { 2203 unsigned int i; 2204 int ret; 2205 struct op_data_entries *hard_data_orig = 2206 &test_vector.entries[DATA_HARD_OUTPUT]; 2207 2208 for (i = 0; i < n; ++i) { 2209 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2210 TEST_ASSERT_SUCCESS(ret, 2211 "Checking status and ordering for encoder failed"); 2212 TEST_ASSERT_SUCCESS(validate_op_chain( 2213 &ops[i]->ldpc_enc.output, 2214 hard_data_orig), 2215 "Output buffers (CB=%u) are not equal", 2216 i); 2217 } 2218 2219 return TEST_SUCCESS; 2220 } 2221 2222 static void 2223 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2224 { 2225 unsigned int i; 2226 struct op_data_entries *entry; 2227 2228 op->turbo_dec = test_vector.turbo_dec; 2229 entry = &test_vector.entries[DATA_INPUT]; 2230 for (i = 0; i < entry->nb_segments; ++i) 2231 op->turbo_dec.input.length += 2232 entry->segments[i].length; 2233 } 2234 2235 static void 2236 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2237 { 2238 unsigned int i; 2239 struct op_data_entries *entry; 2240 2241 op->ldpc_dec = test_vector.ldpc_dec; 2242 entry = &test_vector.entries[DATA_INPUT]; 2243 for (i = 0; i < entry->nb_segments; ++i) 2244 op->ldpc_dec.input.length += 2245 entry->segments[i].length; 2246 if (test_vector.ldpc_dec.op_flags & 2247 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2248 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2249 for (i = 0; i < entry->nb_segments; ++i) 2250 op->ldpc_dec.harq_combined_input.length += 2251 entry->segments[i].length; 2252 } 2253 } 2254 2255 2256 static void 2257 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2258 { 2259 unsigned int i; 2260 struct op_data_entries *entry; 2261 2262 op->turbo_enc = test_vector.turbo_enc; 2263 entry = &test_vector.entries[DATA_INPUT]; 2264 for (i = 0; i < entry->nb_segments; ++i) 2265 op->turbo_enc.input.length += 2266 entry->segments[i].length; 2267 } 2268 2269 static void 2270 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2271 { 2272 unsigned int i; 2273 struct op_data_entries *entry; 2274 2275 op->ldpc_enc = test_vector.ldpc_enc; 2276 entry = &test_vector.entries[DATA_INPUT]; 2277 for (i = 0; i < entry->nb_segments; ++i) 2278 op->ldpc_enc.input.length += 2279 entry->segments[i].length; 2280 } 2281 2282 static uint32_t 2283 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2284 { 2285 uint8_t i; 2286 uint32_t c, r, tb_size = 0; 2287 2288 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2289 tb_size = op->turbo_dec.tb_params.k_neg; 2290 } else { 2291 c = op->turbo_dec.tb_params.c; 2292 r = op->turbo_dec.tb_params.r; 2293 for (i = 0; i < c-r; i++) 2294 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2295 op->turbo_dec.tb_params.k_neg : 2296 op->turbo_dec.tb_params.k_pos; 2297 } 2298 return tb_size; 2299 } 2300 2301 static uint32_t 2302 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2303 { 2304 uint8_t i; 2305 uint32_t c, r, tb_size = 0; 2306 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2307 2308 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2309 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 2310 } else { 2311 c = op->ldpc_dec.tb_params.c; 2312 r = op->ldpc_dec.tb_params.r; 2313 for (i = 0; i < c-r; i++) 2314 tb_size += sys_cols * op->ldpc_dec.z_c 2315 - op->ldpc_dec.n_filler; 2316 } 2317 return tb_size; 2318 } 2319 2320 static uint32_t 2321 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2322 { 2323 uint8_t i; 2324 uint32_t c, r, tb_size = 0; 2325 2326 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2327 tb_size = op->turbo_enc.tb_params.k_neg; 2328 } else { 2329 c = op->turbo_enc.tb_params.c; 2330 r = op->turbo_enc.tb_params.r; 2331 for (i = 0; i < c-r; i++) 2332 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2333 op->turbo_enc.tb_params.k_neg : 2334 op->turbo_enc.tb_params.k_pos; 2335 } 2336 return tb_size; 2337 } 2338 2339 static uint32_t 2340 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2341 { 2342 uint8_t i; 2343 uint32_t c, r, tb_size = 0; 2344 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2345 2346 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) { 2347 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 2348 } else { 2349 c = op->turbo_enc.tb_params.c; 2350 r = op->turbo_enc.tb_params.r; 2351 for (i = 0; i < c-r; i++) 2352 tb_size += sys_cols * op->ldpc_enc.z_c 2353 - op->ldpc_enc.n_filler; 2354 } 2355 return tb_size; 2356 } 2357 2358 2359 static int 2360 init_test_op_params(struct test_op_params *op_params, 2361 enum rte_bbdev_op_type op_type, const int expected_status, 2362 const int vector_mask, struct rte_mempool *ops_mp, 2363 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2364 { 2365 int ret = 0; 2366 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2367 op_type == RTE_BBDEV_OP_LDPC_DEC) 2368 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2369 &op_params->ref_dec_op, 1); 2370 else 2371 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2372 &op_params->ref_enc_op, 1); 2373 2374 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2375 2376 op_params->mp = ops_mp; 2377 op_params->burst_sz = burst_sz; 2378 op_params->num_to_process = num_to_process; 2379 op_params->num_lcores = num_lcores; 2380 op_params->vector_mask = vector_mask; 2381 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2382 op_type == RTE_BBDEV_OP_LDPC_DEC) 2383 op_params->ref_dec_op->status = expected_status; 2384 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2385 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2386 op_params->ref_enc_op->status = expected_status; 2387 return 0; 2388 } 2389 2390 static int 2391 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2392 struct test_op_params *op_params) 2393 { 2394 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2395 unsigned int i; 2396 struct active_device *ad; 2397 unsigned int burst_sz = get_burst_sz(); 2398 enum rte_bbdev_op_type op_type = test_vector.op_type; 2399 const struct rte_bbdev_op_cap *capabilities = NULL; 2400 2401 ad = &active_devs[dev_id]; 2402 2403 /* Check if device supports op_type */ 2404 if (!is_avail_op(ad, test_vector.op_type)) 2405 return TEST_SUCCESS; 2406 2407 struct rte_bbdev_info info; 2408 rte_bbdev_info_get(ad->dev_id, &info); 2409 socket_id = GET_SOCKET(info.socket_id); 2410 2411 f_ret = create_mempools(ad, socket_id, op_type, 2412 get_num_ops()); 2413 if (f_ret != TEST_SUCCESS) { 2414 printf("Couldn't create mempools"); 2415 goto fail; 2416 } 2417 if (op_type == RTE_BBDEV_OP_NONE) 2418 op_type = RTE_BBDEV_OP_TURBO_ENC; 2419 2420 f_ret = init_test_op_params(op_params, test_vector.op_type, 2421 test_vector.expected_status, 2422 test_vector.mask, 2423 ad->ops_mempool, 2424 burst_sz, 2425 get_num_ops(), 2426 get_num_lcores()); 2427 if (f_ret != TEST_SUCCESS) { 2428 printf("Couldn't init test op params"); 2429 goto fail; 2430 } 2431 2432 2433 /* Find capabilities */ 2434 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2435 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 2436 if (cap->type == test_vector.op_type) { 2437 capabilities = cap; 2438 break; 2439 } 2440 cap++; 2441 } 2442 TEST_ASSERT_NOT_NULL(capabilities, 2443 "Couldn't find capabilities"); 2444 2445 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2446 create_reference_dec_op(op_params->ref_dec_op); 2447 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2448 create_reference_enc_op(op_params->ref_enc_op); 2449 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2450 create_reference_ldpc_enc_op(op_params->ref_enc_op); 2451 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2452 create_reference_ldpc_dec_op(op_params->ref_dec_op); 2453 2454 for (i = 0; i < ad->nb_queues; ++i) { 2455 f_ret = fill_queue_buffers(op_params, 2456 ad->in_mbuf_pool, 2457 ad->hard_out_mbuf_pool, 2458 ad->soft_out_mbuf_pool, 2459 ad->harq_in_mbuf_pool, 2460 ad->harq_out_mbuf_pool, 2461 ad->queue_ids[i], 2462 capabilities, 2463 info.drv.min_alignment, 2464 socket_id); 2465 if (f_ret != TEST_SUCCESS) { 2466 printf("Couldn't init queue buffers"); 2467 goto fail; 2468 } 2469 } 2470 2471 /* Run test case function */ 2472 t_ret = test_case_func(ad, op_params); 2473 2474 /* Free active device resources and return */ 2475 free_buffers(ad, op_params); 2476 return t_ret; 2477 2478 fail: 2479 free_buffers(ad, op_params); 2480 return TEST_FAILED; 2481 } 2482 2483 /* Run given test function per active device per supported op type 2484 * per burst size. 2485 */ 2486 static int 2487 run_test_case(test_case_function *test_case_func) 2488 { 2489 int ret = 0; 2490 uint8_t dev; 2491 2492 /* Alloc op_params */ 2493 struct test_op_params *op_params = rte_zmalloc(NULL, 2494 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 2495 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 2496 RTE_ALIGN(sizeof(struct test_op_params), 2497 RTE_CACHE_LINE_SIZE)); 2498 2499 /* For each device run test case function */ 2500 for (dev = 0; dev < nb_active_devs; ++dev) 2501 ret |= run_test_case_on_device(test_case_func, dev, op_params); 2502 2503 rte_free(op_params); 2504 2505 return ret; 2506 } 2507 2508 2509 /* Push back the HARQ output from DDR to host */ 2510 static void 2511 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2512 struct rte_bbdev_dec_op **ops, 2513 const uint16_t n) 2514 { 2515 uint16_t j; 2516 int save_status, ret; 2517 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2518 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 2519 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2520 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 2521 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2522 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2523 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2524 for (j = 0; j < n; ++j) { 2525 if ((loopback && mem_out) || hc_out) { 2526 save_status = ops[j]->status; 2527 ops[j]->ldpc_dec.op_flags = 2528 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2529 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2530 if (h_comp) 2531 ops[j]->ldpc_dec.op_flags += 2532 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2533 ops[j]->ldpc_dec.harq_combined_input.offset = 2534 harq_offset; 2535 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 2536 harq_offset += HARQ_INCR; 2537 if (!loopback) 2538 ops[j]->ldpc_dec.harq_combined_input.length = 2539 ops[j]->ldpc_dec.harq_combined_output.length; 2540 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2541 &ops[j], 1); 2542 ret = 0; 2543 while (ret == 0) 2544 ret = rte_bbdev_dequeue_ldpc_dec_ops( 2545 dev_id, queue_id, 2546 &ops_deq[j], 1); 2547 ops[j]->ldpc_dec.op_flags = flags; 2548 ops[j]->status = save_status; 2549 } 2550 } 2551 } 2552 2553 /* 2554 * Push back the HARQ output from HW DDR to Host 2555 * Preload HARQ memory input and adjust HARQ offset 2556 */ 2557 static void 2558 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2559 struct rte_bbdev_dec_op **ops, const uint16_t n, 2560 bool preload) 2561 { 2562 uint16_t j; 2563 int deq; 2564 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2565 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS]; 2566 struct rte_bbdev_dec_op *ops_deq[MAX_OPS]; 2567 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2568 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2569 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 2570 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2571 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2572 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2573 if ((mem_in || hc_in) && preload) { 2574 for (j = 0; j < n; ++j) { 2575 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input; 2576 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output; 2577 ops[j]->ldpc_dec.op_flags = 2578 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2579 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2580 if (h_comp) 2581 ops[j]->ldpc_dec.op_flags += 2582 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2583 ops[j]->ldpc_dec.harq_combined_output.offset = 2584 harq_offset; 2585 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 2586 harq_offset += HARQ_INCR; 2587 } 2588 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n); 2589 deq = 0; 2590 while (deq != n) 2591 deq += rte_bbdev_dequeue_ldpc_dec_ops( 2592 dev_id, queue_id, &ops_deq[deq], 2593 n - deq); 2594 /* Restore the operations */ 2595 for (j = 0; j < n; ++j) { 2596 ops[j]->ldpc_dec.op_flags = flags; 2597 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j]; 2598 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j]; 2599 } 2600 } 2601 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS; 2602 for (j = 0; j < n; ++j) { 2603 /* Adjust HARQ offset when we reach external DDR */ 2604 if (mem_in || hc_in) 2605 ops[j]->ldpc_dec.harq_combined_input.offset 2606 = harq_offset; 2607 if (mem_out || hc_out) 2608 ops[j]->ldpc_dec.harq_combined_output.offset 2609 = harq_offset; 2610 harq_offset += HARQ_INCR; 2611 } 2612 } 2613 2614 static void 2615 dequeue_event_callback(uint16_t dev_id, 2616 enum rte_bbdev_event_type event, void *cb_arg, 2617 void *ret_param) 2618 { 2619 int ret; 2620 uint16_t i; 2621 uint64_t total_time; 2622 uint16_t deq, burst_sz, num_ops; 2623 uint16_t queue_id = *(uint16_t *) ret_param; 2624 struct rte_bbdev_info info; 2625 double tb_len_bits; 2626 struct thread_params *tp = cb_arg; 2627 2628 /* Find matching thread params using queue_id */ 2629 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 2630 if (tp->queue_id == queue_id) 2631 break; 2632 2633 if (i == MAX_QUEUES) { 2634 printf("%s: Queue_id from interrupt details was not found!\n", 2635 __func__); 2636 return; 2637 } 2638 2639 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 2640 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2641 printf( 2642 "Dequeue interrupt handler called for incorrect event!\n"); 2643 return; 2644 } 2645 2646 burst_sz = __atomic_load_n(&tp->burst_sz, __ATOMIC_RELAXED); 2647 num_ops = tp->op_params->num_to_process; 2648 2649 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2650 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2651 &tp->dec_ops[ 2652 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2653 burst_sz); 2654 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2655 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2656 &tp->dec_ops[ 2657 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2658 burst_sz); 2659 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2660 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2661 &tp->enc_ops[ 2662 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2663 burst_sz); 2664 else /*RTE_BBDEV_OP_TURBO_ENC*/ 2665 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2666 &tp->enc_ops[ 2667 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)], 2668 burst_sz); 2669 2670 if (deq < burst_sz) { 2671 printf( 2672 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 2673 burst_sz, deq); 2674 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2675 return; 2676 } 2677 2678 if (__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) + deq < num_ops) { 2679 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 2680 return; 2681 } 2682 2683 total_time = rte_rdtsc_precise() - tp->start_time; 2684 2685 rte_bbdev_info_get(dev_id, &info); 2686 2687 ret = TEST_SUCCESS; 2688 2689 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2690 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2691 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 2692 tp->op_params->vector_mask); 2693 /* get the max of iter_count for all dequeued ops */ 2694 for (i = 0; i < num_ops; ++i) 2695 tp->iter_count = RTE_MAX( 2696 tp->dec_ops[i]->turbo_dec.iter_count, 2697 tp->iter_count); 2698 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2699 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 2700 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2701 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 2702 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2703 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 2704 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2705 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 2706 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2707 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 2708 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2709 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 2710 tp->op_params->vector_mask); 2711 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2712 } 2713 2714 if (ret) { 2715 printf("Buffers validation failed\n"); 2716 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2717 } 2718 2719 switch (test_vector.op_type) { 2720 case RTE_BBDEV_OP_TURBO_DEC: 2721 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 2722 break; 2723 case RTE_BBDEV_OP_TURBO_ENC: 2724 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 2725 break; 2726 case RTE_BBDEV_OP_LDPC_DEC: 2727 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 2728 break; 2729 case RTE_BBDEV_OP_LDPC_ENC: 2730 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 2731 break; 2732 case RTE_BBDEV_OP_NONE: 2733 tb_len_bits = 0.0; 2734 break; 2735 default: 2736 printf("Unknown op type: %d\n", test_vector.op_type); 2737 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED); 2738 return; 2739 } 2740 2741 tp->ops_per_sec += ((double)num_ops) / 2742 ((double)total_time / (double)rte_get_tsc_hz()); 2743 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 2744 ((double)total_time / (double)rte_get_tsc_hz()); 2745 2746 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED); 2747 } 2748 2749 static int 2750 throughput_intr_lcore_ldpc_dec(void *arg) 2751 { 2752 struct thread_params *tp = arg; 2753 unsigned int enqueued; 2754 const uint16_t queue_id = tp->queue_id; 2755 const uint16_t burst_sz = tp->op_params->burst_sz; 2756 const uint16_t num_to_process = tp->op_params->num_to_process; 2757 struct rte_bbdev_dec_op *ops[num_to_process]; 2758 struct test_buffers *bufs = NULL; 2759 struct rte_bbdev_info info; 2760 int ret, i, j; 2761 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2762 uint16_t num_to_enq, enq; 2763 2764 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 2765 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 2766 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 2767 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 2768 2769 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2770 "BURST_SIZE should be <= %u", MAX_BURST); 2771 2772 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2773 "Failed to enable interrupts for dev: %u, queue_id: %u", 2774 tp->dev_id, queue_id); 2775 2776 rte_bbdev_info_get(tp->dev_id, &info); 2777 2778 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2779 "NUM_OPS cannot exceed %u for this device", 2780 info.drv.queue_size_lim); 2781 2782 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2783 2784 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 2785 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 2786 2787 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 2788 2789 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2790 num_to_process); 2791 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2792 num_to_process); 2793 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2794 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 2795 bufs->hard_outputs, bufs->soft_outputs, 2796 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2797 2798 /* Set counter to validate the ordering */ 2799 for (j = 0; j < num_to_process; ++j) 2800 ops[j]->opaque_data = (void *)(uintptr_t)j; 2801 2802 for (j = 0; j < TEST_REPETITIONS; ++j) { 2803 for (i = 0; i < num_to_process; ++i) { 2804 if (!loopback) 2805 rte_pktmbuf_reset( 2806 ops[i]->ldpc_dec.hard_output.data); 2807 if (hc_out || loopback) 2808 mbuf_reset( 2809 ops[i]->ldpc_dec.harq_combined_output.data); 2810 } 2811 2812 tp->start_time = rte_rdtsc_precise(); 2813 for (enqueued = 0; enqueued < num_to_process;) { 2814 num_to_enq = burst_sz; 2815 2816 if (unlikely(num_to_process - enqueued < num_to_enq)) 2817 num_to_enq = num_to_process - enqueued; 2818 2819 enq = 0; 2820 do { 2821 enq += rte_bbdev_enqueue_ldpc_dec_ops( 2822 tp->dev_id, 2823 queue_id, &ops[enqueued], 2824 num_to_enq); 2825 } while (unlikely(num_to_enq != enq)); 2826 enqueued += enq; 2827 2828 /* Write to thread burst_sz current number of enqueued 2829 * descriptors. It ensures that proper number of 2830 * descriptors will be dequeued in callback 2831 * function - needed for last batch in case where 2832 * the number of operations is not a multiple of 2833 * burst size. 2834 */ 2835 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 2836 2837 /* Wait until processing of previous batch is 2838 * completed 2839 */ 2840 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 2841 } 2842 if (j != TEST_REPETITIONS - 1) 2843 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 2844 } 2845 2846 return TEST_SUCCESS; 2847 } 2848 2849 static int 2850 throughput_intr_lcore_dec(void *arg) 2851 { 2852 struct thread_params *tp = arg; 2853 unsigned int enqueued; 2854 const uint16_t queue_id = tp->queue_id; 2855 const uint16_t burst_sz = tp->op_params->burst_sz; 2856 const uint16_t num_to_process = tp->op_params->num_to_process; 2857 struct rte_bbdev_dec_op *ops[num_to_process]; 2858 struct test_buffers *bufs = NULL; 2859 struct rte_bbdev_info info; 2860 int ret, i, j; 2861 uint16_t num_to_enq, enq; 2862 2863 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2864 "BURST_SIZE should be <= %u", MAX_BURST); 2865 2866 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2867 "Failed to enable interrupts for dev: %u, queue_id: %u", 2868 tp->dev_id, queue_id); 2869 2870 rte_bbdev_info_get(tp->dev_id, &info); 2871 2872 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2873 "NUM_OPS cannot exceed %u for this device", 2874 info.drv.queue_size_lim); 2875 2876 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2877 2878 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 2879 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 2880 2881 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 2882 2883 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2884 num_to_process); 2885 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2886 num_to_process); 2887 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2888 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 2889 bufs->hard_outputs, bufs->soft_outputs, 2890 tp->op_params->ref_dec_op); 2891 2892 /* Set counter to validate the ordering */ 2893 for (j = 0; j < num_to_process; ++j) 2894 ops[j]->opaque_data = (void *)(uintptr_t)j; 2895 2896 for (j = 0; j < TEST_REPETITIONS; ++j) { 2897 for (i = 0; i < num_to_process; ++i) 2898 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 2899 2900 tp->start_time = rte_rdtsc_precise(); 2901 for (enqueued = 0; enqueued < num_to_process;) { 2902 num_to_enq = burst_sz; 2903 2904 if (unlikely(num_to_process - enqueued < num_to_enq)) 2905 num_to_enq = num_to_process - enqueued; 2906 2907 enq = 0; 2908 do { 2909 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2910 queue_id, &ops[enqueued], 2911 num_to_enq); 2912 } while (unlikely(num_to_enq != enq)); 2913 enqueued += enq; 2914 2915 /* Write to thread burst_sz current number of enqueued 2916 * descriptors. It ensures that proper number of 2917 * descriptors will be dequeued in callback 2918 * function - needed for last batch in case where 2919 * the number of operations is not a multiple of 2920 * burst size. 2921 */ 2922 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 2923 2924 /* Wait until processing of previous batch is 2925 * completed 2926 */ 2927 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 2928 } 2929 if (j != TEST_REPETITIONS - 1) 2930 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 2931 } 2932 2933 return TEST_SUCCESS; 2934 } 2935 2936 static int 2937 throughput_intr_lcore_enc(void *arg) 2938 { 2939 struct thread_params *tp = arg; 2940 unsigned int enqueued; 2941 const uint16_t queue_id = tp->queue_id; 2942 const uint16_t burst_sz = tp->op_params->burst_sz; 2943 const uint16_t num_to_process = tp->op_params->num_to_process; 2944 struct rte_bbdev_enc_op *ops[num_to_process]; 2945 struct test_buffers *bufs = NULL; 2946 struct rte_bbdev_info info; 2947 int ret, i, j; 2948 uint16_t num_to_enq, enq; 2949 2950 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2951 "BURST_SIZE should be <= %u", MAX_BURST); 2952 2953 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2954 "Failed to enable interrupts for dev: %u, queue_id: %u", 2955 tp->dev_id, queue_id); 2956 2957 rte_bbdev_info_get(tp->dev_id, &info); 2958 2959 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2960 "NUM_OPS cannot exceed %u for this device", 2961 info.drv.queue_size_lim); 2962 2963 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2964 2965 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 2966 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 2967 2968 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 2969 2970 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 2971 num_to_process); 2972 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2973 num_to_process); 2974 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2975 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 2976 bufs->hard_outputs, tp->op_params->ref_enc_op); 2977 2978 /* Set counter to validate the ordering */ 2979 for (j = 0; j < num_to_process; ++j) 2980 ops[j]->opaque_data = (void *)(uintptr_t)j; 2981 2982 for (j = 0; j < TEST_REPETITIONS; ++j) { 2983 for (i = 0; i < num_to_process; ++i) 2984 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 2985 2986 tp->start_time = rte_rdtsc_precise(); 2987 for (enqueued = 0; enqueued < num_to_process;) { 2988 num_to_enq = burst_sz; 2989 2990 if (unlikely(num_to_process - enqueued < num_to_enq)) 2991 num_to_enq = num_to_process - enqueued; 2992 2993 enq = 0; 2994 do { 2995 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2996 queue_id, &ops[enqueued], 2997 num_to_enq); 2998 } while (unlikely(enq != num_to_enq)); 2999 enqueued += enq; 3000 3001 /* Write to thread burst_sz current number of enqueued 3002 * descriptors. It ensures that proper number of 3003 * descriptors will be dequeued in callback 3004 * function - needed for last batch in case where 3005 * the number of operations is not a multiple of 3006 * burst size. 3007 */ 3008 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3009 3010 /* Wait until processing of previous batch is 3011 * completed 3012 */ 3013 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3014 } 3015 if (j != TEST_REPETITIONS - 1) 3016 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3017 } 3018 3019 return TEST_SUCCESS; 3020 } 3021 3022 3023 static int 3024 throughput_intr_lcore_ldpc_enc(void *arg) 3025 { 3026 struct thread_params *tp = arg; 3027 unsigned int enqueued; 3028 const uint16_t queue_id = tp->queue_id; 3029 const uint16_t burst_sz = tp->op_params->burst_sz; 3030 const uint16_t num_to_process = tp->op_params->num_to_process; 3031 struct rte_bbdev_enc_op *ops[num_to_process]; 3032 struct test_buffers *bufs = NULL; 3033 struct rte_bbdev_info info; 3034 int ret, i, j; 3035 uint16_t num_to_enq, enq; 3036 3037 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3038 "BURST_SIZE should be <= %u", MAX_BURST); 3039 3040 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 3041 "Failed to enable interrupts for dev: %u, queue_id: %u", 3042 tp->dev_id, queue_id); 3043 3044 rte_bbdev_info_get(tp->dev_id, &info); 3045 3046 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3047 "NUM_OPS cannot exceed %u for this device", 3048 info.drv.queue_size_lim); 3049 3050 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3051 3052 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED); 3053 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3054 3055 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3056 3057 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3058 num_to_process); 3059 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3060 num_to_process); 3061 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3062 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3063 bufs->inputs, bufs->hard_outputs, 3064 tp->op_params->ref_enc_op); 3065 3066 /* Set counter to validate the ordering */ 3067 for (j = 0; j < num_to_process; ++j) 3068 ops[j]->opaque_data = (void *)(uintptr_t)j; 3069 3070 for (j = 0; j < TEST_REPETITIONS; ++j) { 3071 for (i = 0; i < num_to_process; ++i) 3072 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 3073 3074 tp->start_time = rte_rdtsc_precise(); 3075 for (enqueued = 0; enqueued < num_to_process;) { 3076 num_to_enq = burst_sz; 3077 3078 if (unlikely(num_to_process - enqueued < num_to_enq)) 3079 num_to_enq = num_to_process - enqueued; 3080 3081 enq = 0; 3082 do { 3083 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3084 tp->dev_id, 3085 queue_id, &ops[enqueued], 3086 num_to_enq); 3087 } while (unlikely(enq != num_to_enq)); 3088 enqueued += enq; 3089 3090 /* Write to thread burst_sz current number of enqueued 3091 * descriptors. It ensures that proper number of 3092 * descriptors will be dequeued in callback 3093 * function - needed for last batch in case where 3094 * the number of operations is not a multiple of 3095 * burst size. 3096 */ 3097 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); 3098 3099 /* Wait until processing of previous batch is 3100 * completed 3101 */ 3102 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED); 3103 } 3104 if (j != TEST_REPETITIONS - 1) 3105 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED); 3106 } 3107 3108 return TEST_SUCCESS; 3109 } 3110 3111 static int 3112 throughput_pmd_lcore_dec(void *arg) 3113 { 3114 struct thread_params *tp = arg; 3115 uint16_t enq, deq; 3116 uint64_t total_time = 0, start_time; 3117 const uint16_t queue_id = tp->queue_id; 3118 const uint16_t burst_sz = tp->op_params->burst_sz; 3119 const uint16_t num_ops = tp->op_params->num_to_process; 3120 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3121 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3122 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3123 struct test_buffers *bufs = NULL; 3124 int i, j, ret; 3125 struct rte_bbdev_info info; 3126 uint16_t num_to_enq; 3127 3128 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3129 "BURST_SIZE should be <= %u", MAX_BURST); 3130 3131 rte_bbdev_info_get(tp->dev_id, &info); 3132 3133 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3134 "NUM_OPS cannot exceed %u for this device", 3135 info.drv.queue_size_lim); 3136 3137 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3138 3139 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3140 3141 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3142 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3143 3144 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3145 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3146 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3147 3148 /* Set counter to validate the ordering */ 3149 for (j = 0; j < num_ops; ++j) 3150 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3151 3152 for (i = 0; i < TEST_REPETITIONS; ++i) { 3153 3154 for (j = 0; j < num_ops; ++j) 3155 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3156 3157 start_time = rte_rdtsc_precise(); 3158 3159 for (enq = 0, deq = 0; enq < num_ops;) { 3160 num_to_enq = burst_sz; 3161 3162 if (unlikely(num_ops - enq < num_to_enq)) 3163 num_to_enq = num_ops - enq; 3164 3165 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3166 queue_id, &ops_enq[enq], num_to_enq); 3167 3168 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3169 queue_id, &ops_deq[deq], enq - deq); 3170 } 3171 3172 /* dequeue the remaining */ 3173 while (deq < enq) { 3174 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3175 queue_id, &ops_deq[deq], enq - deq); 3176 } 3177 3178 total_time += rte_rdtsc_precise() - start_time; 3179 } 3180 3181 tp->iter_count = 0; 3182 /* get the max of iter_count for all dequeued ops */ 3183 for (i = 0; i < num_ops; ++i) { 3184 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3185 tp->iter_count); 3186 } 3187 3188 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3189 ret = validate_dec_op(ops_deq, num_ops, ref_op, 3190 tp->op_params->vector_mask); 3191 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3192 } 3193 3194 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3195 3196 double tb_len_bits = calc_dec_TB_size(ref_op); 3197 3198 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3199 ((double)total_time / (double)rte_get_tsc_hz()); 3200 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3201 1000000.0) / ((double)total_time / 3202 (double)rte_get_tsc_hz()); 3203 3204 return TEST_SUCCESS; 3205 } 3206 3207 static int 3208 bler_pmd_lcore_ldpc_dec(void *arg) 3209 { 3210 struct thread_params *tp = arg; 3211 uint16_t enq, deq; 3212 uint64_t total_time = 0, start_time; 3213 const uint16_t queue_id = tp->queue_id; 3214 const uint16_t burst_sz = tp->op_params->burst_sz; 3215 const uint16_t num_ops = tp->op_params->num_to_process; 3216 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3217 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3218 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3219 struct test_buffers *bufs = NULL; 3220 int i, j, ret; 3221 float parity_bler = 0; 3222 struct rte_bbdev_info info; 3223 uint16_t num_to_enq; 3224 bool extDdr = check_bit(ldpc_cap_flags, 3225 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3226 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3227 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3228 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3229 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3230 3231 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3232 "BURST_SIZE should be <= %u", MAX_BURST); 3233 3234 rte_bbdev_info_get(tp->dev_id, &info); 3235 3236 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3237 "NUM_OPS cannot exceed %u for this device", 3238 info.drv.queue_size_lim); 3239 3240 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3241 3242 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3243 3244 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3245 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3246 3247 /* For BLER tests we need to enable early termination */ 3248 if (!check_bit(ref_op->ldpc_dec.op_flags, 3249 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3250 ref_op->ldpc_dec.op_flags += 3251 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3252 ref_op->ldpc_dec.iter_max = get_iter_max(); 3253 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3254 3255 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3256 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3257 bufs->hard_outputs, bufs->soft_outputs, 3258 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3259 generate_llr_input(num_ops, bufs->inputs, ref_op); 3260 3261 /* Set counter to validate the ordering */ 3262 for (j = 0; j < num_ops; ++j) 3263 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3264 3265 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 3266 for (j = 0; j < num_ops; ++j) { 3267 if (!loopback) 3268 mbuf_reset( 3269 ops_enq[j]->ldpc_dec.hard_output.data); 3270 if (hc_out || loopback) 3271 mbuf_reset( 3272 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3273 } 3274 if (extDdr) 3275 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3276 num_ops, true); 3277 start_time = rte_rdtsc_precise(); 3278 3279 for (enq = 0, deq = 0; enq < num_ops;) { 3280 num_to_enq = burst_sz; 3281 3282 if (unlikely(num_ops - enq < num_to_enq)) 3283 num_to_enq = num_ops - enq; 3284 3285 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3286 queue_id, &ops_enq[enq], num_to_enq); 3287 3288 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3289 queue_id, &ops_deq[deq], enq - deq); 3290 } 3291 3292 /* dequeue the remaining */ 3293 while (deq < enq) { 3294 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3295 queue_id, &ops_deq[deq], enq - deq); 3296 } 3297 3298 total_time += rte_rdtsc_precise() - start_time; 3299 } 3300 3301 tp->iter_count = 0; 3302 tp->iter_average = 0; 3303 /* get the max of iter_count for all dequeued ops */ 3304 for (i = 0; i < num_ops; ++i) { 3305 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3306 tp->iter_count); 3307 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 3308 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 3309 parity_bler += 1.0; 3310 } 3311 3312 parity_bler /= num_ops; /* This one is based on SYND */ 3313 tp->iter_average /= num_ops; 3314 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 3315 3316 if (test_vector.op_type != RTE_BBDEV_OP_NONE 3317 && tp->bler == 0 3318 && parity_bler == 0 3319 && !hc_out) { 3320 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3321 tp->op_params->vector_mask); 3322 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3323 } 3324 3325 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3326 3327 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3328 tp->ops_per_sec = ((double)num_ops * 1) / 3329 ((double)total_time / (double)rte_get_tsc_hz()); 3330 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 3331 1000000.0) / ((double)total_time / 3332 (double)rte_get_tsc_hz()); 3333 3334 return TEST_SUCCESS; 3335 } 3336 3337 static int 3338 throughput_pmd_lcore_ldpc_dec(void *arg) 3339 { 3340 struct thread_params *tp = arg; 3341 uint16_t enq, deq; 3342 uint64_t total_time = 0, start_time; 3343 const uint16_t queue_id = tp->queue_id; 3344 const uint16_t burst_sz = tp->op_params->burst_sz; 3345 const uint16_t num_ops = tp->op_params->num_to_process; 3346 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3347 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3348 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3349 struct test_buffers *bufs = NULL; 3350 int i, j, ret; 3351 struct rte_bbdev_info info; 3352 uint16_t num_to_enq; 3353 bool extDdr = check_bit(ldpc_cap_flags, 3354 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3355 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3356 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3357 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3358 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3359 3360 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3361 "BURST_SIZE should be <= %u", MAX_BURST); 3362 3363 rte_bbdev_info_get(tp->dev_id, &info); 3364 3365 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3366 "NUM_OPS cannot exceed %u for this device", 3367 info.drv.queue_size_lim); 3368 3369 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3370 3371 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3372 3373 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3374 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3375 3376 /* For throughput tests we need to disable early termination */ 3377 if (check_bit(ref_op->ldpc_dec.op_flags, 3378 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3379 ref_op->ldpc_dec.op_flags -= 3380 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3381 ref_op->ldpc_dec.iter_max = get_iter_max(); 3382 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3383 3384 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3385 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3386 bufs->hard_outputs, bufs->soft_outputs, 3387 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3388 3389 /* Set counter to validate the ordering */ 3390 for (j = 0; j < num_ops; ++j) 3391 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3392 3393 for (i = 0; i < TEST_REPETITIONS; ++i) { 3394 for (j = 0; j < num_ops; ++j) { 3395 if (!loopback) 3396 mbuf_reset( 3397 ops_enq[j]->ldpc_dec.hard_output.data); 3398 if (hc_out || loopback) 3399 mbuf_reset( 3400 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3401 } 3402 if (extDdr) 3403 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3404 num_ops, true); 3405 start_time = rte_rdtsc_precise(); 3406 3407 for (enq = 0, deq = 0; enq < num_ops;) { 3408 num_to_enq = burst_sz; 3409 3410 if (unlikely(num_ops - enq < num_to_enq)) 3411 num_to_enq = num_ops - enq; 3412 3413 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3414 queue_id, &ops_enq[enq], num_to_enq); 3415 3416 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3417 queue_id, &ops_deq[deq], enq - deq); 3418 } 3419 3420 /* dequeue the remaining */ 3421 while (deq < enq) { 3422 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3423 queue_id, &ops_deq[deq], enq - deq); 3424 } 3425 3426 total_time += rte_rdtsc_precise() - start_time; 3427 } 3428 3429 tp->iter_count = 0; 3430 /* get the max of iter_count for all dequeued ops */ 3431 for (i = 0; i < num_ops; ++i) { 3432 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3433 tp->iter_count); 3434 } 3435 if (extDdr) { 3436 /* Read loopback is not thread safe */ 3437 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 3438 } 3439 3440 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3441 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3442 tp->op_params->vector_mask); 3443 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3444 } 3445 3446 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3447 3448 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3449 3450 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3451 ((double)total_time / (double)rte_get_tsc_hz()); 3452 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3453 1000000.0) / ((double)total_time / 3454 (double)rte_get_tsc_hz()); 3455 3456 return TEST_SUCCESS; 3457 } 3458 3459 static int 3460 throughput_pmd_lcore_enc(void *arg) 3461 { 3462 struct thread_params *tp = arg; 3463 uint16_t enq, deq; 3464 uint64_t total_time = 0, start_time; 3465 const uint16_t queue_id = tp->queue_id; 3466 const uint16_t burst_sz = tp->op_params->burst_sz; 3467 const uint16_t num_ops = tp->op_params->num_to_process; 3468 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3469 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3470 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3471 struct test_buffers *bufs = NULL; 3472 int i, j, ret; 3473 struct rte_bbdev_info info; 3474 uint16_t num_to_enq; 3475 3476 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3477 "BURST_SIZE should be <= %u", MAX_BURST); 3478 3479 rte_bbdev_info_get(tp->dev_id, &info); 3480 3481 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3482 "NUM_OPS cannot exceed %u for this device", 3483 info.drv.queue_size_lim); 3484 3485 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3486 3487 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3488 3489 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3490 num_ops); 3491 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3492 num_ops); 3493 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3494 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3495 bufs->hard_outputs, ref_op); 3496 3497 /* Set counter to validate the ordering */ 3498 for (j = 0; j < num_ops; ++j) 3499 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3500 3501 for (i = 0; i < TEST_REPETITIONS; ++i) { 3502 3503 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3504 for (j = 0; j < num_ops; ++j) 3505 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3506 3507 start_time = rte_rdtsc_precise(); 3508 3509 for (enq = 0, deq = 0; enq < num_ops;) { 3510 num_to_enq = burst_sz; 3511 3512 if (unlikely(num_ops - enq < num_to_enq)) 3513 num_to_enq = num_ops - enq; 3514 3515 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3516 queue_id, &ops_enq[enq], num_to_enq); 3517 3518 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3519 queue_id, &ops_deq[deq], enq - deq); 3520 } 3521 3522 /* dequeue the remaining */ 3523 while (deq < enq) { 3524 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3525 queue_id, &ops_deq[deq], enq - deq); 3526 } 3527 3528 total_time += rte_rdtsc_precise() - start_time; 3529 } 3530 3531 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3532 ret = validate_enc_op(ops_deq, num_ops, ref_op); 3533 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3534 } 3535 3536 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3537 3538 double tb_len_bits = calc_enc_TB_size(ref_op); 3539 3540 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3541 ((double)total_time / (double)rte_get_tsc_hz()); 3542 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3543 / 1000000.0) / ((double)total_time / 3544 (double)rte_get_tsc_hz()); 3545 3546 return TEST_SUCCESS; 3547 } 3548 3549 static int 3550 throughput_pmd_lcore_ldpc_enc(void *arg) 3551 { 3552 struct thread_params *tp = arg; 3553 uint16_t enq, deq; 3554 uint64_t total_time = 0, start_time; 3555 const uint16_t queue_id = tp->queue_id; 3556 const uint16_t burst_sz = tp->op_params->burst_sz; 3557 const uint16_t num_ops = tp->op_params->num_to_process; 3558 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3559 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3560 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3561 struct test_buffers *bufs = NULL; 3562 int i, j, ret; 3563 struct rte_bbdev_info info; 3564 uint16_t num_to_enq; 3565 3566 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3567 "BURST_SIZE should be <= %u", MAX_BURST); 3568 3569 rte_bbdev_info_get(tp->dev_id, &info); 3570 3571 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3572 "NUM_OPS cannot exceed %u for this device", 3573 info.drv.queue_size_lim); 3574 3575 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3576 3577 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3578 3579 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3580 num_ops); 3581 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3582 num_ops); 3583 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3584 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3585 bufs->hard_outputs, ref_op); 3586 3587 /* Set counter to validate the ordering */ 3588 for (j = 0; j < num_ops; ++j) 3589 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3590 3591 for (i = 0; i < TEST_REPETITIONS; ++i) { 3592 3593 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3594 for (j = 0; j < num_ops; ++j) 3595 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3596 3597 start_time = rte_rdtsc_precise(); 3598 3599 for (enq = 0, deq = 0; enq < num_ops;) { 3600 num_to_enq = burst_sz; 3601 3602 if (unlikely(num_ops - enq < num_to_enq)) 3603 num_to_enq = num_ops - enq; 3604 3605 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 3606 queue_id, &ops_enq[enq], num_to_enq); 3607 3608 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3609 queue_id, &ops_deq[deq], enq - deq); 3610 } 3611 3612 /* dequeue the remaining */ 3613 while (deq < enq) { 3614 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3615 queue_id, &ops_deq[deq], enq - deq); 3616 } 3617 3618 total_time += rte_rdtsc_precise() - start_time; 3619 } 3620 3621 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3622 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 3623 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3624 } 3625 3626 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3627 3628 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 3629 3630 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3631 ((double)total_time / (double)rte_get_tsc_hz()); 3632 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3633 / 1000000.0) / ((double)total_time / 3634 (double)rte_get_tsc_hz()); 3635 3636 return TEST_SUCCESS; 3637 } 3638 3639 static void 3640 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 3641 { 3642 unsigned int iter = 0; 3643 double total_mops = 0, total_mbps = 0; 3644 3645 for (iter = 0; iter < used_cores; iter++) { 3646 printf( 3647 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 3648 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 3649 t_params[iter].mbps); 3650 total_mops += t_params[iter].ops_per_sec; 3651 total_mbps += t_params[iter].mbps; 3652 } 3653 printf( 3654 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 3655 used_cores, total_mops, total_mbps); 3656 } 3657 3658 /* Aggregate the performance results over the number of cores used */ 3659 static void 3660 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 3661 { 3662 unsigned int core_idx = 0; 3663 double total_mops = 0, total_mbps = 0; 3664 uint8_t iter_count = 0; 3665 3666 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3667 printf( 3668 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 3669 t_params[core_idx].lcore_id, 3670 t_params[core_idx].ops_per_sec, 3671 t_params[core_idx].mbps, 3672 t_params[core_idx].iter_count); 3673 total_mops += t_params[core_idx].ops_per_sec; 3674 total_mbps += t_params[core_idx].mbps; 3675 iter_count = RTE_MAX(iter_count, 3676 t_params[core_idx].iter_count); 3677 } 3678 printf( 3679 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 3680 used_cores, total_mops, total_mbps, iter_count); 3681 } 3682 3683 /* Aggregate the performance results over the number of cores used */ 3684 static void 3685 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 3686 { 3687 unsigned int core_idx = 0; 3688 double total_mbps = 0, total_bler = 0, total_iter = 0; 3689 double snr = get_snr(); 3690 3691 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3692 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 3693 t_params[core_idx].lcore_id, 3694 t_params[core_idx].bler * 100, 3695 t_params[core_idx].iter_average, 3696 t_params[core_idx].mbps, 3697 get_vector_filename()); 3698 total_mbps += t_params[core_idx].mbps; 3699 total_bler += t_params[core_idx].bler; 3700 total_iter += t_params[core_idx].iter_average; 3701 } 3702 total_bler /= used_cores; 3703 total_iter /= used_cores; 3704 3705 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n", 3706 snr, total_bler * 100, total_iter, get_iter_max(), 3707 total_mbps, get_vector_filename()); 3708 } 3709 3710 /* 3711 * Test function that determines BLER wireless performance 3712 */ 3713 static int 3714 bler_test(struct active_device *ad, 3715 struct test_op_params *op_params) 3716 { 3717 int ret; 3718 unsigned int lcore_id, used_cores = 0; 3719 struct thread_params *t_params; 3720 struct rte_bbdev_info info; 3721 lcore_function_t *bler_function; 3722 uint16_t num_lcores; 3723 const char *op_type_str; 3724 3725 rte_bbdev_info_get(ad->dev_id, &info); 3726 3727 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3728 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3729 test_vector.op_type); 3730 3731 printf("+ ------------------------------------------------------- +\n"); 3732 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3733 info.dev_name, ad->nb_queues, op_params->burst_sz, 3734 op_params->num_to_process, op_params->num_lcores, 3735 op_type_str, 3736 intr_enabled ? "Interrupt mode" : "PMD mode", 3737 (double)rte_get_tsc_hz() / 1000000000.0); 3738 3739 /* Set number of lcores */ 3740 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3741 ? ad->nb_queues 3742 : op_params->num_lcores; 3743 3744 /* Allocate memory for thread parameters structure */ 3745 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3746 RTE_CACHE_LINE_SIZE); 3747 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3748 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3749 RTE_CACHE_LINE_SIZE)); 3750 3751 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) && 3752 !check_bit(test_vector.ldpc_dec.op_flags, 3753 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 3754 && !check_bit(test_vector.ldpc_dec.op_flags, 3755 RTE_BBDEV_LDPC_LLR_COMPRESSION)) 3756 bler_function = bler_pmd_lcore_ldpc_dec; 3757 else 3758 return TEST_SKIPPED; 3759 3760 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 3761 3762 /* Main core is set at first entry */ 3763 t_params[0].dev_id = ad->dev_id; 3764 t_params[0].lcore_id = rte_lcore_id(); 3765 t_params[0].op_params = op_params; 3766 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3767 t_params[0].iter_count = 0; 3768 3769 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3770 if (used_cores >= num_lcores) 3771 break; 3772 3773 t_params[used_cores].dev_id = ad->dev_id; 3774 t_params[used_cores].lcore_id = lcore_id; 3775 t_params[used_cores].op_params = op_params; 3776 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3777 t_params[used_cores].iter_count = 0; 3778 3779 rte_eal_remote_launch(bler_function, 3780 &t_params[used_cores++], lcore_id); 3781 } 3782 3783 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3784 ret = bler_function(&t_params[0]); 3785 3786 /* Main core is always used */ 3787 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3788 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3789 3790 print_dec_bler(t_params, num_lcores); 3791 3792 /* Return if test failed */ 3793 if (ret) { 3794 rte_free(t_params); 3795 return ret; 3796 } 3797 3798 /* Function to print something here*/ 3799 rte_free(t_params); 3800 return ret; 3801 } 3802 3803 /* 3804 * Test function that determines how long an enqueue + dequeue of a burst 3805 * takes on available lcores. 3806 */ 3807 static int 3808 throughput_test(struct active_device *ad, 3809 struct test_op_params *op_params) 3810 { 3811 int ret; 3812 unsigned int lcore_id, used_cores = 0; 3813 struct thread_params *t_params, *tp; 3814 struct rte_bbdev_info info; 3815 lcore_function_t *throughput_function; 3816 uint16_t num_lcores; 3817 const char *op_type_str; 3818 3819 rte_bbdev_info_get(ad->dev_id, &info); 3820 3821 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3822 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3823 test_vector.op_type); 3824 3825 printf("+ ------------------------------------------------------- +\n"); 3826 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3827 info.dev_name, ad->nb_queues, op_params->burst_sz, 3828 op_params->num_to_process, op_params->num_lcores, 3829 op_type_str, 3830 intr_enabled ? "Interrupt mode" : "PMD mode", 3831 (double)rte_get_tsc_hz() / 1000000000.0); 3832 3833 /* Set number of lcores */ 3834 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3835 ? ad->nb_queues 3836 : op_params->num_lcores; 3837 3838 /* Allocate memory for thread parameters structure */ 3839 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3840 RTE_CACHE_LINE_SIZE); 3841 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3842 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3843 RTE_CACHE_LINE_SIZE)); 3844 3845 if (intr_enabled) { 3846 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3847 throughput_function = throughput_intr_lcore_dec; 3848 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3849 throughput_function = throughput_intr_lcore_ldpc_dec; 3850 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3851 throughput_function = throughput_intr_lcore_enc; 3852 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3853 throughput_function = throughput_intr_lcore_ldpc_enc; 3854 else 3855 throughput_function = throughput_intr_lcore_enc; 3856 3857 /* Dequeue interrupt callback registration */ 3858 ret = rte_bbdev_callback_register(ad->dev_id, 3859 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 3860 t_params); 3861 if (ret < 0) { 3862 rte_free(t_params); 3863 return ret; 3864 } 3865 } else { 3866 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3867 throughput_function = throughput_pmd_lcore_dec; 3868 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3869 throughput_function = throughput_pmd_lcore_ldpc_dec; 3870 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3871 throughput_function = throughput_pmd_lcore_enc; 3872 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3873 throughput_function = throughput_pmd_lcore_ldpc_enc; 3874 else 3875 throughput_function = throughput_pmd_lcore_enc; 3876 } 3877 3878 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED); 3879 3880 /* Main core is set at first entry */ 3881 t_params[0].dev_id = ad->dev_id; 3882 t_params[0].lcore_id = rte_lcore_id(); 3883 t_params[0].op_params = op_params; 3884 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3885 t_params[0].iter_count = 0; 3886 3887 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3888 if (used_cores >= num_lcores) 3889 break; 3890 3891 t_params[used_cores].dev_id = ad->dev_id; 3892 t_params[used_cores].lcore_id = lcore_id; 3893 t_params[used_cores].op_params = op_params; 3894 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3895 t_params[used_cores].iter_count = 0; 3896 3897 rte_eal_remote_launch(throughput_function, 3898 &t_params[used_cores++], lcore_id); 3899 } 3900 3901 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED); 3902 ret = throughput_function(&t_params[0]); 3903 3904 /* Main core is always used */ 3905 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3906 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3907 3908 /* Return if test failed */ 3909 if (ret) { 3910 rte_free(t_params); 3911 return ret; 3912 } 3913 3914 /* Print throughput if interrupts are disabled and test passed */ 3915 if (!intr_enabled) { 3916 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3917 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3918 print_dec_throughput(t_params, num_lcores); 3919 else 3920 print_enc_throughput(t_params, num_lcores); 3921 rte_free(t_params); 3922 return ret; 3923 } 3924 3925 /* In interrupt TC we need to wait for the interrupt callback to deqeue 3926 * all pending operations. Skip waiting for queues which reported an 3927 * error using processing_status variable. 3928 * Wait for main lcore operations. 3929 */ 3930 tp = &t_params[0]; 3931 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 3932 op_params->num_to_process) && 3933 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 3934 TEST_FAILED)) 3935 rte_pause(); 3936 3937 tp->ops_per_sec /= TEST_REPETITIONS; 3938 tp->mbps /= TEST_REPETITIONS; 3939 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 3940 3941 /* Wait for worker lcores operations */ 3942 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 3943 tp = &t_params[used_cores]; 3944 3945 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) < 3946 op_params->num_to_process) && 3947 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) != 3948 TEST_FAILED)) 3949 rte_pause(); 3950 3951 tp->ops_per_sec /= TEST_REPETITIONS; 3952 tp->mbps /= TEST_REPETITIONS; 3953 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED); 3954 } 3955 3956 /* Print throughput if test passed */ 3957 if (!ret) { 3958 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3959 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3960 print_dec_throughput(t_params, num_lcores); 3961 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 3962 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3963 print_enc_throughput(t_params, num_lcores); 3964 } 3965 3966 rte_free(t_params); 3967 return ret; 3968 } 3969 3970 static int 3971 latency_test_dec(struct rte_mempool *mempool, 3972 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 3973 int vector_mask, uint16_t dev_id, uint16_t queue_id, 3974 const uint16_t num_to_process, uint16_t burst_sz, 3975 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 3976 { 3977 int ret = TEST_SUCCESS; 3978 uint16_t i, j, dequeued; 3979 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3980 uint64_t start_time = 0, last_time = 0; 3981 3982 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3983 uint16_t enq = 0, deq = 0; 3984 bool first_time = true; 3985 last_time = 0; 3986 3987 if (unlikely(num_to_process - dequeued < burst_sz)) 3988 burst_sz = num_to_process - dequeued; 3989 3990 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3991 TEST_ASSERT_SUCCESS(ret, 3992 "rte_bbdev_dec_op_alloc_bulk() failed"); 3993 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3994 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3995 bufs->inputs, 3996 bufs->hard_outputs, 3997 bufs->soft_outputs, 3998 ref_op); 3999 4000 /* Set counter to validate the ordering */ 4001 for (j = 0; j < burst_sz; ++j) 4002 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4003 4004 start_time = rte_rdtsc_precise(); 4005 4006 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 4007 burst_sz); 4008 TEST_ASSERT(enq == burst_sz, 4009 "Error enqueueing burst, expected %u, got %u", 4010 burst_sz, enq); 4011 4012 /* Dequeue */ 4013 do { 4014 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4015 &ops_deq[deq], burst_sz - deq); 4016 if (likely(first_time && (deq > 0))) { 4017 last_time = rte_rdtsc_precise() - start_time; 4018 first_time = false; 4019 } 4020 } while (unlikely(burst_sz != deq)); 4021 4022 *max_time = RTE_MAX(*max_time, last_time); 4023 *min_time = RTE_MIN(*min_time, last_time); 4024 *total_time += last_time; 4025 4026 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4027 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 4028 vector_mask); 4029 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4030 } 4031 4032 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4033 dequeued += deq; 4034 } 4035 4036 return i; 4037 } 4038 4039 /* Test case for latency/validation for LDPC Decoder */ 4040 static int 4041 latency_test_ldpc_dec(struct rte_mempool *mempool, 4042 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 4043 int vector_mask, uint16_t dev_id, uint16_t queue_id, 4044 const uint16_t num_to_process, uint16_t burst_sz, 4045 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 4046 bool disable_et) 4047 { 4048 int ret = TEST_SUCCESS; 4049 uint16_t i, j, dequeued; 4050 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4051 uint64_t start_time = 0, last_time = 0; 4052 bool extDdr = ldpc_cap_flags & 4053 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4054 4055 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4056 uint16_t enq = 0, deq = 0; 4057 bool first_time = true; 4058 last_time = 0; 4059 4060 if (unlikely(num_to_process - dequeued < burst_sz)) 4061 burst_sz = num_to_process - dequeued; 4062 4063 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4064 TEST_ASSERT_SUCCESS(ret, 4065 "rte_bbdev_dec_op_alloc_bulk() failed"); 4066 4067 /* For latency tests we need to disable early termination */ 4068 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 4069 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4070 ref_op->ldpc_dec.op_flags -= 4071 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4072 ref_op->ldpc_dec.iter_max = get_iter_max(); 4073 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4074 4075 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4076 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4077 bufs->inputs, 4078 bufs->hard_outputs, 4079 bufs->soft_outputs, 4080 bufs->harq_inputs, 4081 bufs->harq_outputs, 4082 ref_op); 4083 4084 if (extDdr) 4085 preload_harq_ddr(dev_id, queue_id, ops_enq, 4086 burst_sz, true); 4087 4088 /* Set counter to validate the ordering */ 4089 for (j = 0; j < burst_sz; ++j) 4090 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4091 4092 start_time = rte_rdtsc_precise(); 4093 4094 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4095 &ops_enq[enq], burst_sz); 4096 TEST_ASSERT(enq == burst_sz, 4097 "Error enqueueing burst, expected %u, got %u", 4098 burst_sz, enq); 4099 4100 /* Dequeue */ 4101 do { 4102 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4103 &ops_deq[deq], burst_sz - deq); 4104 if (likely(first_time && (deq > 0))) { 4105 last_time = rte_rdtsc_precise() - start_time; 4106 first_time = false; 4107 } 4108 } while (unlikely(burst_sz != deq)); 4109 4110 *max_time = RTE_MAX(*max_time, last_time); 4111 *min_time = RTE_MIN(*min_time, last_time); 4112 *total_time += last_time; 4113 4114 if (extDdr) 4115 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4116 4117 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4118 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 4119 vector_mask); 4120 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4121 } 4122 4123 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4124 dequeued += deq; 4125 } 4126 return i; 4127 } 4128 4129 static int 4130 latency_test_enc(struct rte_mempool *mempool, 4131 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4132 uint16_t dev_id, uint16_t queue_id, 4133 const uint16_t num_to_process, uint16_t burst_sz, 4134 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4135 { 4136 int ret = TEST_SUCCESS; 4137 uint16_t i, j, dequeued; 4138 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4139 uint64_t start_time = 0, last_time = 0; 4140 4141 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4142 uint16_t enq = 0, deq = 0; 4143 bool first_time = true; 4144 last_time = 0; 4145 4146 if (unlikely(num_to_process - dequeued < burst_sz)) 4147 burst_sz = num_to_process - dequeued; 4148 4149 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4150 TEST_ASSERT_SUCCESS(ret, 4151 "rte_bbdev_enc_op_alloc_bulk() failed"); 4152 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4153 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4154 bufs->inputs, 4155 bufs->hard_outputs, 4156 ref_op); 4157 4158 /* Set counter to validate the ordering */ 4159 for (j = 0; j < burst_sz; ++j) 4160 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4161 4162 start_time = rte_rdtsc_precise(); 4163 4164 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 4165 burst_sz); 4166 TEST_ASSERT(enq == burst_sz, 4167 "Error enqueueing burst, expected %u, got %u", 4168 burst_sz, enq); 4169 4170 /* Dequeue */ 4171 do { 4172 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4173 &ops_deq[deq], burst_sz - deq); 4174 if (likely(first_time && (deq > 0))) { 4175 last_time += rte_rdtsc_precise() - start_time; 4176 first_time = false; 4177 } 4178 } while (unlikely(burst_sz != deq)); 4179 4180 *max_time = RTE_MAX(*max_time, last_time); 4181 *min_time = RTE_MIN(*min_time, last_time); 4182 *total_time += last_time; 4183 4184 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4185 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4186 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4187 } 4188 4189 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4190 dequeued += deq; 4191 } 4192 4193 return i; 4194 } 4195 4196 static int 4197 latency_test_ldpc_enc(struct rte_mempool *mempool, 4198 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4199 uint16_t dev_id, uint16_t queue_id, 4200 const uint16_t num_to_process, uint16_t burst_sz, 4201 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4202 { 4203 int ret = TEST_SUCCESS; 4204 uint16_t i, j, dequeued; 4205 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4206 uint64_t start_time = 0, last_time = 0; 4207 4208 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4209 uint16_t enq = 0, deq = 0; 4210 bool first_time = true; 4211 last_time = 0; 4212 4213 if (unlikely(num_to_process - dequeued < burst_sz)) 4214 burst_sz = num_to_process - dequeued; 4215 4216 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4217 TEST_ASSERT_SUCCESS(ret, 4218 "rte_bbdev_enc_op_alloc_bulk() failed"); 4219 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4220 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4221 bufs->inputs, 4222 bufs->hard_outputs, 4223 ref_op); 4224 4225 /* Set counter to validate the ordering */ 4226 for (j = 0; j < burst_sz; ++j) 4227 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4228 4229 start_time = rte_rdtsc_precise(); 4230 4231 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4232 &ops_enq[enq], burst_sz); 4233 TEST_ASSERT(enq == burst_sz, 4234 "Error enqueueing burst, expected %u, got %u", 4235 burst_sz, enq); 4236 4237 /* Dequeue */ 4238 do { 4239 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4240 &ops_deq[deq], burst_sz - deq); 4241 if (likely(first_time && (deq > 0))) { 4242 last_time += rte_rdtsc_precise() - start_time; 4243 first_time = false; 4244 } 4245 } while (unlikely(burst_sz != deq)); 4246 4247 *max_time = RTE_MAX(*max_time, last_time); 4248 *min_time = RTE_MIN(*min_time, last_time); 4249 *total_time += last_time; 4250 4251 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4252 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4253 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4254 } 4255 4256 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4257 dequeued += deq; 4258 } 4259 4260 return i; 4261 } 4262 4263 /* Common function for running validation and latency test cases */ 4264 static int 4265 validation_latency_test(struct active_device *ad, 4266 struct test_op_params *op_params, bool latency_flag) 4267 { 4268 int iter; 4269 uint16_t burst_sz = op_params->burst_sz; 4270 const uint16_t num_to_process = op_params->num_to_process; 4271 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4272 const uint16_t queue_id = ad->queue_ids[0]; 4273 struct test_buffers *bufs = NULL; 4274 struct rte_bbdev_info info; 4275 uint64_t total_time, min_time, max_time; 4276 const char *op_type_str; 4277 4278 total_time = max_time = 0; 4279 min_time = UINT64_MAX; 4280 4281 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4282 "BURST_SIZE should be <= %u", MAX_BURST); 4283 4284 rte_bbdev_info_get(ad->dev_id, &info); 4285 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4286 4287 op_type_str = rte_bbdev_op_type_str(op_type); 4288 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4289 4290 printf("+ ------------------------------------------------------- +\n"); 4291 if (latency_flag) 4292 printf("== test: latency\ndev:"); 4293 else 4294 printf("== test: validation\ndev:"); 4295 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 4296 info.dev_name, burst_sz, num_to_process, op_type_str); 4297 4298 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4299 iter = latency_test_dec(op_params->mp, bufs, 4300 op_params->ref_dec_op, op_params->vector_mask, 4301 ad->dev_id, queue_id, num_to_process, 4302 burst_sz, &total_time, &min_time, &max_time); 4303 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4304 iter = latency_test_ldpc_enc(op_params->mp, bufs, 4305 op_params->ref_enc_op, ad->dev_id, queue_id, 4306 num_to_process, burst_sz, &total_time, 4307 &min_time, &max_time); 4308 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4309 iter = latency_test_ldpc_dec(op_params->mp, bufs, 4310 op_params->ref_dec_op, op_params->vector_mask, 4311 ad->dev_id, queue_id, num_to_process, 4312 burst_sz, &total_time, &min_time, &max_time, 4313 latency_flag); 4314 else /* RTE_BBDEV_OP_TURBO_ENC */ 4315 iter = latency_test_enc(op_params->mp, bufs, 4316 op_params->ref_enc_op, 4317 ad->dev_id, queue_id, 4318 num_to_process, burst_sz, &total_time, 4319 &min_time, &max_time); 4320 4321 if (iter <= 0) 4322 return TEST_FAILED; 4323 4324 printf("Operation latency:\n" 4325 "\tavg: %lg cycles, %lg us\n" 4326 "\tmin: %lg cycles, %lg us\n" 4327 "\tmax: %lg cycles, %lg us\n", 4328 (double)total_time / (double)iter, 4329 (double)(total_time * 1000000) / (double)iter / 4330 (double)rte_get_tsc_hz(), (double)min_time, 4331 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 4332 (double)max_time, (double)(max_time * 1000000) / 4333 (double)rte_get_tsc_hz()); 4334 4335 return TEST_SUCCESS; 4336 } 4337 4338 static int 4339 latency_test(struct active_device *ad, struct test_op_params *op_params) 4340 { 4341 return validation_latency_test(ad, op_params, true); 4342 } 4343 4344 static int 4345 validation_test(struct active_device *ad, struct test_op_params *op_params) 4346 { 4347 return validation_latency_test(ad, op_params, false); 4348 } 4349 4350 #ifdef RTE_BBDEV_OFFLOAD_COST 4351 static int 4352 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 4353 struct rte_bbdev_stats *stats) 4354 { 4355 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 4356 struct rte_bbdev_stats *q_stats; 4357 4358 if (queue_id >= dev->data->num_queues) 4359 return -1; 4360 4361 q_stats = &dev->data->queues[queue_id].queue_stats; 4362 4363 stats->enqueued_count = q_stats->enqueued_count; 4364 stats->dequeued_count = q_stats->dequeued_count; 4365 stats->enqueue_err_count = q_stats->enqueue_err_count; 4366 stats->dequeue_err_count = q_stats->dequeue_err_count; 4367 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 4368 4369 return 0; 4370 } 4371 4372 static int 4373 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 4374 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4375 uint16_t queue_id, const uint16_t num_to_process, 4376 uint16_t burst_sz, struct test_time_stats *time_st) 4377 { 4378 int i, dequeued, ret; 4379 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4380 uint64_t enq_start_time, deq_start_time; 4381 uint64_t enq_sw_last_time, deq_last_time; 4382 struct rte_bbdev_stats stats; 4383 4384 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4385 uint16_t enq = 0, deq = 0; 4386 4387 if (unlikely(num_to_process - dequeued < burst_sz)) 4388 burst_sz = num_to_process - dequeued; 4389 4390 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4391 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4392 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 4393 bufs->inputs, 4394 bufs->hard_outputs, 4395 bufs->soft_outputs, 4396 ref_op); 4397 4398 /* Start time meas for enqueue function offload latency */ 4399 enq_start_time = rte_rdtsc_precise(); 4400 do { 4401 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 4402 &ops_enq[enq], burst_sz - enq); 4403 } while (unlikely(burst_sz != enq)); 4404 4405 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4406 TEST_ASSERT_SUCCESS(ret, 4407 "Failed to get stats for queue (%u) of device (%u)", 4408 queue_id, dev_id); 4409 4410 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 4411 stats.acc_offload_cycles; 4412 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4413 enq_sw_last_time); 4414 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4415 enq_sw_last_time); 4416 time_st->enq_sw_total_time += enq_sw_last_time; 4417 4418 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4419 stats.acc_offload_cycles); 4420 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4421 stats.acc_offload_cycles); 4422 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4423 4424 /* give time for device to process ops */ 4425 rte_delay_us(WAIT_OFFLOAD_US); 4426 4427 /* Start time meas for dequeue function offload latency */ 4428 deq_start_time = rte_rdtsc_precise(); 4429 /* Dequeue one operation */ 4430 do { 4431 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4432 &ops_deq[deq], enq); 4433 } while (unlikely(deq == 0)); 4434 4435 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4436 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4437 deq_last_time); 4438 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4439 deq_last_time); 4440 time_st->deq_total_time += deq_last_time; 4441 4442 /* Dequeue remaining operations if needed*/ 4443 while (burst_sz != deq) 4444 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4445 &ops_deq[deq], burst_sz - deq); 4446 4447 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4448 dequeued += deq; 4449 } 4450 4451 return i; 4452 } 4453 4454 static int 4455 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 4456 struct test_buffers *bufs, 4457 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4458 uint16_t queue_id, const uint16_t num_to_process, 4459 uint16_t burst_sz, struct test_time_stats *time_st) 4460 { 4461 int i, dequeued, ret; 4462 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4463 uint64_t enq_start_time, deq_start_time; 4464 uint64_t enq_sw_last_time, deq_last_time; 4465 struct rte_bbdev_stats stats; 4466 bool extDdr = ldpc_cap_flags & 4467 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4468 4469 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4470 uint16_t enq = 0, deq = 0; 4471 4472 if (unlikely(num_to_process - dequeued < burst_sz)) 4473 burst_sz = num_to_process - dequeued; 4474 4475 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4476 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4477 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4478 bufs->inputs, 4479 bufs->hard_outputs, 4480 bufs->soft_outputs, 4481 bufs->harq_inputs, 4482 bufs->harq_outputs, 4483 ref_op); 4484 4485 if (extDdr) 4486 preload_harq_ddr(dev_id, queue_id, ops_enq, 4487 burst_sz, true); 4488 4489 /* Start time meas for enqueue function offload latency */ 4490 enq_start_time = rte_rdtsc_precise(); 4491 do { 4492 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4493 &ops_enq[enq], burst_sz - enq); 4494 } while (unlikely(burst_sz != enq)); 4495 4496 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4497 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4498 TEST_ASSERT_SUCCESS(ret, 4499 "Failed to get stats for queue (%u) of device (%u)", 4500 queue_id, dev_id); 4501 4502 enq_sw_last_time -= stats.acc_offload_cycles; 4503 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4504 enq_sw_last_time); 4505 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4506 enq_sw_last_time); 4507 time_st->enq_sw_total_time += enq_sw_last_time; 4508 4509 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4510 stats.acc_offload_cycles); 4511 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4512 stats.acc_offload_cycles); 4513 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4514 4515 /* give time for device to process ops */ 4516 rte_delay_us(WAIT_OFFLOAD_US); 4517 4518 /* Start time meas for dequeue function offload latency */ 4519 deq_start_time = rte_rdtsc_precise(); 4520 /* Dequeue one operation */ 4521 do { 4522 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4523 &ops_deq[deq], enq); 4524 } while (unlikely(deq == 0)); 4525 4526 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4527 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4528 deq_last_time); 4529 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4530 deq_last_time); 4531 time_st->deq_total_time += deq_last_time; 4532 4533 /* Dequeue remaining operations if needed*/ 4534 while (burst_sz != deq) 4535 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4536 &ops_deq[deq], burst_sz - deq); 4537 4538 if (extDdr) { 4539 /* Read loopback is not thread safe */ 4540 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4541 } 4542 4543 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4544 dequeued += deq; 4545 } 4546 4547 return i; 4548 } 4549 4550 static int 4551 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 4552 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4553 uint16_t queue_id, const uint16_t num_to_process, 4554 uint16_t burst_sz, struct test_time_stats *time_st) 4555 { 4556 int i, dequeued, ret; 4557 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4558 uint64_t enq_start_time, deq_start_time; 4559 uint64_t enq_sw_last_time, deq_last_time; 4560 struct rte_bbdev_stats stats; 4561 4562 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4563 uint16_t enq = 0, deq = 0; 4564 4565 if (unlikely(num_to_process - dequeued < burst_sz)) 4566 burst_sz = num_to_process - dequeued; 4567 4568 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4569 TEST_ASSERT_SUCCESS(ret, 4570 "rte_bbdev_enc_op_alloc_bulk() failed"); 4571 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4572 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4573 bufs->inputs, 4574 bufs->hard_outputs, 4575 ref_op); 4576 4577 /* Start time meas for enqueue function offload latency */ 4578 enq_start_time = rte_rdtsc_precise(); 4579 do { 4580 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 4581 &ops_enq[enq], burst_sz - enq); 4582 } while (unlikely(burst_sz != enq)); 4583 4584 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4585 4586 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4587 TEST_ASSERT_SUCCESS(ret, 4588 "Failed to get stats for queue (%u) of device (%u)", 4589 queue_id, dev_id); 4590 enq_sw_last_time -= stats.acc_offload_cycles; 4591 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4592 enq_sw_last_time); 4593 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4594 enq_sw_last_time); 4595 time_st->enq_sw_total_time += enq_sw_last_time; 4596 4597 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4598 stats.acc_offload_cycles); 4599 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4600 stats.acc_offload_cycles); 4601 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4602 4603 /* give time for device to process ops */ 4604 rte_delay_us(WAIT_OFFLOAD_US); 4605 4606 /* Start time meas for dequeue function offload latency */ 4607 deq_start_time = rte_rdtsc_precise(); 4608 /* Dequeue one operation */ 4609 do { 4610 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4611 &ops_deq[deq], enq); 4612 } while (unlikely(deq == 0)); 4613 4614 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4615 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4616 deq_last_time); 4617 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4618 deq_last_time); 4619 time_st->deq_total_time += deq_last_time; 4620 4621 while (burst_sz != deq) 4622 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4623 &ops_deq[deq], burst_sz - deq); 4624 4625 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4626 dequeued += deq; 4627 } 4628 4629 return i; 4630 } 4631 4632 static int 4633 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 4634 struct test_buffers *bufs, 4635 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4636 uint16_t queue_id, const uint16_t num_to_process, 4637 uint16_t burst_sz, struct test_time_stats *time_st) 4638 { 4639 int i, dequeued, ret; 4640 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4641 uint64_t enq_start_time, deq_start_time; 4642 uint64_t enq_sw_last_time, deq_last_time; 4643 struct rte_bbdev_stats stats; 4644 4645 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4646 uint16_t enq = 0, deq = 0; 4647 4648 if (unlikely(num_to_process - dequeued < burst_sz)) 4649 burst_sz = num_to_process - dequeued; 4650 4651 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4652 TEST_ASSERT_SUCCESS(ret, 4653 "rte_bbdev_enc_op_alloc_bulk() failed"); 4654 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4655 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4656 bufs->inputs, 4657 bufs->hard_outputs, 4658 ref_op); 4659 4660 /* Start time meas for enqueue function offload latency */ 4661 enq_start_time = rte_rdtsc_precise(); 4662 do { 4663 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4664 &ops_enq[enq], burst_sz - enq); 4665 } while (unlikely(burst_sz != enq)); 4666 4667 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4668 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4669 TEST_ASSERT_SUCCESS(ret, 4670 "Failed to get stats for queue (%u) of device (%u)", 4671 queue_id, dev_id); 4672 4673 enq_sw_last_time -= stats.acc_offload_cycles; 4674 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4675 enq_sw_last_time); 4676 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4677 enq_sw_last_time); 4678 time_st->enq_sw_total_time += enq_sw_last_time; 4679 4680 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4681 stats.acc_offload_cycles); 4682 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4683 stats.acc_offload_cycles); 4684 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4685 4686 /* give time for device to process ops */ 4687 rte_delay_us(WAIT_OFFLOAD_US); 4688 4689 /* Start time meas for dequeue function offload latency */ 4690 deq_start_time = rte_rdtsc_precise(); 4691 /* Dequeue one operation */ 4692 do { 4693 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4694 &ops_deq[deq], enq); 4695 } while (unlikely(deq == 0)); 4696 4697 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4698 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4699 deq_last_time); 4700 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4701 deq_last_time); 4702 time_st->deq_total_time += deq_last_time; 4703 4704 while (burst_sz != deq) 4705 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4706 &ops_deq[deq], burst_sz - deq); 4707 4708 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4709 dequeued += deq; 4710 } 4711 4712 return i; 4713 } 4714 #endif 4715 4716 static int 4717 offload_cost_test(struct active_device *ad, 4718 struct test_op_params *op_params) 4719 { 4720 #ifndef RTE_BBDEV_OFFLOAD_COST 4721 RTE_SET_USED(ad); 4722 RTE_SET_USED(op_params); 4723 printf("Offload latency test is disabled.\n"); 4724 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4725 return TEST_SKIPPED; 4726 #else 4727 int iter; 4728 uint16_t burst_sz = op_params->burst_sz; 4729 const uint16_t num_to_process = op_params->num_to_process; 4730 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4731 const uint16_t queue_id = ad->queue_ids[0]; 4732 struct test_buffers *bufs = NULL; 4733 struct rte_bbdev_info info; 4734 const char *op_type_str; 4735 struct test_time_stats time_st; 4736 4737 memset(&time_st, 0, sizeof(struct test_time_stats)); 4738 time_st.enq_sw_min_time = UINT64_MAX; 4739 time_st.enq_acc_min_time = UINT64_MAX; 4740 time_st.deq_min_time = UINT64_MAX; 4741 4742 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4743 "BURST_SIZE should be <= %u", MAX_BURST); 4744 4745 rte_bbdev_info_get(ad->dev_id, &info); 4746 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4747 4748 op_type_str = rte_bbdev_op_type_str(op_type); 4749 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4750 4751 printf("+ ------------------------------------------------------- +\n"); 4752 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4753 info.dev_name, burst_sz, num_to_process, op_type_str); 4754 4755 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4756 iter = offload_latency_test_dec(op_params->mp, bufs, 4757 op_params->ref_dec_op, ad->dev_id, queue_id, 4758 num_to_process, burst_sz, &time_st); 4759 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 4760 iter = offload_latency_test_enc(op_params->mp, bufs, 4761 op_params->ref_enc_op, ad->dev_id, queue_id, 4762 num_to_process, burst_sz, &time_st); 4763 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4764 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 4765 op_params->ref_enc_op, ad->dev_id, queue_id, 4766 num_to_process, burst_sz, &time_st); 4767 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4768 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 4769 op_params->ref_dec_op, ad->dev_id, queue_id, 4770 num_to_process, burst_sz, &time_st); 4771 else 4772 iter = offload_latency_test_enc(op_params->mp, bufs, 4773 op_params->ref_enc_op, ad->dev_id, queue_id, 4774 num_to_process, burst_sz, &time_st); 4775 4776 if (iter <= 0) 4777 return TEST_FAILED; 4778 4779 printf("Enqueue driver offload cost latency:\n" 4780 "\tavg: %lg cycles, %lg us\n" 4781 "\tmin: %lg cycles, %lg us\n" 4782 "\tmax: %lg cycles, %lg us\n" 4783 "Enqueue accelerator offload cost latency:\n" 4784 "\tavg: %lg cycles, %lg us\n" 4785 "\tmin: %lg cycles, %lg us\n" 4786 "\tmax: %lg cycles, %lg us\n", 4787 (double)time_st.enq_sw_total_time / (double)iter, 4788 (double)(time_st.enq_sw_total_time * 1000000) / 4789 (double)iter / (double)rte_get_tsc_hz(), 4790 (double)time_st.enq_sw_min_time, 4791 (double)(time_st.enq_sw_min_time * 1000000) / 4792 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 4793 (double)(time_st.enq_sw_max_time * 1000000) / 4794 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 4795 (double)iter, 4796 (double)(time_st.enq_acc_total_time * 1000000) / 4797 (double)iter / (double)rte_get_tsc_hz(), 4798 (double)time_st.enq_acc_min_time, 4799 (double)(time_st.enq_acc_min_time * 1000000) / 4800 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 4801 (double)(time_st.enq_acc_max_time * 1000000) / 4802 rte_get_tsc_hz()); 4803 4804 printf("Dequeue offload cost latency - one op:\n" 4805 "\tavg: %lg cycles, %lg us\n" 4806 "\tmin: %lg cycles, %lg us\n" 4807 "\tmax: %lg cycles, %lg us\n", 4808 (double)time_st.deq_total_time / (double)iter, 4809 (double)(time_st.deq_total_time * 1000000) / 4810 (double)iter / (double)rte_get_tsc_hz(), 4811 (double)time_st.deq_min_time, 4812 (double)(time_st.deq_min_time * 1000000) / 4813 rte_get_tsc_hz(), (double)time_st.deq_max_time, 4814 (double)(time_st.deq_max_time * 1000000) / 4815 rte_get_tsc_hz()); 4816 4817 struct rte_bbdev_stats stats = {0}; 4818 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 4819 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 4820 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 4821 "Mismatch in enqueue count %10"PRIu64" %d", 4822 stats.enqueued_count, num_to_process); 4823 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 4824 "Mismatch in dequeue count %10"PRIu64" %d", 4825 stats.dequeued_count, num_to_process); 4826 } 4827 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 4828 "Enqueue count Error %10"PRIu64"", 4829 stats.enqueue_err_count); 4830 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 4831 "Dequeue count Error (%10"PRIu64"", 4832 stats.dequeue_err_count); 4833 4834 return TEST_SUCCESS; 4835 #endif 4836 } 4837 4838 #ifdef RTE_BBDEV_OFFLOAD_COST 4839 static int 4840 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 4841 const uint16_t num_to_process, uint16_t burst_sz, 4842 uint64_t *deq_total_time, uint64_t *deq_min_time, 4843 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4844 { 4845 int i, deq_total; 4846 struct rte_bbdev_dec_op *ops[MAX_BURST]; 4847 uint64_t deq_start_time, deq_last_time; 4848 4849 /* Test deq offload latency from an empty queue */ 4850 4851 for (i = 0, deq_total = 0; deq_total < num_to_process; 4852 ++i, deq_total += burst_sz) { 4853 deq_start_time = rte_rdtsc_precise(); 4854 4855 if (unlikely(num_to_process - deq_total < burst_sz)) 4856 burst_sz = num_to_process - deq_total; 4857 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4858 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 4859 burst_sz); 4860 else 4861 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 4862 burst_sz); 4863 4864 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4865 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4866 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4867 *deq_total_time += deq_last_time; 4868 } 4869 4870 return i; 4871 } 4872 4873 static int 4874 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 4875 const uint16_t num_to_process, uint16_t burst_sz, 4876 uint64_t *deq_total_time, uint64_t *deq_min_time, 4877 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4878 { 4879 int i, deq_total; 4880 struct rte_bbdev_enc_op *ops[MAX_BURST]; 4881 uint64_t deq_start_time, deq_last_time; 4882 4883 /* Test deq offload latency from an empty queue */ 4884 for (i = 0, deq_total = 0; deq_total < num_to_process; 4885 ++i, deq_total += burst_sz) { 4886 deq_start_time = rte_rdtsc_precise(); 4887 4888 if (unlikely(num_to_process - deq_total < burst_sz)) 4889 burst_sz = num_to_process - deq_total; 4890 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4891 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 4892 burst_sz); 4893 else 4894 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 4895 burst_sz); 4896 4897 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4898 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4899 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4900 *deq_total_time += deq_last_time; 4901 } 4902 4903 return i; 4904 } 4905 4906 #endif 4907 4908 static int 4909 offload_latency_empty_q_test(struct active_device *ad, 4910 struct test_op_params *op_params) 4911 { 4912 #ifndef RTE_BBDEV_OFFLOAD_COST 4913 RTE_SET_USED(ad); 4914 RTE_SET_USED(op_params); 4915 printf("Offload latency empty dequeue test is disabled.\n"); 4916 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4917 return TEST_SKIPPED; 4918 #else 4919 int iter; 4920 uint64_t deq_total_time, deq_min_time, deq_max_time; 4921 uint16_t burst_sz = op_params->burst_sz; 4922 const uint16_t num_to_process = op_params->num_to_process; 4923 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4924 const uint16_t queue_id = ad->queue_ids[0]; 4925 struct rte_bbdev_info info; 4926 const char *op_type_str; 4927 4928 deq_total_time = deq_max_time = 0; 4929 deq_min_time = UINT64_MAX; 4930 4931 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4932 "BURST_SIZE should be <= %u", MAX_BURST); 4933 4934 rte_bbdev_info_get(ad->dev_id, &info); 4935 4936 op_type_str = rte_bbdev_op_type_str(op_type); 4937 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4938 4939 printf("+ ------------------------------------------------------- +\n"); 4940 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4941 info.dev_name, burst_sz, num_to_process, op_type_str); 4942 4943 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 4944 op_type == RTE_BBDEV_OP_LDPC_DEC) 4945 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 4946 num_to_process, burst_sz, &deq_total_time, 4947 &deq_min_time, &deq_max_time, op_type); 4948 else 4949 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 4950 num_to_process, burst_sz, &deq_total_time, 4951 &deq_min_time, &deq_max_time, op_type); 4952 4953 if (iter <= 0) 4954 return TEST_FAILED; 4955 4956 printf("Empty dequeue offload:\n" 4957 "\tavg: %lg cycles, %lg us\n" 4958 "\tmin: %lg cycles, %lg us\n" 4959 "\tmax: %lg cycles, %lg us\n", 4960 (double)deq_total_time / (double)iter, 4961 (double)(deq_total_time * 1000000) / (double)iter / 4962 (double)rte_get_tsc_hz(), (double)deq_min_time, 4963 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 4964 (double)deq_max_time, (double)(deq_max_time * 1000000) / 4965 rte_get_tsc_hz()); 4966 4967 return TEST_SUCCESS; 4968 #endif 4969 } 4970 4971 static int 4972 bler_tc(void) 4973 { 4974 return run_test_case(bler_test); 4975 } 4976 4977 static int 4978 throughput_tc(void) 4979 { 4980 return run_test_case(throughput_test); 4981 } 4982 4983 static int 4984 offload_cost_tc(void) 4985 { 4986 return run_test_case(offload_cost_test); 4987 } 4988 4989 static int 4990 offload_latency_empty_q_tc(void) 4991 { 4992 return run_test_case(offload_latency_empty_q_test); 4993 } 4994 4995 static int 4996 latency_tc(void) 4997 { 4998 return run_test_case(latency_test); 4999 } 5000 5001 static int 5002 validation_tc(void) 5003 { 5004 return run_test_case(validation_test); 5005 } 5006 5007 static int 5008 interrupt_tc(void) 5009 { 5010 return run_test_case(throughput_test); 5011 } 5012 5013 static struct unit_test_suite bbdev_bler_testsuite = { 5014 .suite_name = "BBdev BLER Tests", 5015 .setup = testsuite_setup, 5016 .teardown = testsuite_teardown, 5017 .unit_test_cases = { 5018 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 5019 TEST_CASES_END() /**< NULL terminate unit test array */ 5020 } 5021 }; 5022 5023 static struct unit_test_suite bbdev_throughput_testsuite = { 5024 .suite_name = "BBdev Throughput Tests", 5025 .setup = testsuite_setup, 5026 .teardown = testsuite_teardown, 5027 .unit_test_cases = { 5028 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 5029 TEST_CASES_END() /**< NULL terminate unit test array */ 5030 } 5031 }; 5032 5033 static struct unit_test_suite bbdev_validation_testsuite = { 5034 .suite_name = "BBdev Validation Tests", 5035 .setup = testsuite_setup, 5036 .teardown = testsuite_teardown, 5037 .unit_test_cases = { 5038 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 5039 TEST_CASES_END() /**< NULL terminate unit test array */ 5040 } 5041 }; 5042 5043 static struct unit_test_suite bbdev_latency_testsuite = { 5044 .suite_name = "BBdev Latency Tests", 5045 .setup = testsuite_setup, 5046 .teardown = testsuite_teardown, 5047 .unit_test_cases = { 5048 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 5049 TEST_CASES_END() /**< NULL terminate unit test array */ 5050 } 5051 }; 5052 5053 static struct unit_test_suite bbdev_offload_cost_testsuite = { 5054 .suite_name = "BBdev Offload Cost Tests", 5055 .setup = testsuite_setup, 5056 .teardown = testsuite_teardown, 5057 .unit_test_cases = { 5058 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 5059 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 5060 TEST_CASES_END() /**< NULL terminate unit test array */ 5061 } 5062 }; 5063 5064 static struct unit_test_suite bbdev_interrupt_testsuite = { 5065 .suite_name = "BBdev Interrupt Tests", 5066 .setup = interrupt_testsuite_setup, 5067 .teardown = testsuite_teardown, 5068 .unit_test_cases = { 5069 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 5070 TEST_CASES_END() /**< NULL terminate unit test array */ 5071 } 5072 }; 5073 5074 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 5075 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 5076 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 5077 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 5078 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 5079 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 5080