1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_malloc.h> 8 #include <rte_eal.h> 9 #include <rte_log.h> 10 #include <rte_compressdev.h> 11 12 #include "comp_perf_test_verify.h" 13 #include "comp_perf_test_common.h" 14 15 void 16 cperf_verify_test_destructor(void *arg) 17 { 18 if (arg) { 19 comp_perf_free_memory( 20 ((struct cperf_verify_ctx *)arg)->options, 21 &((struct cperf_verify_ctx *)arg)->mem); 22 rte_free(arg); 23 } 24 } 25 26 void * 27 cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, 28 struct comp_test_data *options) 29 { 30 struct cperf_verify_ctx *ctx = NULL; 31 32 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 33 34 if (ctx == NULL) 35 return NULL; 36 37 ctx->mem.dev_id = dev_id; 38 ctx->mem.qp_id = qp_id; 39 ctx->options = options; 40 41 if (!comp_perf_allocate_memory(ctx->options, &ctx->mem) && 42 !prepare_bufs(ctx->options, &ctx->mem)) 43 return ctx; 44 45 cperf_verify_test_destructor(ctx); 46 return NULL; 47 } 48 49 static int 50 main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type) 51 { 52 struct comp_test_data *test_data = ctx->options; 53 uint8_t *output_data_ptr = NULL; 54 size_t *output_data_sz = NULL; 55 struct cperf_mem_resources *mem = &ctx->mem; 56 57 uint8_t dev_id = mem->dev_id; 58 uint32_t i, iter, num_iter; 59 struct rte_comp_op **ops, **deq_ops; 60 void *priv_xform = NULL; 61 struct rte_comp_xform xform; 62 size_t output_size = 0; 63 struct rte_mbuf **input_bufs, **output_bufs; 64 int res = 0; 65 int allocated = 0; 66 uint32_t out_seg_sz; 67 68 if (test_data == NULL || !test_data->burst_sz) { 69 RTE_LOG(ERR, USER1, 70 "Unknown burst size\n"); 71 return -1; 72 } 73 74 ops = rte_zmalloc_socket(NULL, 75 2 * mem->total_bufs * sizeof(struct rte_comp_op *), 76 0, rte_socket_id()); 77 78 if (ops == NULL) { 79 RTE_LOG(ERR, USER1, 80 "Can't allocate memory for ops structures\n"); 81 return -1; 82 } 83 84 deq_ops = &ops[mem->total_bufs]; 85 86 if (type == RTE_COMP_COMPRESS) { 87 xform = (struct rte_comp_xform) { 88 .type = RTE_COMP_COMPRESS, 89 .compress = { 90 .algo = test_data->test_algo, 91 .level = test_data->level, 92 .window_size = test_data->window_sz, 93 .chksum = RTE_COMP_CHECKSUM_NONE, 94 .hash_algo = RTE_COMP_HASH_ALGO_NONE 95 } 96 }; 97 if (test_data->test_algo == RTE_COMP_ALGO_DEFLATE) 98 xform.compress.deflate.huffman = test_data->huffman_enc; 99 else if (test_data->test_algo == RTE_COMP_ALGO_LZ4) 100 xform.compress.lz4.flags = test_data->lz4_flags; 101 output_data_ptr = ctx->mem.compressed_data; 102 output_data_sz = &ctx->comp_data_sz; 103 input_bufs = mem->decomp_bufs; 104 output_bufs = mem->comp_bufs; 105 out_seg_sz = test_data->out_seg_sz; 106 } else { 107 xform = (struct rte_comp_xform) { 108 .type = RTE_COMP_DECOMPRESS, 109 .decompress = { 110 .algo = test_data->test_algo, 111 .chksum = RTE_COMP_CHECKSUM_NONE, 112 .window_size = test_data->window_sz, 113 .hash_algo = RTE_COMP_HASH_ALGO_NONE 114 } 115 }; 116 if (test_data->test_algo == RTE_COMP_ALGO_LZ4) 117 xform.decompress.lz4.flags = test_data->lz4_flags; 118 output_data_ptr = ctx->mem.decompressed_data; 119 output_data_sz = &ctx->decomp_data_sz; 120 input_bufs = mem->comp_bufs; 121 output_bufs = mem->decomp_bufs; 122 out_seg_sz = (test_data->test_op & COMPRESS) ? 123 test_data->seg_sz : test_data->out_seg_sz; 124 } 125 126 /* Create private xform */ 127 if (rte_compressdev_private_xform_create(dev_id, &xform, 128 &priv_xform) < 0) { 129 RTE_LOG(ERR, USER1, "Private xform could not be created\n"); 130 res = -1; 131 goto end; 132 } 133 134 num_iter = 1; 135 136 for (iter = 0; iter < num_iter; iter++) { 137 uint32_t total_ops = mem->total_bufs; 138 uint32_t remaining_ops = mem->total_bufs; 139 uint32_t total_deq_ops = 0; 140 uint32_t total_enq_ops = 0; 141 uint16_t ops_unused = 0; 142 uint16_t num_enq = 0; 143 uint16_t num_deq = 0; 144 145 output_size = 0; 146 147 while (remaining_ops > 0) { 148 uint16_t num_ops = RTE_MIN(remaining_ops, 149 test_data->burst_sz); 150 uint16_t ops_needed = num_ops - ops_unused; 151 152 /* 153 * Move the unused operations from the previous 154 * enqueue_burst call to the front, to maintain order 155 */ 156 if ((ops_unused > 0) && (num_enq > 0)) { 157 size_t nb_b_to_mov = 158 ops_unused * sizeof(struct rte_comp_op *); 159 160 memmove(ops, &ops[num_enq], nb_b_to_mov); 161 } 162 163 /* Allocate compression operations */ 164 if (ops_needed && !rte_comp_op_bulk_alloc( 165 mem->op_pool, 166 &ops[ops_unused], 167 ops_needed)) { 168 RTE_LOG(ERR, USER1, 169 "Could not allocate enough operations\n"); 170 res = -1; 171 goto end; 172 } 173 allocated += ops_needed; 174 175 for (i = 0; i < ops_needed; i++) { 176 /* 177 * Calculate next buffer to attach to operation 178 */ 179 uint32_t buf_id = total_enq_ops + i + 180 ops_unused; 181 uint16_t op_id = ops_unused + i; 182 /* Reset all data in output buffers */ 183 struct rte_mbuf *m = output_bufs[buf_id]; 184 185 m->pkt_len = out_seg_sz * m->nb_segs; 186 while (m) { 187 m->data_len = m->buf_len - m->data_off; 188 m = m->next; 189 } 190 ops[op_id]->m_src = input_bufs[buf_id]; 191 ops[op_id]->m_dst = output_bufs[buf_id]; 192 ops[op_id]->src.offset = 0; 193 ops[op_id]->src.length = 194 rte_pktmbuf_pkt_len(input_bufs[buf_id]); 195 ops[op_id]->dst.offset = 0; 196 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL; 197 ops[op_id]->input_chksum = buf_id; 198 ops[op_id]->private_xform = priv_xform; 199 } 200 201 if (unlikely(test_data->perf_comp_force_stop)) 202 goto end; 203 204 num_enq = rte_compressdev_enqueue_burst(dev_id, 205 mem->qp_id, ops, 206 num_ops); 207 if (num_enq == 0) { 208 struct rte_compressdev_stats stats; 209 210 rte_compressdev_stats_get(dev_id, &stats); 211 if (stats.enqueue_err_count) { 212 res = -1; 213 goto end; 214 } 215 } 216 217 ops_unused = num_ops - num_enq; 218 remaining_ops -= num_enq; 219 total_enq_ops += num_enq; 220 221 num_deq = rte_compressdev_dequeue_burst(dev_id, 222 mem->qp_id, 223 deq_ops, 224 test_data->burst_sz); 225 total_deq_ops += num_deq; 226 227 for (i = 0; i < num_deq; i++) { 228 struct rte_comp_op *op = deq_ops[i]; 229 230 if (op->status == 231 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED || 232 op->status == 233 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { 234 RTE_LOG(ERR, USER1, 235 "Out of space error occurred due to incompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); 236 res = -1; 237 goto end; 238 } else if (op->status != 239 RTE_COMP_OP_STATUS_SUCCESS) { 240 RTE_LOG(ERR, USER1, 241 "Some operations were not successful\n"); 242 goto end; 243 } 244 245 const void *read_data_addr = 246 rte_pktmbuf_read(op->m_dst, 0, 247 op->produced, output_data_ptr); 248 if (read_data_addr == NULL) { 249 RTE_LOG(ERR, USER1, 250 "Could not copy buffer in destination\n"); 251 res = -1; 252 goto end; 253 } 254 255 if (read_data_addr != output_data_ptr) 256 rte_memcpy(output_data_ptr, 257 rte_pktmbuf_mtod(op->m_dst, 258 uint8_t *), 259 op->produced); 260 output_data_ptr += op->produced; 261 output_size += op->produced; 262 263 } 264 265 266 if (iter == num_iter - 1) { 267 for (i = 0; i < num_deq; i++) { 268 struct rte_comp_op *op = deq_ops[i]; 269 struct rte_mbuf *m = op->m_dst; 270 271 m->pkt_len = op->produced; 272 uint32_t remaining_data = op->produced; 273 uint16_t data_to_append; 274 275 while (remaining_data > 0) { 276 data_to_append = 277 RTE_MIN(remaining_data, 278 out_seg_sz); 279 m->data_len = data_to_append; 280 remaining_data -= 281 data_to_append; 282 m = m->next; 283 } 284 } 285 } 286 rte_mempool_put_bulk(mem->op_pool, 287 (void **)deq_ops, num_deq); 288 allocated -= num_deq; 289 } 290 291 /* Dequeue the last operations */ 292 while (total_deq_ops < total_ops) { 293 if (unlikely(test_data->perf_comp_force_stop)) 294 goto end; 295 296 num_deq = rte_compressdev_dequeue_burst(dev_id, 297 mem->qp_id, 298 deq_ops, 299 test_data->burst_sz); 300 if (num_deq == 0) { 301 struct rte_compressdev_stats stats; 302 303 rte_compressdev_stats_get(dev_id, &stats); 304 if (stats.dequeue_err_count) { 305 res = -1; 306 goto end; 307 } 308 } 309 310 total_deq_ops += num_deq; 311 312 for (i = 0; i < num_deq; i++) { 313 struct rte_comp_op *op = deq_ops[i]; 314 315 if (op->status == 316 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED || 317 op->status == 318 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { 319 RTE_LOG(ERR, USER1, 320 "Out of space error occurred due to incompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); 321 res = -1; 322 goto end; 323 } else if (op->status != 324 RTE_COMP_OP_STATUS_SUCCESS) { 325 RTE_LOG(ERR, USER1, 326 "Some operations were not successful\n"); 327 goto end; 328 } 329 const void *read_data_addr = 330 rte_pktmbuf_read(op->m_dst, 331 op->dst.offset, 332 op->produced, output_data_ptr); 333 if (read_data_addr == NULL) { 334 RTE_LOG(ERR, USER1, 335 "Could not copy buffer in destination\n"); 336 res = -1; 337 goto end; 338 } 339 340 if (read_data_addr != output_data_ptr) 341 rte_memcpy(output_data_ptr, 342 rte_pktmbuf_mtod( 343 op->m_dst, uint8_t *), 344 op->produced); 345 output_data_ptr += op->produced; 346 output_size += op->produced; 347 348 } 349 350 if (iter == num_iter - 1) { 351 for (i = 0; i < num_deq; i++) { 352 struct rte_comp_op *op = deq_ops[i]; 353 struct rte_mbuf *m = op->m_dst; 354 355 m->pkt_len = op->produced; 356 uint32_t remaining_data = op->produced; 357 uint16_t data_to_append; 358 359 while (remaining_data > 0) { 360 data_to_append = 361 RTE_MIN(remaining_data, 362 out_seg_sz); 363 m->data_len = data_to_append; 364 remaining_data -= 365 data_to_append; 366 m = m->next; 367 } 368 } 369 } 370 rte_mempool_put_bulk(mem->op_pool, 371 (void **)deq_ops, num_deq); 372 allocated -= num_deq; 373 } 374 } 375 376 if (output_data_sz) 377 *output_data_sz = output_size; 378 end: 379 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated); 380 rte_compressdev_private_xform_free(dev_id, priv_xform); 381 rte_free(ops); 382 383 if (test_data->perf_comp_force_stop) { 384 RTE_LOG(ERR, USER1, 385 "lcore: %d Perf. test has been aborted by user\n", 386 mem->lcore_id); 387 res = -1; 388 } 389 390 return res; 391 } 392 393 int 394 cperf_verify_test_runner(void *test_ctx) 395 { 396 struct cperf_verify_ctx *ctx = test_ctx; 397 struct comp_test_data *test_data = ctx->options; 398 int ret = EXIT_SUCCESS; 399 static uint16_t display_once; 400 uint32_t lcore = rte_lcore_id(); 401 uint16_t exp = 0; 402 403 ctx->mem.lcore_id = lcore; 404 405 test_data->ratio = 0; 406 407 if (test_data->test_op & COMPRESS) { 408 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { 409 ret = EXIT_FAILURE; 410 goto end; 411 } 412 } 413 414 if (test_data->test_op & DECOMPRESS) { 415 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { 416 ret = EXIT_FAILURE; 417 goto end; 418 } 419 420 if (!(test_data->test_op & COMPRESS)) { 421 /* 422 * For DECOMPRESS_ONLY mode there is no more 423 * verifications, reset the 'ratio' and 'comp_data_sz' 424 * fields for other tests report. 425 */ 426 ctx->comp_data_sz = 0; 427 ctx->ratio = 0; 428 goto end; 429 } 430 431 if (ctx->decomp_data_sz != test_data->input_data_sz) { 432 RTE_LOG(ERR, USER1, 433 "Decompressed data length not equal to input data length\n"); 434 RTE_LOG(ERR, USER1, 435 "Decompressed size = %zu, expected = %zu\n", 436 ctx->decomp_data_sz, test_data->input_data_sz); 437 ret = EXIT_FAILURE; 438 goto end; 439 } else { 440 if (memcmp(ctx->mem.decompressed_data, 441 test_data->input_data, 442 test_data->input_data_sz) != 0) { 443 RTE_LOG(ERR, USER1, 444 "Decompressed data is not the same as file data\n"); 445 ret = EXIT_FAILURE; 446 goto end; 447 } 448 } 449 } 450 451 ctx->ratio = (double) ctx->comp_data_sz / 452 test_data->input_data_sz * 100; 453 454 if (!ctx->silent) { 455 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 456 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { 457 printf("%12s%6s%12s%17s\n", 458 "lcore id", "Level", "Comp size", "Comp ratio [%]"); 459 } 460 printf("%12u%6u%12zu%17.2f\n", 461 ctx->mem.lcore_id, 462 test_data->level, ctx->comp_data_sz, ctx->ratio); 463 } 464 465 end: 466 return ret; 467 } 468