1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_eal.h> 7 #include <rte_log.h> 8 #include <rte_compressdev.h> 9 10 #include "comp_perf_test_verify.h" 11 #include "comp_perf_test_common.h" 12 13 void 14 cperf_verify_test_destructor(void *arg) 15 { 16 if (arg) { 17 comp_perf_free_memory( 18 ((struct cperf_verify_ctx *)arg)->options, 19 &((struct cperf_verify_ctx *)arg)->mem); 20 rte_free(arg); 21 } 22 } 23 24 void * 25 cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, 26 struct comp_test_data *options) 27 { 28 struct cperf_verify_ctx *ctx = NULL; 29 30 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 31 32 if (ctx == NULL) 33 return NULL; 34 35 ctx->mem.dev_id = dev_id; 36 ctx->mem.qp_id = qp_id; 37 ctx->options = options; 38 39 if (!comp_perf_allocate_memory(ctx->options, &ctx->mem) && 40 !prepare_bufs(ctx->options, &ctx->mem)) 41 return ctx; 42 43 cperf_verify_test_destructor(ctx); 44 return NULL; 45 } 46 47 static int 48 main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type) 49 { 50 struct comp_test_data *test_data = ctx->options; 51 uint8_t *output_data_ptr = NULL; 52 size_t *output_data_sz = NULL; 53 struct cperf_mem_resources *mem = &ctx->mem; 54 55 uint8_t dev_id = mem->dev_id; 56 uint32_t i, iter, num_iter; 57 struct rte_comp_op **ops, **deq_ops; 58 void *priv_xform = NULL; 59 struct rte_comp_xform xform; 60 size_t output_size = 0; 61 struct rte_mbuf **input_bufs, **output_bufs; 62 int res = 0; 63 int allocated = 0; 64 uint32_t out_seg_sz; 65 66 if (test_data == NULL || !test_data->burst_sz) { 67 RTE_LOG(ERR, USER1, 68 "Unknown burst size\n"); 69 return -1; 70 } 71 72 ops = rte_zmalloc_socket(NULL, 73 2 * mem->total_bufs * sizeof(struct rte_comp_op *), 74 0, rte_socket_id()); 75 76 if (ops == NULL) { 77 RTE_LOG(ERR, USER1, 78 "Can't allocate memory for ops strucures\n"); 79 return -1; 80 } 81 82 deq_ops = &ops[mem->total_bufs]; 83 84 if (type == RTE_COMP_COMPRESS) { 85 xform = (struct rte_comp_xform) { 86 .type = RTE_COMP_COMPRESS, 87 .compress = { 88 .algo = RTE_COMP_ALGO_DEFLATE, 89 .deflate.huffman = test_data->huffman_enc, 90 .level = test_data->level, 91 .window_size = test_data->window_sz, 92 .chksum = RTE_COMP_CHECKSUM_NONE, 93 .hash_algo = RTE_COMP_HASH_ALGO_NONE 94 } 95 }; 96 output_data_ptr = ctx->mem.compressed_data; 97 output_data_sz = &ctx->comp_data_sz; 98 input_bufs = mem->decomp_bufs; 99 output_bufs = mem->comp_bufs; 100 out_seg_sz = test_data->out_seg_sz; 101 } else { 102 xform = (struct rte_comp_xform) { 103 .type = RTE_COMP_DECOMPRESS, 104 .decompress = { 105 .algo = RTE_COMP_ALGO_DEFLATE, 106 .chksum = RTE_COMP_CHECKSUM_NONE, 107 .window_size = test_data->window_sz, 108 .hash_algo = RTE_COMP_HASH_ALGO_NONE 109 } 110 }; 111 output_data_ptr = ctx->mem.decompressed_data; 112 output_data_sz = &ctx->decomp_data_sz; 113 input_bufs = mem->comp_bufs; 114 output_bufs = mem->decomp_bufs; 115 out_seg_sz = test_data->seg_sz; 116 } 117 118 /* Create private xform */ 119 if (rte_compressdev_private_xform_create(dev_id, &xform, 120 &priv_xform) < 0) { 121 RTE_LOG(ERR, USER1, "Private xform could not be created\n"); 122 res = -1; 123 goto end; 124 } 125 126 num_iter = 1; 127 128 for (iter = 0; iter < num_iter; iter++) { 129 uint32_t total_ops = mem->total_bufs; 130 uint32_t remaining_ops = mem->total_bufs; 131 uint32_t total_deq_ops = 0; 132 uint32_t total_enq_ops = 0; 133 uint16_t ops_unused = 0; 134 uint16_t num_enq = 0; 135 uint16_t num_deq = 0; 136 137 output_size = 0; 138 139 while (remaining_ops > 0) { 140 uint16_t num_ops = RTE_MIN(remaining_ops, 141 test_data->burst_sz); 142 uint16_t ops_needed = num_ops - ops_unused; 143 144 /* 145 * Move the unused operations from the previous 146 * enqueue_burst call to the front, to maintain order 147 */ 148 if ((ops_unused > 0) && (num_enq > 0)) { 149 size_t nb_b_to_mov = 150 ops_unused * sizeof(struct rte_comp_op *); 151 152 memmove(ops, &ops[num_enq], nb_b_to_mov); 153 } 154 155 /* Allocate compression operations */ 156 if (ops_needed && !rte_comp_op_bulk_alloc( 157 mem->op_pool, 158 &ops[ops_unused], 159 ops_needed)) { 160 RTE_LOG(ERR, USER1, 161 "Could not allocate enough operations\n"); 162 res = -1; 163 goto end; 164 } 165 allocated += ops_needed; 166 167 for (i = 0; i < ops_needed; i++) { 168 /* 169 * Calculate next buffer to attach to operation 170 */ 171 uint32_t buf_id = total_enq_ops + i + 172 ops_unused; 173 uint16_t op_id = ops_unused + i; 174 /* Reset all data in output buffers */ 175 struct rte_mbuf *m = output_bufs[buf_id]; 176 177 m->pkt_len = out_seg_sz * m->nb_segs; 178 while (m) { 179 m->data_len = m->buf_len - m->data_off; 180 m = m->next; 181 } 182 ops[op_id]->m_src = input_bufs[buf_id]; 183 ops[op_id]->m_dst = output_bufs[buf_id]; 184 ops[op_id]->src.offset = 0; 185 ops[op_id]->src.length = 186 rte_pktmbuf_pkt_len(input_bufs[buf_id]); 187 ops[op_id]->dst.offset = 0; 188 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL; 189 ops[op_id]->input_chksum = buf_id; 190 ops[op_id]->private_xform = priv_xform; 191 } 192 193 if (unlikely(test_data->perf_comp_force_stop)) 194 goto end; 195 196 num_enq = rte_compressdev_enqueue_burst(dev_id, 197 mem->qp_id, ops, 198 num_ops); 199 if (num_enq == 0) { 200 struct rte_compressdev_stats stats; 201 202 rte_compressdev_stats_get(dev_id, &stats); 203 if (stats.enqueue_err_count) { 204 res = -1; 205 goto end; 206 } 207 } 208 209 ops_unused = num_ops - num_enq; 210 remaining_ops -= num_enq; 211 total_enq_ops += num_enq; 212 213 num_deq = rte_compressdev_dequeue_burst(dev_id, 214 mem->qp_id, 215 deq_ops, 216 test_data->burst_sz); 217 total_deq_ops += num_deq; 218 219 for (i = 0; i < num_deq; i++) { 220 struct rte_comp_op *op = deq_ops[i]; 221 222 if (op->status == 223 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED || 224 op->status == 225 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { 226 RTE_LOG(ERR, USER1, 227 "Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); 228 res = -1; 229 goto end; 230 } else if (op->status != 231 RTE_COMP_OP_STATUS_SUCCESS) { 232 RTE_LOG(ERR, USER1, 233 "Some operations were not successful\n"); 234 goto end; 235 } 236 237 const void *read_data_addr = 238 rte_pktmbuf_read(op->m_dst, 0, 239 op->produced, output_data_ptr); 240 if (read_data_addr == NULL) { 241 RTE_LOG(ERR, USER1, 242 "Could not copy buffer in destination\n"); 243 res = -1; 244 goto end; 245 } 246 247 if (read_data_addr != output_data_ptr) 248 rte_memcpy(output_data_ptr, 249 rte_pktmbuf_mtod(op->m_dst, 250 uint8_t *), 251 op->produced); 252 output_data_ptr += op->produced; 253 output_size += op->produced; 254 255 } 256 257 258 if (iter == num_iter - 1) { 259 for (i = 0; i < num_deq; i++) { 260 struct rte_comp_op *op = deq_ops[i]; 261 struct rte_mbuf *m = op->m_dst; 262 263 m->pkt_len = op->produced; 264 uint32_t remaining_data = op->produced; 265 uint16_t data_to_append; 266 267 while (remaining_data > 0) { 268 data_to_append = 269 RTE_MIN(remaining_data, 270 out_seg_sz); 271 m->data_len = data_to_append; 272 remaining_data -= 273 data_to_append; 274 m = m->next; 275 } 276 } 277 } 278 rte_mempool_put_bulk(mem->op_pool, 279 (void **)deq_ops, num_deq); 280 allocated -= num_deq; 281 } 282 283 /* Dequeue the last operations */ 284 while (total_deq_ops < total_ops) { 285 if (unlikely(test_data->perf_comp_force_stop)) 286 goto end; 287 288 num_deq = rte_compressdev_dequeue_burst(dev_id, 289 mem->qp_id, 290 deq_ops, 291 test_data->burst_sz); 292 if (num_deq == 0) { 293 struct rte_compressdev_stats stats; 294 295 rte_compressdev_stats_get(dev_id, &stats); 296 if (stats.dequeue_err_count) { 297 res = -1; 298 goto end; 299 } 300 } 301 302 total_deq_ops += num_deq; 303 304 for (i = 0; i < num_deq; i++) { 305 struct rte_comp_op *op = deq_ops[i]; 306 307 if (op->status == 308 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED || 309 op->status == 310 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { 311 RTE_LOG(ERR, USER1, 312 "Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); 313 res = -1; 314 goto end; 315 } else if (op->status != 316 RTE_COMP_OP_STATUS_SUCCESS) { 317 RTE_LOG(ERR, USER1, 318 "Some operations were not successful\n"); 319 goto end; 320 } 321 const void *read_data_addr = 322 rte_pktmbuf_read(op->m_dst, 323 op->dst.offset, 324 op->produced, output_data_ptr); 325 if (read_data_addr == NULL) { 326 RTE_LOG(ERR, USER1, 327 "Could not copy buffer in destination\n"); 328 res = -1; 329 goto end; 330 } 331 332 if (read_data_addr != output_data_ptr) 333 rte_memcpy(output_data_ptr, 334 rte_pktmbuf_mtod( 335 op->m_dst, uint8_t *), 336 op->produced); 337 output_data_ptr += op->produced; 338 output_size += op->produced; 339 340 } 341 342 if (iter == num_iter - 1) { 343 for (i = 0; i < num_deq; i++) { 344 struct rte_comp_op *op = deq_ops[i]; 345 struct rte_mbuf *m = op->m_dst; 346 347 m->pkt_len = op->produced; 348 uint32_t remaining_data = op->produced; 349 uint16_t data_to_append; 350 351 while (remaining_data > 0) { 352 data_to_append = 353 RTE_MIN(remaining_data, 354 out_seg_sz); 355 m->data_len = data_to_append; 356 remaining_data -= 357 data_to_append; 358 m = m->next; 359 } 360 } 361 } 362 rte_mempool_put_bulk(mem->op_pool, 363 (void **)deq_ops, num_deq); 364 allocated -= num_deq; 365 } 366 } 367 368 if (output_data_sz) 369 *output_data_sz = output_size; 370 end: 371 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated); 372 rte_compressdev_private_xform_free(dev_id, priv_xform); 373 rte_free(ops); 374 375 if (test_data->perf_comp_force_stop) { 376 RTE_LOG(ERR, USER1, 377 "lcore: %d Perf. test has been aborted by user\n", 378 mem->lcore_id); 379 res = -1; 380 } 381 382 return res; 383 } 384 385 int 386 cperf_verify_test_runner(void *test_ctx) 387 { 388 struct cperf_verify_ctx *ctx = test_ctx; 389 struct comp_test_data *test_data = ctx->options; 390 int ret = EXIT_SUCCESS; 391 static uint16_t display_once; 392 uint32_t lcore = rte_lcore_id(); 393 394 ctx->mem.lcore_id = lcore; 395 396 test_data->ratio = 0; 397 398 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { 399 ret = EXIT_FAILURE; 400 goto end; 401 } 402 403 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { 404 ret = EXIT_FAILURE; 405 goto end; 406 } 407 408 if (ctx->decomp_data_sz != test_data->input_data_sz) { 409 RTE_LOG(ERR, USER1, 410 "Decompressed data length not equal to input data length\n"); 411 RTE_LOG(ERR, USER1, 412 "Decompressed size = %zu, expected = %zu\n", 413 ctx->decomp_data_sz, test_data->input_data_sz); 414 ret = EXIT_FAILURE; 415 goto end; 416 } else { 417 if (memcmp(ctx->mem.decompressed_data, 418 test_data->input_data, 419 test_data->input_data_sz) != 0) { 420 RTE_LOG(ERR, USER1, 421 "Decompressed data is not the same as file data\n"); 422 ret = EXIT_FAILURE; 423 goto end; 424 } 425 } 426 427 ctx->ratio = (double) ctx->comp_data_sz / 428 test_data->input_data_sz * 100; 429 430 uint16_t exp = 0; 431 if (!ctx->silent) { 432 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 433 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { 434 printf("%12s%6s%12s%17s\n", 435 "lcore id", "Level", "Comp size", "Comp ratio [%]"); 436 } 437 printf("%12u%6u%12zu%17.2f\n", 438 ctx->mem.lcore_id, 439 test_data->level, ctx->comp_data_sz, ctx->ratio); 440 } 441 442 end: 443 return ret; 444 } 445