1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_eal.h>
7 #include <rte_log.h>
8 #include <rte_compressdev.h>
9
10 #include "comp_perf.h"
11 #include "comp_perf_options.h"
12 #include "comp_perf_test_throughput.h"
13 #include "comp_perf_test_cyclecount.h"
14 #include "comp_perf_test_common.h"
15 #include "comp_perf_test_verify.h"
16
17
18 #define DIV_CEIL(a, b) ((a) / (b) + ((a) % (b) != 0))
19
20 struct cperf_buffer_info {
21 uint16_t total_segments;
22 uint16_t segment_sz;
23 uint16_t last_segment_sz;
24 uint32_t total_buffs; /*number of buffers = number of ops*/
25 uint16_t segments_per_buff;
26 uint16_t segments_per_last_buff;
27 size_t input_data_sz;
28 };
29
30 static struct cperf_buffer_info buffer_info;
31
32 int
param_range_check(uint16_t size,const struct rte_param_log2_range * range)33 param_range_check(uint16_t size, const struct rte_param_log2_range *range)
34 {
35 unsigned int next_size;
36
37 /* Check lower/upper bounds */
38 if (size < range->min)
39 return -1;
40
41 if (size > range->max)
42 return -1;
43
44 /* If range is actually only one value, size is correct */
45 if (range->increment == 0)
46 return 0;
47
48 /* Check if value is one of the supported sizes */
49 for (next_size = range->min; next_size <= range->max;
50 next_size += range->increment)
51 if (size == next_size)
52 return 0;
53
54 return -1;
55 }
56
57 static uint32_t
find_buf_size(uint32_t input_size)58 find_buf_size(uint32_t input_size)
59 {
60 uint32_t i;
61
62 /* From performance point of view the buffer size should be a
63 * power of 2 but also should be enough to store incompressible data
64 */
65
66 /* We're looking for nearest power of 2 buffer size, which is greater
67 * than input_size
68 */
69 uint32_t size =
70 !input_size ? MIN_COMPRESSED_BUF_SIZE : (input_size << 1);
71
72 for (i = UINT16_MAX + 1; !(i & size); i >>= 1)
73 ;
74
75 return i > ((UINT16_MAX + 1) >> 1)
76 ? (uint32_t)((float)input_size * EXPANSE_RATIO)
77 : i;
78 }
79
80 void
comp_perf_free_memory(struct comp_test_data * test_data,struct cperf_mem_resources * mem)81 comp_perf_free_memory(struct comp_test_data *test_data,
82 struct cperf_mem_resources *mem)
83 {
84 uint32_t i;
85
86 if (mem->decomp_bufs != NULL)
87 for (i = 0; i < mem->total_bufs; i++)
88 rte_pktmbuf_free(mem->decomp_bufs[i]);
89
90 if (mem->comp_bufs != NULL)
91 for (i = 0; i < mem->total_bufs; i++)
92 rte_pktmbuf_free(mem->comp_bufs[i]);
93
94 rte_free(mem->decomp_bufs);
95 rte_free(mem->comp_bufs);
96 rte_free(mem->decompressed_data);
97 rte_free(mem->compressed_data);
98 rte_mempool_free(mem->op_pool);
99 rte_mempool_free(mem->decomp_buf_pool);
100 rte_mempool_free(mem->comp_buf_pool);
101
102 /* external mbuf support */
103 if (mem->decomp_memzones != NULL) {
104 for (i = 0; i < test_data->total_segs; i++)
105 rte_memzone_free(mem->decomp_memzones[i]);
106 rte_free(mem->decomp_memzones);
107 }
108 if (mem->comp_memzones != NULL) {
109 for (i = 0; i < test_data->total_segs; i++)
110 rte_memzone_free(mem->comp_memzones[i]);
111 rte_free(mem->comp_memzones);
112 }
113 rte_free(mem->decomp_buf_infos);
114 rte_free(mem->comp_buf_infos);
115 }
116
117 static void
comp_perf_extbuf_free_cb(void * addr __rte_unused,void * opaque __rte_unused)118 comp_perf_extbuf_free_cb(void *addr __rte_unused, void *opaque __rte_unused)
119 {
120 }
121
122 static const struct rte_memzone *
comp_perf_make_memzone(const char * name,struct cperf_mem_resources * mem,unsigned int number,size_t size)123 comp_perf_make_memzone(const char *name, struct cperf_mem_resources *mem,
124 unsigned int number, size_t size)
125 {
126 unsigned int socket_id = rte_socket_id();
127 char mz_name[RTE_MEMZONE_NAMESIZE];
128 const struct rte_memzone *memzone;
129
130 snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_s%u_d%u_q%u_%d", name,
131 socket_id, mem->dev_id, mem->qp_id, number);
132 memzone = rte_memzone_lookup(mz_name);
133 if (memzone != NULL && memzone->len != size) {
134 rte_memzone_free(memzone);
135 memzone = NULL;
136 }
137 if (memzone == NULL) {
138 memzone = rte_memzone_reserve_aligned(mz_name, size, socket_id,
139 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
140 if (memzone == NULL)
141 RTE_LOG(ERR, USER1, "Can't allocate memory zone %s\n",
142 mz_name);
143 }
144 return memzone;
145 }
146
147 static int
comp_perf_allocate_external_mbufs(struct comp_test_data * test_data,struct cperf_mem_resources * mem)148 comp_perf_allocate_external_mbufs(struct comp_test_data *test_data,
149 struct cperf_mem_resources *mem)
150 {
151 uint32_t i;
152
153 mem->comp_memzones = rte_zmalloc_socket(NULL,
154 test_data->total_segs * sizeof(struct rte_memzone *),
155 0, rte_socket_id());
156
157 if (mem->comp_memzones == NULL) {
158 RTE_LOG(ERR, USER1,
159 "Memory to hold the compression memzones could not be allocated\n");
160 return -1;
161 }
162
163 mem->decomp_memzones = rte_zmalloc_socket(NULL,
164 test_data->total_segs * sizeof(struct rte_memzone *),
165 0, rte_socket_id());
166
167 if (mem->decomp_memzones == NULL) {
168 RTE_LOG(ERR, USER1,
169 "Memory to hold the decompression memzones could not be allocated\n");
170 return -1;
171 }
172
173 mem->comp_buf_infos = rte_zmalloc_socket(NULL,
174 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info),
175 0, rte_socket_id());
176
177 if (mem->comp_buf_infos == NULL) {
178 RTE_LOG(ERR, USER1,
179 "Memory to hold the compression buf infos could not be allocated\n");
180 return -1;
181 }
182
183 mem->decomp_buf_infos = rte_zmalloc_socket(NULL,
184 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info),
185 0, rte_socket_id());
186
187 if (mem->decomp_buf_infos == NULL) {
188 RTE_LOG(ERR, USER1,
189 "Memory to hold the decompression buf infos could not be allocated\n");
190 return -1;
191 }
192
193 for (i = 0; i < test_data->total_segs; i++) {
194 mem->comp_memzones[i] = comp_perf_make_memzone("comp", mem,
195 i, test_data->out_seg_sz);
196 if (mem->comp_memzones[i] == NULL) {
197 RTE_LOG(ERR, USER1,
198 "Memory to hold the compression memzone could not be allocated\n");
199 return -1;
200 }
201
202 mem->decomp_memzones[i] = comp_perf_make_memzone("decomp", mem,
203 i, test_data->seg_sz);
204 if (mem->decomp_memzones[i] == NULL) {
205 RTE_LOG(ERR, USER1,
206 "Memory to hold the decompression memzone could not be allocated\n");
207 return -1;
208 }
209
210 mem->comp_buf_infos[i].free_cb =
211 comp_perf_extbuf_free_cb;
212 mem->comp_buf_infos[i].fcb_opaque = NULL;
213 rte_mbuf_ext_refcnt_set(&mem->comp_buf_infos[i], 1);
214
215 mem->decomp_buf_infos[i].free_cb =
216 comp_perf_extbuf_free_cb;
217 mem->decomp_buf_infos[i].fcb_opaque = NULL;
218 rte_mbuf_ext_refcnt_set(&mem->decomp_buf_infos[i], 1);
219 }
220
221 return 0;
222 }
223
224 int
comp_perf_allocate_memory(struct comp_test_data * test_data,struct cperf_mem_resources * mem)225 comp_perf_allocate_memory(struct comp_test_data *test_data,
226 struct cperf_mem_resources *mem)
227 {
228 uint16_t comp_mbuf_size;
229 uint16_t decomp_mbuf_size;
230 size_t comp_data_size;
231 size_t decomp_data_size;
232 size_t output_data_sz;
233
234 test_data->out_seg_sz = find_buf_size(test_data->seg_sz);
235
236 if (test_data->test_op & COMPRESS) {
237 /*
238 * Number of segments for input and output
239 * (compression and decompression)
240 */
241 test_data->total_segs = DIV_CEIL(test_data->input_data_sz,
242 test_data->seg_sz);
243 } else {
244 /*
245 * When application does decompression only, input data is
246 * compressed and smaller than the output. The expected size of
247 * uncompressed data given by the user in segment size argument.
248 */
249 test_data->total_segs = test_data->max_sgl_segs;
250 }
251
252 output_data_sz = (size_t) test_data->out_seg_sz * test_data->total_segs;
253 output_data_sz =
254 RTE_MAX(output_data_sz, (size_t) MIN_COMPRESSED_BUF_SIZE);
255
256 if (test_data->use_external_mbufs != 0) {
257 if (comp_perf_allocate_external_mbufs(test_data, mem) < 0)
258 return -1;
259 comp_mbuf_size = 0;
260 decomp_mbuf_size = 0;
261 } else if (test_data->test_op & COMPRESS) {
262 comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM;
263 decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM;
264 } else {
265 comp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM;
266 decomp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM;
267 }
268
269 char pool_name[32] = "";
270
271 snprintf(pool_name, sizeof(pool_name), "comp_buf_pool_%u_qp_%u",
272 mem->dev_id, mem->qp_id);
273 mem->comp_buf_pool = rte_pktmbuf_pool_create(pool_name,
274 test_data->total_segs,
275 0, 0,
276 comp_mbuf_size,
277 rte_socket_id());
278 if (mem->comp_buf_pool == NULL) {
279 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n");
280 return -1;
281 }
282
283 snprintf(pool_name, sizeof(pool_name), "decomp_buf_pool_%u_qp_%u",
284 mem->dev_id, mem->qp_id);
285 mem->decomp_buf_pool = rte_pktmbuf_pool_create(pool_name,
286 test_data->total_segs,
287 0, 0,
288 decomp_mbuf_size,
289 rte_socket_id());
290 if (mem->decomp_buf_pool == NULL) {
291 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n");
292 return -1;
293 }
294
295 mem->total_bufs = DIV_CEIL(test_data->total_segs,
296 test_data->max_sgl_segs);
297
298 snprintf(pool_name, sizeof(pool_name), "op_pool_%u_qp_%u",
299 mem->dev_id, mem->qp_id);
300
301 /* one mempool for both src and dst mbufs */
302 mem->op_pool = rte_comp_op_pool_create(pool_name,
303 mem->total_bufs * 2,
304 0, 0, rte_socket_id());
305 if (mem->op_pool == NULL) {
306 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n");
307 return -1;
308 }
309
310 if (test_data->test_op & COMPRESS) {
311 /*
312 * Compressed data might be a bit larger than input data,
313 * if data cannot be compressed
314 */
315 comp_data_size = output_data_sz;
316 decomp_data_size = test_data->input_data_sz;
317 } else {
318 comp_data_size = test_data->input_data_sz;
319 decomp_data_size = output_data_sz;
320 }
321
322 mem->compressed_data = rte_zmalloc_socket(NULL, comp_data_size, 0,
323 rte_socket_id());
324 if (mem->compressed_data == NULL) {
325 RTE_LOG(ERR, USER1, "Memory to hold the data from the input "
326 "file could not be allocated\n");
327 return -1;
328 }
329
330 mem->decompressed_data = rte_zmalloc_socket(NULL, decomp_data_size, 0,
331 rte_socket_id());
332 if (mem->decompressed_data == NULL) {
333 RTE_LOG(ERR, USER1, "Memory to hold the data from the input "
334 "file could not be allocated\n");
335 return -1;
336 }
337
338 mem->comp_bufs = rte_zmalloc_socket(NULL,
339 mem->total_bufs * sizeof(struct rte_mbuf *),
340 0, rte_socket_id());
341 if (mem->comp_bufs == NULL) {
342 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs"
343 " could not be allocated\n");
344 return -1;
345 }
346
347 mem->decomp_bufs = rte_zmalloc_socket(NULL,
348 mem->total_bufs * sizeof(struct rte_mbuf *),
349 0, rte_socket_id());
350 if (mem->decomp_bufs == NULL) {
351 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs"
352 " could not be allocated\n");
353 return -1;
354 }
355
356 buffer_info.total_segments = test_data->total_segs;
357 buffer_info.segment_sz = test_data->seg_sz;
358 buffer_info.total_buffs = mem->total_bufs;
359 buffer_info.segments_per_buff = test_data->max_sgl_segs;
360 buffer_info.input_data_sz = test_data->input_data_sz;
361
362 return 0;
363 }
364
365 int
prepare_bufs(struct comp_test_data * test_data,struct cperf_mem_resources * mem)366 prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem)
367 {
368 uint32_t remaining_data = test_data->input_data_sz;
369 uint32_t remaining_data_decomp = test_data->input_data_sz;
370 uint8_t *input_data_ptr = test_data->input_data;
371 size_t data_sz = 0;
372 uint8_t *data_addr;
373 uint32_t i, j;
374 uint16_t segs_per_mbuf = 0;
375 uint32_t cmz = 0;
376 uint32_t dmz = 0;
377 bool decompress_only = !!(test_data->test_op == DECOMPRESS);
378
379 for (i = 0; i < mem->total_bufs; i++) {
380 /* Allocate data in input mbuf and copy data from input file */
381 mem->decomp_bufs[i] =
382 rte_pktmbuf_alloc(mem->decomp_buf_pool);
383 if (mem->decomp_bufs[i] == NULL) {
384 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n");
385 return -1;
386 }
387
388 if (test_data->use_external_mbufs != 0) {
389 rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i],
390 mem->decomp_memzones[dmz]->addr,
391 mem->decomp_memzones[dmz]->iova,
392 test_data->seg_sz,
393 &mem->decomp_buf_infos[dmz]);
394 dmz++;
395 }
396
397 if (!decompress_only)
398 data_sz = RTE_MIN(remaining_data, test_data->seg_sz);
399 else
400 data_sz = test_data->out_seg_sz;
401
402 data_addr = (uint8_t *) rte_pktmbuf_append(
403 mem->decomp_bufs[i], data_sz);
404 if (data_addr == NULL) {
405 RTE_LOG(ERR, USER1, "Could not append data\n");
406 return -1;
407 }
408
409 if (!decompress_only) {
410 rte_memcpy(data_addr, input_data_ptr, data_sz);
411 input_data_ptr += data_sz;
412 remaining_data -= data_sz;
413 }
414
415 /* Already one segment in the mbuf */
416 segs_per_mbuf = 1;
417
418 /* Chain mbufs if needed for input mbufs */
419 while (segs_per_mbuf < test_data->max_sgl_segs
420 && remaining_data > 0) {
421 struct rte_mbuf *next_seg =
422 rte_pktmbuf_alloc(mem->decomp_buf_pool);
423
424 if (next_seg == NULL) {
425 RTE_LOG(ERR, USER1,
426 "Could not allocate mbuf\n");
427 return -1;
428 }
429
430 if (test_data->use_external_mbufs != 0) {
431 rte_pktmbuf_attach_extbuf(
432 next_seg,
433 mem->decomp_memzones[dmz]->addr,
434 mem->decomp_memzones[dmz]->iova,
435 test_data->seg_sz,
436 &mem->decomp_buf_infos[dmz]);
437 dmz++;
438 }
439
440 if (!decompress_only)
441 data_sz = RTE_MIN(remaining_data,
442 test_data->seg_sz);
443 else
444 data_sz = test_data->out_seg_sz;
445
446 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg,
447 data_sz);
448
449 if (data_addr == NULL) {
450 RTE_LOG(ERR, USER1, "Could not append data\n");
451 return -1;
452 }
453
454 if (!decompress_only) {
455 rte_memcpy(data_addr, input_data_ptr, data_sz);
456 input_data_ptr += data_sz;
457 remaining_data -= data_sz;
458 }
459
460 if (rte_pktmbuf_chain(mem->decomp_bufs[i],
461 next_seg) < 0) {
462 RTE_LOG(ERR, USER1, "Could not chain mbufs\n");
463 return -1;
464 }
465 segs_per_mbuf++;
466 }
467
468 /* Allocate data in output mbuf */
469 mem->comp_bufs[i] =
470 rte_pktmbuf_alloc(mem->comp_buf_pool);
471 if (mem->comp_bufs[i] == NULL) {
472 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n");
473 return -1;
474 }
475
476 if (test_data->use_external_mbufs != 0) {
477 rte_pktmbuf_attach_extbuf(mem->comp_bufs[i],
478 mem->comp_memzones[cmz]->addr,
479 mem->comp_memzones[cmz]->iova,
480 test_data->out_seg_sz,
481 &mem->comp_buf_infos[cmz]);
482 cmz++;
483 }
484
485 if (decompress_only)
486 data_sz = RTE_MIN(remaining_data_decomp, test_data->seg_sz);
487 else
488 data_sz = test_data->out_seg_sz;
489
490 data_addr = (uint8_t *) rte_pktmbuf_append(mem->comp_bufs[i],
491 data_sz);
492 if (data_addr == NULL) {
493 RTE_LOG(ERR, USER1, "Could not append data\n");
494 return -1;
495 }
496
497 if (decompress_only) {
498 rte_memcpy(data_addr, input_data_ptr, data_sz);
499 input_data_ptr += data_sz;
500 remaining_data_decomp -= data_sz;
501 }
502
503 /* Chain mbufs if needed for output mbufs */
504 for (j = 1; j < segs_per_mbuf && remaining_data_decomp > 0; j++) {
505 struct rte_mbuf *next_seg =
506 rte_pktmbuf_alloc(mem->comp_buf_pool);
507
508 if (next_seg == NULL) {
509 RTE_LOG(ERR, USER1,
510 "Could not allocate mbuf\n");
511 return -1;
512 }
513
514 if (test_data->use_external_mbufs != 0) {
515 rte_pktmbuf_attach_extbuf(
516 next_seg,
517 mem->comp_memzones[cmz]->addr,
518 mem->comp_memzones[cmz]->iova,
519 test_data->out_seg_sz,
520 &mem->comp_buf_infos[cmz]);
521 cmz++;
522 }
523
524 if (decompress_only)
525 data_sz = RTE_MIN(remaining_data_decomp,
526 test_data->seg_sz);
527 else
528 data_sz = test_data->out_seg_sz;
529
530 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg,
531 data_sz);
532 if (data_addr == NULL) {
533 RTE_LOG(ERR, USER1, "Could not append data\n");
534 return -1;
535 }
536
537 if (decompress_only) {
538 rte_memcpy(data_addr, input_data_ptr, data_sz);
539 input_data_ptr += data_sz;
540 remaining_data_decomp -= data_sz;
541 }
542
543 if (rte_pktmbuf_chain(mem->comp_bufs[i],
544 next_seg) < 0) {
545 RTE_LOG(ERR, USER1, "Could not chain mbufs\n");
546 return -1;
547 }
548 }
549 }
550
551 buffer_info.segments_per_last_buff = segs_per_mbuf;
552 buffer_info.last_segment_sz = data_sz;
553
554 return 0;
555 }
556
557 void
print_test_dynamics(const struct comp_test_data * test_data)558 print_test_dynamics(const struct comp_test_data *test_data)
559 {
560 uint32_t opt_total_segs = DIV_CEIL(buffer_info.input_data_sz,
561 MAX_SEG_SIZE);
562
563 if (buffer_info.total_buffs > 1) {
564 if (test_data->test == CPERF_TEST_TYPE_THROUGHPUT) {
565 printf("\nWarning: for the current input parameters, number"
566 " of ops is higher than one, which may result"
567 " in sub-optimal performance.\n");
568 printf("To improve the performance (for the current"
569 " input data) following parameters are"
570 " suggested:\n");
571 printf(" * Segment size: %d\n",
572 MAX_SEG_SIZE);
573 printf(" * Number of segments: %u\n",
574 opt_total_segs);
575 }
576 } else if (buffer_info.total_buffs == 1) {
577 printf("\nInfo: there is only one op with %u segments -"
578 " the compression ratio is the best.\n",
579 buffer_info.segments_per_last_buff);
580 if (buffer_info.segment_sz < MAX_SEG_SIZE)
581 printf("To reduce compression time, please use"
582 " bigger segment size: %d.\n",
583 MAX_SEG_SIZE);
584 else if (buffer_info.segment_sz == MAX_SEG_SIZE)
585 printf("Segment size is optimal for the best"
586 " performance.\n");
587 } else
588 printf("Warning: something wrong happened!!\n");
589
590 printf("\nFor the current input parameters (segment size = %u,"
591 " maximum segments per SGL = %u):\n",
592 buffer_info.segment_sz,
593 buffer_info.segments_per_buff);
594 printf(" * Total number of buffers: %d\n",
595 buffer_info.total_segments);
596 printf(" * %u buffer(s) %u bytes long, last buffer %u"
597 " byte(s) long\n",
598 buffer_info.total_segments - 1,
599 buffer_info.segment_sz,
600 buffer_info.last_segment_sz);
601 printf(" * Number of ops: %u\n", buffer_info.total_buffs);
602 printf(" * Total memory allocation: %u\n",
603 (buffer_info.total_segments - 1) * buffer_info.segment_sz
604 + buffer_info.last_segment_sz);
605 if (buffer_info.total_buffs > 1)
606 printf(" * %u ops: %u segment(s) in each,"
607 " segment size %u\n",
608 buffer_info.total_buffs - 1,
609 buffer_info.segments_per_buff,
610 buffer_info.segment_sz);
611 if (buffer_info.segments_per_last_buff > 1) {
612 printf(" * 1 op %u segments:\n",
613 buffer_info.segments_per_last_buff);
614 printf(" o %u segment size %u\n",
615 buffer_info.segments_per_last_buff - 1,
616 buffer_info.segment_sz);
617 printf(" o last segment size %u\n",
618 buffer_info.last_segment_sz);
619 } else if (buffer_info.segments_per_last_buff == 1) {
620 printf(" * 1 op (the last one): %u segment %u"
621 " byte(s) long\n\n",
622 buffer_info.segments_per_last_buff,
623 buffer_info.last_segment_sz);
624 }
625 printf("\n");
626 }
627