1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_eal.h> 7 #include <rte_log.h> 8 #include <rte_compressdev.h> 9 10 #include "comp_perf.h" 11 #include "comp_perf_options.h" 12 #include "comp_perf_test_throughput.h" 13 #include "comp_perf_test_cyclecount.h" 14 #include "comp_perf_test_common.h" 15 #include "comp_perf_test_verify.h" 16 17 18 #define DIV_CEIL(a, b) ((a) / (b) + ((a) % (b) != 0)) 19 20 struct cperf_buffer_info { 21 uint16_t total_segments; 22 uint16_t segment_sz; 23 uint16_t last_segment_sz; 24 uint32_t total_buffs; /*number of buffers = number of ops*/ 25 uint16_t segments_per_buff; 26 uint16_t segments_per_last_buff; 27 size_t input_data_sz; 28 }; 29 30 static struct cperf_buffer_info buffer_info; 31 32 int 33 param_range_check(uint16_t size, const struct rte_param_log2_range *range) 34 { 35 unsigned int next_size; 36 37 /* Check lower/upper bounds */ 38 if (size < range->min) 39 return -1; 40 41 if (size > range->max) 42 return -1; 43 44 /* If range is actually only one value, size is correct */ 45 if (range->increment == 0) 46 return 0; 47 48 /* Check if value is one of the supported sizes */ 49 for (next_size = range->min; next_size <= range->max; 50 next_size += range->increment) 51 if (size == next_size) 52 return 0; 53 54 return -1; 55 } 56 57 static uint32_t 58 find_buf_size(uint32_t input_size) 59 { 60 uint32_t i; 61 62 /* From performance point of view the buffer size should be a 63 * power of 2 but also should be enough to store incompressible data 64 */ 65 66 /* We're looking for nearest power of 2 buffer size, which is greater 67 * than input_size 68 */ 69 uint32_t size = 70 !input_size ? MIN_COMPRESSED_BUF_SIZE : (input_size << 1); 71 72 for (i = UINT16_MAX + 1; !(i & size); i >>= 1) 73 ; 74 75 return i > ((UINT16_MAX + 1) >> 1) 76 ? (uint32_t)((float)input_size * EXPANSE_RATIO) 77 : i; 78 } 79 80 void 81 comp_perf_free_memory(struct comp_test_data *test_data, 82 struct cperf_mem_resources *mem) 83 { 84 uint32_t i; 85 86 if (mem->decomp_bufs != NULL) 87 for (i = 0; i < mem->total_bufs; i++) 88 rte_pktmbuf_free(mem->decomp_bufs[i]); 89 90 if (mem->comp_bufs != NULL) 91 for (i = 0; i < mem->total_bufs; i++) 92 rte_pktmbuf_free(mem->comp_bufs[i]); 93 94 rte_free(mem->decomp_bufs); 95 rte_free(mem->comp_bufs); 96 rte_free(mem->decompressed_data); 97 rte_free(mem->compressed_data); 98 rte_mempool_free(mem->op_pool); 99 rte_mempool_free(mem->decomp_buf_pool); 100 rte_mempool_free(mem->comp_buf_pool); 101 102 /* external mbuf support */ 103 if (mem->decomp_memzones != NULL) { 104 for (i = 0; i < test_data->total_segs; i++) 105 rte_memzone_free(mem->decomp_memzones[i]); 106 rte_free(mem->decomp_memzones); 107 } 108 if (mem->comp_memzones != NULL) { 109 for (i = 0; i < test_data->total_segs; i++) 110 rte_memzone_free(mem->comp_memzones[i]); 111 rte_free(mem->comp_memzones); 112 } 113 rte_free(mem->decomp_buf_infos); 114 rte_free(mem->comp_buf_infos); 115 } 116 117 static void 118 comp_perf_extbuf_free_cb(void *addr __rte_unused, void *opaque __rte_unused) 119 { 120 } 121 122 static const struct rte_memzone * 123 comp_perf_make_memzone(const char *name, struct cperf_mem_resources *mem, 124 unsigned int number, size_t size) 125 { 126 unsigned int socket_id = rte_socket_id(); 127 char mz_name[RTE_MEMZONE_NAMESIZE]; 128 const struct rte_memzone *memzone; 129 130 snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_s%u_d%u_q%u_%d", name, 131 socket_id, mem->dev_id, mem->qp_id, number); 132 memzone = rte_memzone_lookup(mz_name); 133 if (memzone != NULL && memzone->len != size) { 134 rte_memzone_free(memzone); 135 memzone = NULL; 136 } 137 if (memzone == NULL) { 138 memzone = rte_memzone_reserve_aligned(mz_name, size, socket_id, 139 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); 140 if (memzone == NULL) 141 RTE_LOG(ERR, USER1, "Can't allocate memory zone %s\n", 142 mz_name); 143 } 144 return memzone; 145 } 146 147 static int 148 comp_perf_allocate_external_mbufs(struct comp_test_data *test_data, 149 struct cperf_mem_resources *mem) 150 { 151 uint32_t i; 152 153 mem->comp_memzones = rte_zmalloc_socket(NULL, 154 test_data->total_segs * sizeof(struct rte_memzone *), 155 0, rte_socket_id()); 156 157 if (mem->comp_memzones == NULL) { 158 RTE_LOG(ERR, USER1, 159 "Memory to hold the compression memzones could not be allocated\n"); 160 return -1; 161 } 162 163 mem->decomp_memzones = rte_zmalloc_socket(NULL, 164 test_data->total_segs * sizeof(struct rte_memzone *), 165 0, rte_socket_id()); 166 167 if (mem->decomp_memzones == NULL) { 168 RTE_LOG(ERR, USER1, 169 "Memory to hold the decompression memzones could not be allocated\n"); 170 return -1; 171 } 172 173 mem->comp_buf_infos = rte_zmalloc_socket(NULL, 174 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 175 0, rte_socket_id()); 176 177 if (mem->comp_buf_infos == NULL) { 178 RTE_LOG(ERR, USER1, 179 "Memory to hold the compression buf infos could not be allocated\n"); 180 return -1; 181 } 182 183 mem->decomp_buf_infos = rte_zmalloc_socket(NULL, 184 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 185 0, rte_socket_id()); 186 187 if (mem->decomp_buf_infos == NULL) { 188 RTE_LOG(ERR, USER1, 189 "Memory to hold the decompression buf infos could not be allocated\n"); 190 return -1; 191 } 192 193 for (i = 0; i < test_data->total_segs; i++) { 194 mem->comp_memzones[i] = comp_perf_make_memzone("comp", mem, 195 i, test_data->out_seg_sz); 196 if (mem->comp_memzones[i] == NULL) { 197 RTE_LOG(ERR, USER1, 198 "Memory to hold the compression memzone could not be allocated\n"); 199 return -1; 200 } 201 202 mem->decomp_memzones[i] = comp_perf_make_memzone("decomp", mem, 203 i, test_data->seg_sz); 204 if (mem->decomp_memzones[i] == NULL) { 205 RTE_LOG(ERR, USER1, 206 "Memory to hold the decompression memzone could not be allocated\n"); 207 return -1; 208 } 209 210 mem->comp_buf_infos[i].free_cb = 211 comp_perf_extbuf_free_cb; 212 mem->comp_buf_infos[i].fcb_opaque = NULL; 213 rte_mbuf_ext_refcnt_set(&mem->comp_buf_infos[i], 1); 214 215 mem->decomp_buf_infos[i].free_cb = 216 comp_perf_extbuf_free_cb; 217 mem->decomp_buf_infos[i].fcb_opaque = NULL; 218 rte_mbuf_ext_refcnt_set(&mem->decomp_buf_infos[i], 1); 219 } 220 221 return 0; 222 } 223 224 int 225 comp_perf_allocate_memory(struct comp_test_data *test_data, 226 struct cperf_mem_resources *mem) 227 { 228 uint16_t comp_mbuf_size; 229 uint16_t decomp_mbuf_size; 230 size_t comp_data_size; 231 size_t decomp_data_size; 232 size_t output_data_sz; 233 234 test_data->out_seg_sz = find_buf_size(test_data->seg_sz); 235 236 if (test_data->test_op & COMPRESS) { 237 /* 238 * Number of segments for input and output 239 * (compression and decompression) 240 */ 241 test_data->total_segs = DIV_CEIL(test_data->input_data_sz, 242 test_data->seg_sz); 243 } else { 244 /* 245 * When application does decompression only, input data is 246 * compressed and smaller than the output. The expected size of 247 * uncompressed data given by the user in segment size argument. 248 */ 249 test_data->total_segs = test_data->max_sgl_segs; 250 } 251 252 output_data_sz = (size_t) test_data->out_seg_sz * test_data->total_segs; 253 output_data_sz = 254 RTE_MAX(output_data_sz, (size_t) MIN_COMPRESSED_BUF_SIZE); 255 256 if (test_data->use_external_mbufs != 0) { 257 if (comp_perf_allocate_external_mbufs(test_data, mem) < 0) 258 return -1; 259 comp_mbuf_size = 0; 260 decomp_mbuf_size = 0; 261 } else if (test_data->test_op & COMPRESS) { 262 comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 263 decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 264 } else { 265 comp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 266 decomp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 267 } 268 269 char pool_name[32] = ""; 270 271 snprintf(pool_name, sizeof(pool_name), "comp_buf_pool_%u_qp_%u", 272 mem->dev_id, mem->qp_id); 273 mem->comp_buf_pool = rte_pktmbuf_pool_create(pool_name, 274 test_data->total_segs, 275 0, 0, 276 comp_mbuf_size, 277 rte_socket_id()); 278 if (mem->comp_buf_pool == NULL) { 279 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 280 return -1; 281 } 282 283 snprintf(pool_name, sizeof(pool_name), "decomp_buf_pool_%u_qp_%u", 284 mem->dev_id, mem->qp_id); 285 mem->decomp_buf_pool = rte_pktmbuf_pool_create(pool_name, 286 test_data->total_segs, 287 0, 0, 288 decomp_mbuf_size, 289 rte_socket_id()); 290 if (mem->decomp_buf_pool == NULL) { 291 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 292 return -1; 293 } 294 295 mem->total_bufs = DIV_CEIL(test_data->total_segs, 296 test_data->max_sgl_segs); 297 298 snprintf(pool_name, sizeof(pool_name), "op_pool_%u_qp_%u", 299 mem->dev_id, mem->qp_id); 300 301 /* one mempool for both src and dst mbufs */ 302 mem->op_pool = rte_comp_op_pool_create(pool_name, 303 mem->total_bufs * 2, 304 0, 0, rte_socket_id()); 305 if (mem->op_pool == NULL) { 306 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n"); 307 return -1; 308 } 309 310 if (test_data->test_op & COMPRESS) { 311 /* 312 * Compressed data might be a bit larger than input data, 313 * if data cannot be compressed 314 */ 315 comp_data_size = output_data_sz; 316 decomp_data_size = test_data->input_data_sz; 317 } else { 318 comp_data_size = test_data->input_data_sz; 319 decomp_data_size = output_data_sz; 320 } 321 322 mem->compressed_data = rte_zmalloc_socket(NULL, comp_data_size, 0, 323 rte_socket_id()); 324 if (mem->compressed_data == NULL) { 325 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 326 "file could not be allocated\n"); 327 return -1; 328 } 329 330 mem->decompressed_data = rte_zmalloc_socket(NULL, decomp_data_size, 0, 331 rte_socket_id()); 332 if (mem->decompressed_data == NULL) { 333 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 334 "file could not be allocated\n"); 335 return -1; 336 } 337 338 mem->comp_bufs = rte_zmalloc_socket(NULL, 339 mem->total_bufs * sizeof(struct rte_mbuf *), 340 0, rte_socket_id()); 341 if (mem->comp_bufs == NULL) { 342 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs" 343 " could not be allocated\n"); 344 return -1; 345 } 346 347 mem->decomp_bufs = rte_zmalloc_socket(NULL, 348 mem->total_bufs * sizeof(struct rte_mbuf *), 349 0, rte_socket_id()); 350 if (mem->decomp_bufs == NULL) { 351 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs" 352 " could not be allocated\n"); 353 return -1; 354 } 355 356 buffer_info.total_segments = test_data->total_segs; 357 buffer_info.segment_sz = test_data->seg_sz; 358 buffer_info.total_buffs = mem->total_bufs; 359 buffer_info.segments_per_buff = test_data->max_sgl_segs; 360 buffer_info.input_data_sz = test_data->input_data_sz; 361 362 return 0; 363 } 364 365 int 366 prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) 367 { 368 uint32_t remaining_data = test_data->input_data_sz; 369 uint32_t remaining_data_decomp = test_data->input_data_sz; 370 uint8_t *input_data_ptr = test_data->input_data; 371 size_t data_sz = 0; 372 uint8_t *data_addr; 373 uint32_t i, j; 374 uint16_t segs_per_mbuf = 0; 375 uint32_t cmz = 0; 376 uint32_t dmz = 0; 377 bool decompress_only = !!(test_data->test_op == DECOMPRESS); 378 379 for (i = 0; i < mem->total_bufs; i++) { 380 /* Allocate data in input mbuf and copy data from input file */ 381 mem->decomp_bufs[i] = 382 rte_pktmbuf_alloc(mem->decomp_buf_pool); 383 if (mem->decomp_bufs[i] == NULL) { 384 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 385 return -1; 386 } 387 388 if (test_data->use_external_mbufs != 0) { 389 rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i], 390 mem->decomp_memzones[dmz]->addr, 391 mem->decomp_memzones[dmz]->iova, 392 test_data->seg_sz, 393 &mem->decomp_buf_infos[dmz]); 394 dmz++; 395 } 396 397 if (!decompress_only) 398 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 399 else 400 data_sz = test_data->out_seg_sz; 401 402 data_addr = (uint8_t *) rte_pktmbuf_append( 403 mem->decomp_bufs[i], data_sz); 404 if (data_addr == NULL) { 405 RTE_LOG(ERR, USER1, "Could not append data\n"); 406 return -1; 407 } 408 409 if (!decompress_only) { 410 rte_memcpy(data_addr, input_data_ptr, data_sz); 411 input_data_ptr += data_sz; 412 remaining_data -= data_sz; 413 } 414 415 /* Already one segment in the mbuf */ 416 segs_per_mbuf = 1; 417 418 /* Chain mbufs if needed for input mbufs */ 419 while (segs_per_mbuf < test_data->max_sgl_segs 420 && remaining_data > 0) { 421 struct rte_mbuf *next_seg = 422 rte_pktmbuf_alloc(mem->decomp_buf_pool); 423 424 if (next_seg == NULL) { 425 RTE_LOG(ERR, USER1, 426 "Could not allocate mbuf\n"); 427 return -1; 428 } 429 430 if (test_data->use_external_mbufs != 0) { 431 rte_pktmbuf_attach_extbuf( 432 next_seg, 433 mem->decomp_memzones[dmz]->addr, 434 mem->decomp_memzones[dmz]->iova, 435 test_data->seg_sz, 436 &mem->decomp_buf_infos[dmz]); 437 dmz++; 438 } 439 440 if (!decompress_only) 441 data_sz = RTE_MIN(remaining_data, 442 test_data->seg_sz); 443 else 444 data_sz = test_data->out_seg_sz; 445 446 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 447 data_sz); 448 449 if (data_addr == NULL) { 450 RTE_LOG(ERR, USER1, "Could not append data\n"); 451 return -1; 452 } 453 454 if (!decompress_only) { 455 rte_memcpy(data_addr, input_data_ptr, data_sz); 456 input_data_ptr += data_sz; 457 remaining_data -= data_sz; 458 } 459 460 if (rte_pktmbuf_chain(mem->decomp_bufs[i], 461 next_seg) < 0) { 462 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 463 return -1; 464 } 465 segs_per_mbuf++; 466 } 467 468 /* Allocate data in output mbuf */ 469 mem->comp_bufs[i] = 470 rte_pktmbuf_alloc(mem->comp_buf_pool); 471 if (mem->comp_bufs[i] == NULL) { 472 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 473 return -1; 474 } 475 476 if (test_data->use_external_mbufs != 0) { 477 rte_pktmbuf_attach_extbuf(mem->comp_bufs[i], 478 mem->comp_memzones[cmz]->addr, 479 mem->comp_memzones[cmz]->iova, 480 test_data->out_seg_sz, 481 &mem->comp_buf_infos[cmz]); 482 cmz++; 483 } 484 485 if (decompress_only) 486 data_sz = RTE_MIN(remaining_data_decomp, test_data->seg_sz); 487 else 488 data_sz = test_data->out_seg_sz; 489 490 data_addr = (uint8_t *) rte_pktmbuf_append(mem->comp_bufs[i], 491 data_sz); 492 if (data_addr == NULL) { 493 RTE_LOG(ERR, USER1, "Could not append data\n"); 494 return -1; 495 } 496 497 if (decompress_only) { 498 rte_memcpy(data_addr, input_data_ptr, data_sz); 499 input_data_ptr += data_sz; 500 remaining_data_decomp -= data_sz; 501 } 502 503 /* Chain mbufs if needed for output mbufs */ 504 for (j = 1; j < segs_per_mbuf && remaining_data_decomp > 0; j++) { 505 struct rte_mbuf *next_seg = 506 rte_pktmbuf_alloc(mem->comp_buf_pool); 507 508 if (next_seg == NULL) { 509 RTE_LOG(ERR, USER1, 510 "Could not allocate mbuf\n"); 511 return -1; 512 } 513 514 if (test_data->use_external_mbufs != 0) { 515 rte_pktmbuf_attach_extbuf( 516 next_seg, 517 mem->comp_memzones[cmz]->addr, 518 mem->comp_memzones[cmz]->iova, 519 test_data->out_seg_sz, 520 &mem->comp_buf_infos[cmz]); 521 cmz++; 522 } 523 524 if (decompress_only) 525 data_sz = RTE_MIN(remaining_data_decomp, 526 test_data->seg_sz); 527 else 528 data_sz = test_data->out_seg_sz; 529 530 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 531 data_sz); 532 if (data_addr == NULL) { 533 RTE_LOG(ERR, USER1, "Could not append data\n"); 534 return -1; 535 } 536 537 if (decompress_only) { 538 rte_memcpy(data_addr, input_data_ptr, data_sz); 539 input_data_ptr += data_sz; 540 remaining_data_decomp -= data_sz; 541 } 542 543 if (rte_pktmbuf_chain(mem->comp_bufs[i], 544 next_seg) < 0) { 545 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 546 return -1; 547 } 548 } 549 } 550 551 buffer_info.segments_per_last_buff = segs_per_mbuf; 552 buffer_info.last_segment_sz = data_sz; 553 554 return 0; 555 } 556 557 void 558 print_test_dynamics(const struct comp_test_data *test_data) 559 { 560 uint32_t opt_total_segs = DIV_CEIL(buffer_info.input_data_sz, 561 MAX_SEG_SIZE); 562 563 if (buffer_info.total_buffs > 1) { 564 if (test_data->test == CPERF_TEST_TYPE_THROUGHPUT) { 565 printf("\nWarning: for the current input parameters, number" 566 " of ops is higher than one, which may result" 567 " in sub-optimal performance.\n"); 568 printf("To improve the performance (for the current" 569 " input data) following parameters are" 570 " suggested:\n"); 571 printf(" * Segment size: %d\n", 572 MAX_SEG_SIZE); 573 printf(" * Number of segments: %u\n", 574 opt_total_segs); 575 } 576 } else if (buffer_info.total_buffs == 1) { 577 printf("\nInfo: there is only one op with %u segments -" 578 " the compression ratio is the best.\n", 579 buffer_info.segments_per_last_buff); 580 if (buffer_info.segment_sz < MAX_SEG_SIZE) 581 printf("To reduce compression time, please use" 582 " bigger segment size: %d.\n", 583 MAX_SEG_SIZE); 584 else if (buffer_info.segment_sz == MAX_SEG_SIZE) 585 printf("Segment size is optimal for the best" 586 " performance.\n"); 587 } else 588 printf("Warning: something wrong happened!!\n"); 589 590 printf("\nFor the current input parameters (segment size = %u," 591 " maximum segments per SGL = %u):\n", 592 buffer_info.segment_sz, 593 buffer_info.segments_per_buff); 594 printf(" * Total number of buffers: %d\n", 595 buffer_info.total_segments); 596 printf(" * %u buffer(s) %u bytes long, last buffer %u" 597 " byte(s) long\n", 598 buffer_info.total_segments - 1, 599 buffer_info.segment_sz, 600 buffer_info.last_segment_sz); 601 printf(" * Number of ops: %u\n", buffer_info.total_buffs); 602 printf(" * Total memory allocation: %u\n", 603 (buffer_info.total_segments - 1) * buffer_info.segment_sz 604 + buffer_info.last_segment_sz); 605 if (buffer_info.total_buffs > 1) 606 printf(" * %u ops: %u segment(s) in each," 607 " segment size %u\n", 608 buffer_info.total_buffs - 1, 609 buffer_info.segments_per_buff, 610 buffer_info.segment_sz); 611 if (buffer_info.segments_per_last_buff > 1) { 612 printf(" * 1 op %u segments:\n", 613 buffer_info.segments_per_last_buff); 614 printf(" o %u segment size %u\n", 615 buffer_info.segments_per_last_buff - 1, 616 buffer_info.segment_sz); 617 printf(" o last segment size %u\n", 618 buffer_info.last_segment_sz); 619 } else if (buffer_info.segments_per_last_buff == 1) { 620 printf(" * 1 op (the last one): %u segment %u" 621 " byte(s) long\n\n", 622 buffer_info.segments_per_last_buff, 623 buffer_info.last_segment_sz); 624 } 625 printf("\n"); 626 } 627