1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_eal.h> 7 #include <rte_log.h> 8 #include <rte_compressdev.h> 9 10 #include "comp_perf.h" 11 #include "comp_perf_options.h" 12 #include "comp_perf_test_throughput.h" 13 #include "comp_perf_test_cyclecount.h" 14 #include "comp_perf_test_common.h" 15 #include "comp_perf_test_verify.h" 16 17 18 #define DIV_CEIL(a, b) ((a) / (b) + ((a) % (b) != 0)) 19 20 struct cperf_buffer_info { 21 uint16_t total_segments; 22 uint16_t segment_sz; 23 uint16_t last_segment_sz; 24 uint32_t total_buffs; /*number of buffers = number of ops*/ 25 uint16_t segments_per_buff; 26 uint16_t segments_per_last_buff; 27 size_t input_data_sz; 28 }; 29 30 static struct cperf_buffer_info buffer_info; 31 32 int 33 param_range_check(uint16_t size, const struct rte_param_log2_range *range) 34 { 35 unsigned int next_size; 36 37 /* Check lower/upper bounds */ 38 if (size < range->min) 39 return -1; 40 41 if (size > range->max) 42 return -1; 43 44 /* If range is actually only one value, size is correct */ 45 if (range->increment == 0) 46 return 0; 47 48 /* Check if value is one of the supported sizes */ 49 for (next_size = range->min; next_size <= range->max; 50 next_size += range->increment) 51 if (size == next_size) 52 return 0; 53 54 return -1; 55 } 56 57 static uint32_t 58 find_buf_size(uint32_t input_size) 59 { 60 uint32_t i; 61 62 /* From performance point of view the buffer size should be a 63 * power of 2 but also should be enough to store incompressible data 64 */ 65 66 /* We're looking for nearest power of 2 buffer size, which is greater 67 * than input_size 68 */ 69 uint32_t size = 70 !input_size ? MIN_COMPRESSED_BUF_SIZE : (input_size << 1); 71 72 for (i = UINT16_MAX + 1; !(i & size); i >>= 1) 73 ; 74 75 return i > ((UINT16_MAX + 1) >> 1) 76 ? (uint32_t)((float)input_size * EXPANSE_RATIO) 77 : i; 78 } 79 80 void 81 comp_perf_free_memory(struct comp_test_data *test_data, 82 struct cperf_mem_resources *mem) 83 { 84 uint32_t i; 85 86 if (mem->decomp_bufs != NULL) 87 for (i = 0; i < mem->total_bufs; i++) 88 rte_pktmbuf_free(mem->decomp_bufs[i]); 89 90 if (mem->comp_bufs != NULL) 91 for (i = 0; i < mem->total_bufs; i++) 92 rte_pktmbuf_free(mem->comp_bufs[i]); 93 94 rte_free(mem->decomp_bufs); 95 rte_free(mem->comp_bufs); 96 rte_free(mem->decompressed_data); 97 rte_free(mem->compressed_data); 98 rte_mempool_free(mem->op_pool); 99 rte_mempool_free(mem->decomp_buf_pool); 100 rte_mempool_free(mem->comp_buf_pool); 101 102 /* external mbuf support */ 103 if (mem->decomp_memzones != NULL) { 104 for (i = 0; i < test_data->total_segs; i++) 105 rte_memzone_free(mem->decomp_memzones[i]); 106 rte_free(mem->decomp_memzones); 107 } 108 if (mem->comp_memzones != NULL) { 109 for (i = 0; i < test_data->total_segs; i++) 110 rte_memzone_free(mem->comp_memzones[i]); 111 rte_free(mem->comp_memzones); 112 } 113 rte_free(mem->decomp_buf_infos); 114 rte_free(mem->comp_buf_infos); 115 } 116 117 static void 118 comp_perf_extbuf_free_cb(void *addr __rte_unused, void *opaque __rte_unused) 119 { 120 } 121 122 static const struct rte_memzone * 123 comp_perf_make_memzone(const char *name, struct cperf_mem_resources *mem, 124 unsigned int number, size_t size) 125 { 126 unsigned int socket_id = rte_socket_id(); 127 char mz_name[RTE_MEMZONE_NAMESIZE]; 128 const struct rte_memzone *memzone; 129 130 snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_s%u_d%u_q%u_%d", name, 131 socket_id, mem->dev_id, mem->qp_id, number); 132 memzone = rte_memzone_lookup(mz_name); 133 if (memzone != NULL && memzone->len != size) { 134 rte_memzone_free(memzone); 135 memzone = NULL; 136 } 137 if (memzone == NULL) { 138 memzone = rte_memzone_reserve_aligned(mz_name, size, socket_id, 139 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); 140 if (memzone == NULL) 141 RTE_LOG(ERR, USER1, "Can't allocate memory zone %s\n", 142 mz_name); 143 } 144 return memzone; 145 } 146 147 static int 148 comp_perf_allocate_external_mbufs(struct comp_test_data *test_data, 149 struct cperf_mem_resources *mem) 150 { 151 uint32_t i; 152 153 mem->comp_memzones = rte_zmalloc_socket(NULL, 154 test_data->total_segs * sizeof(struct rte_memzone *), 155 0, rte_socket_id()); 156 157 if (mem->comp_memzones == NULL) { 158 RTE_LOG(ERR, USER1, 159 "Memory to hold the compression memzones could not be allocated\n"); 160 return -1; 161 } 162 163 mem->decomp_memzones = rte_zmalloc_socket(NULL, 164 test_data->total_segs * sizeof(struct rte_memzone *), 165 0, rte_socket_id()); 166 167 if (mem->decomp_memzones == NULL) { 168 RTE_LOG(ERR, USER1, 169 "Memory to hold the decompression memzones could not be allocated\n"); 170 return -1; 171 } 172 173 mem->comp_buf_infos = rte_zmalloc_socket(NULL, 174 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 175 0, rte_socket_id()); 176 177 if (mem->comp_buf_infos == NULL) { 178 RTE_LOG(ERR, USER1, 179 "Memory to hold the compression buf infos could not be allocated\n"); 180 return -1; 181 } 182 183 mem->decomp_buf_infos = rte_zmalloc_socket(NULL, 184 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 185 0, rte_socket_id()); 186 187 if (mem->decomp_buf_infos == NULL) { 188 RTE_LOG(ERR, USER1, 189 "Memory to hold the decompression buf infos could not be allocated\n"); 190 return -1; 191 } 192 193 for (i = 0; i < test_data->total_segs; i++) { 194 mem->comp_memzones[i] = comp_perf_make_memzone("comp", mem, 195 i, test_data->out_seg_sz); 196 if (mem->comp_memzones[i] == NULL) { 197 RTE_LOG(ERR, USER1, 198 "Memory to hold the compression memzone could not be allocated\n"); 199 return -1; 200 } 201 202 mem->decomp_memzones[i] = comp_perf_make_memzone("decomp", mem, 203 i, test_data->seg_sz); 204 if (mem->decomp_memzones[i] == NULL) { 205 RTE_LOG(ERR, USER1, 206 "Memory to hold the decompression memzone could not be allocated\n"); 207 return -1; 208 } 209 210 mem->comp_buf_infos[i].free_cb = 211 comp_perf_extbuf_free_cb; 212 mem->comp_buf_infos[i].fcb_opaque = NULL; 213 rte_mbuf_ext_refcnt_set(&mem->comp_buf_infos[i], 1); 214 215 mem->decomp_buf_infos[i].free_cb = 216 comp_perf_extbuf_free_cb; 217 mem->decomp_buf_infos[i].fcb_opaque = NULL; 218 rte_mbuf_ext_refcnt_set(&mem->decomp_buf_infos[i], 1); 219 } 220 221 return 0; 222 } 223 224 int 225 comp_perf_allocate_memory(struct comp_test_data *test_data, 226 struct cperf_mem_resources *mem) 227 { 228 uint16_t comp_mbuf_size; 229 uint16_t decomp_mbuf_size; 230 size_t comp_data_size; 231 size_t decomp_data_size; 232 size_t output_data_sz; 233 234 test_data->out_seg_sz = find_buf_size(test_data->seg_sz); 235 236 if (test_data->test_op & COMPRESS) { 237 /* 238 * Number of segments for input and output 239 * (compression and decompression) 240 */ 241 test_data->total_segs = DIV_CEIL(test_data->input_data_sz, 242 test_data->seg_sz); 243 } else { 244 /* 245 * When application does decompression only, input data is 246 * compressed and smaller than the output. The expected size of 247 * uncompressed data given by the user in segment size argument. 248 */ 249 test_data->total_segs = test_data->max_sgl_segs; 250 } 251 252 output_data_sz = (size_t) test_data->out_seg_sz * test_data->total_segs; 253 output_data_sz = 254 RTE_MAX(output_data_sz, (size_t) MIN_COMPRESSED_BUF_SIZE); 255 256 if (test_data->use_external_mbufs != 0) { 257 if (comp_perf_allocate_external_mbufs(test_data, mem) < 0) 258 return -1; 259 comp_mbuf_size = 0; 260 decomp_mbuf_size = 0; 261 } else if (test_data->test_op & COMPRESS) { 262 comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 263 decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 264 } else { 265 comp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 266 decomp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 267 } 268 269 char pool_name[32] = ""; 270 271 snprintf(pool_name, sizeof(pool_name), "comp_buf_pool_%u_qp_%u", 272 mem->dev_id, mem->qp_id); 273 mem->comp_buf_pool = rte_pktmbuf_pool_create(pool_name, 274 test_data->total_segs, 275 0, 0, 276 comp_mbuf_size, 277 rte_socket_id()); 278 if (mem->comp_buf_pool == NULL) { 279 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 280 return -1; 281 } 282 283 snprintf(pool_name, sizeof(pool_name), "decomp_buf_pool_%u_qp_%u", 284 mem->dev_id, mem->qp_id); 285 mem->decomp_buf_pool = rte_pktmbuf_pool_create(pool_name, 286 test_data->total_segs, 287 0, 0, 288 decomp_mbuf_size, 289 rte_socket_id()); 290 if (mem->decomp_buf_pool == NULL) { 291 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 292 return -1; 293 } 294 295 mem->total_bufs = DIV_CEIL(test_data->total_segs, 296 test_data->max_sgl_segs); 297 298 snprintf(pool_name, sizeof(pool_name), "op_pool_%u_qp_%u", 299 mem->dev_id, mem->qp_id); 300 301 /* one mempool for both src and dst mbufs */ 302 mem->op_pool = rte_comp_op_pool_create(pool_name, 303 mem->total_bufs * 2, 304 0, 0, rte_socket_id()); 305 if (mem->op_pool == NULL) { 306 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n"); 307 return -1; 308 } 309 310 if (test_data->test_op & COMPRESS) { 311 /* 312 * Compressed data might be a bit larger than input data, 313 * if data cannot be compressed 314 */ 315 comp_data_size = output_data_sz; 316 decomp_data_size = test_data->input_data_sz; 317 } else { 318 comp_data_size = test_data->input_data_sz; 319 decomp_data_size = output_data_sz; 320 } 321 322 mem->compressed_data = rte_zmalloc_socket(NULL, comp_data_size, 0, 323 rte_socket_id()); 324 if (mem->compressed_data == NULL) { 325 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 326 "file could not be allocated\n"); 327 return -1; 328 } 329 330 mem->decompressed_data = rte_zmalloc_socket(NULL, decomp_data_size, 0, 331 rte_socket_id()); 332 if (mem->decompressed_data == NULL) { 333 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 334 "file could not be allocated\n"); 335 return -1; 336 } 337 338 mem->comp_bufs = rte_zmalloc_socket(NULL, 339 mem->total_bufs * sizeof(struct rte_mbuf *), 340 0, rte_socket_id()); 341 if (mem->comp_bufs == NULL) { 342 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs" 343 " could not be allocated\n"); 344 return -1; 345 } 346 347 mem->decomp_bufs = rte_zmalloc_socket(NULL, 348 mem->total_bufs * sizeof(struct rte_mbuf *), 349 0, rte_socket_id()); 350 if (mem->decomp_bufs == NULL) { 351 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs" 352 " could not be allocated\n"); 353 return -1; 354 } 355 356 buffer_info.total_segments = test_data->total_segs; 357 buffer_info.segment_sz = test_data->seg_sz; 358 buffer_info.total_buffs = mem->total_bufs; 359 buffer_info.segments_per_buff = test_data->max_sgl_segs; 360 buffer_info.input_data_sz = test_data->input_data_sz; 361 362 return 0; 363 } 364 365 int 366 prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) 367 { 368 uint32_t remaining_data = test_data->input_data_sz; 369 uint8_t *input_data_ptr = test_data->input_data; 370 size_t data_sz = 0; 371 uint8_t *data_addr; 372 uint32_t i, j; 373 uint16_t segs_per_mbuf = 0; 374 uint32_t cmz = 0; 375 uint32_t dmz = 0; 376 bool decompress_only = !!(test_data->test_op == DECOMPRESS); 377 378 for (i = 0; i < mem->total_bufs; i++) { 379 /* Allocate data in input mbuf and copy data from input file */ 380 mem->decomp_bufs[i] = 381 rte_pktmbuf_alloc(mem->decomp_buf_pool); 382 if (mem->decomp_bufs[i] == NULL) { 383 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 384 return -1; 385 } 386 387 if (test_data->use_external_mbufs != 0) { 388 rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i], 389 mem->decomp_memzones[dmz]->addr, 390 mem->decomp_memzones[dmz]->iova, 391 test_data->seg_sz, 392 &mem->decomp_buf_infos[dmz]); 393 dmz++; 394 } 395 396 if (!decompress_only) 397 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 398 else 399 data_sz = test_data->out_seg_sz; 400 401 data_addr = (uint8_t *) rte_pktmbuf_append( 402 mem->decomp_bufs[i], data_sz); 403 if (data_addr == NULL) { 404 RTE_LOG(ERR, USER1, "Could not append data\n"); 405 return -1; 406 } 407 408 if (!decompress_only) { 409 rte_memcpy(data_addr, input_data_ptr, data_sz); 410 input_data_ptr += data_sz; 411 remaining_data -= data_sz; 412 } 413 414 /* Already one segment in the mbuf */ 415 segs_per_mbuf = 1; 416 417 /* Chain mbufs if needed for input mbufs */ 418 while (segs_per_mbuf < test_data->max_sgl_segs 419 && remaining_data > 0) { 420 struct rte_mbuf *next_seg = 421 rte_pktmbuf_alloc(mem->decomp_buf_pool); 422 423 if (next_seg == NULL) { 424 RTE_LOG(ERR, USER1, 425 "Could not allocate mbuf\n"); 426 return -1; 427 } 428 429 if (test_data->use_external_mbufs != 0) { 430 rte_pktmbuf_attach_extbuf( 431 next_seg, 432 mem->decomp_memzones[dmz]->addr, 433 mem->decomp_memzones[dmz]->iova, 434 test_data->seg_sz, 435 &mem->decomp_buf_infos[dmz]); 436 dmz++; 437 } 438 439 if (!decompress_only) 440 data_sz = RTE_MIN(remaining_data, 441 test_data->seg_sz); 442 else 443 data_sz = test_data->out_seg_sz; 444 445 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 446 data_sz); 447 448 if (data_addr == NULL) { 449 RTE_LOG(ERR, USER1, "Could not append data\n"); 450 return -1; 451 } 452 453 if (!decompress_only) { 454 rte_memcpy(data_addr, input_data_ptr, data_sz); 455 input_data_ptr += data_sz; 456 remaining_data -= data_sz; 457 } 458 459 if (rte_pktmbuf_chain(mem->decomp_bufs[i], 460 next_seg) < 0) { 461 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 462 return -1; 463 } 464 segs_per_mbuf++; 465 } 466 467 /* Allocate data in output mbuf */ 468 mem->comp_bufs[i] = 469 rte_pktmbuf_alloc(mem->comp_buf_pool); 470 if (mem->comp_bufs[i] == NULL) { 471 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 472 return -1; 473 } 474 475 if (test_data->use_external_mbufs != 0) { 476 rte_pktmbuf_attach_extbuf(mem->comp_bufs[i], 477 mem->comp_memzones[cmz]->addr, 478 mem->comp_memzones[cmz]->iova, 479 test_data->out_seg_sz, 480 &mem->comp_buf_infos[cmz]); 481 cmz++; 482 } 483 484 if (decompress_only) 485 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 486 else 487 data_sz = test_data->out_seg_sz; 488 489 data_addr = (uint8_t *) rte_pktmbuf_append(mem->comp_bufs[i], 490 data_sz); 491 if (data_addr == NULL) { 492 RTE_LOG(ERR, USER1, "Could not append data\n"); 493 return -1; 494 } 495 496 if (decompress_only) { 497 rte_memcpy(data_addr, input_data_ptr, data_sz); 498 input_data_ptr += data_sz; 499 remaining_data -= data_sz; 500 } 501 502 /* Chain mbufs if needed for output mbufs */ 503 for (j = 1; j < segs_per_mbuf && remaining_data > 0; j++) { 504 struct rte_mbuf *next_seg = 505 rte_pktmbuf_alloc(mem->comp_buf_pool); 506 507 if (next_seg == NULL) { 508 RTE_LOG(ERR, USER1, 509 "Could not allocate mbuf\n"); 510 return -1; 511 } 512 513 if (test_data->use_external_mbufs != 0) { 514 rte_pktmbuf_attach_extbuf( 515 next_seg, 516 mem->comp_memzones[cmz]->addr, 517 mem->comp_memzones[cmz]->iova, 518 test_data->out_seg_sz, 519 &mem->comp_buf_infos[cmz]); 520 cmz++; 521 } 522 523 if (decompress_only) 524 data_sz = RTE_MIN(remaining_data, 525 test_data->seg_sz); 526 else 527 data_sz = test_data->out_seg_sz; 528 529 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 530 data_sz); 531 if (data_addr == NULL) { 532 RTE_LOG(ERR, USER1, "Could not append data\n"); 533 return -1; 534 } 535 536 if (decompress_only) { 537 rte_memcpy(data_addr, input_data_ptr, data_sz); 538 input_data_ptr += data_sz; 539 remaining_data -= data_sz; 540 } 541 542 if (rte_pktmbuf_chain(mem->comp_bufs[i], 543 next_seg) < 0) { 544 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 545 return -1; 546 } 547 } 548 } 549 550 buffer_info.segments_per_last_buff = segs_per_mbuf; 551 buffer_info.last_segment_sz = data_sz; 552 553 return 0; 554 } 555 556 void 557 print_test_dynamics(const struct comp_test_data *test_data) 558 { 559 uint32_t opt_total_segs = DIV_CEIL(buffer_info.input_data_sz, 560 MAX_SEG_SIZE); 561 562 if (buffer_info.total_buffs > 1) { 563 if (test_data->test == CPERF_TEST_TYPE_THROUGHPUT) { 564 printf("\nWarning: for the current input parameters, number" 565 " of ops is higher than one, which may result" 566 " in sub-optimal performance.\n"); 567 printf("To improve the performance (for the current" 568 " input data) following parameters are" 569 " suggested:\n"); 570 printf(" * Segment size: %d\n", 571 MAX_SEG_SIZE); 572 printf(" * Number of segments: %u\n", 573 opt_total_segs); 574 } 575 } else if (buffer_info.total_buffs == 1) { 576 printf("\nInfo: there is only one op with %u segments -" 577 " the compression ratio is the best.\n", 578 buffer_info.segments_per_last_buff); 579 if (buffer_info.segment_sz < MAX_SEG_SIZE) 580 printf("To reduce compression time, please use" 581 " bigger segment size: %d.\n", 582 MAX_SEG_SIZE); 583 else if (buffer_info.segment_sz == MAX_SEG_SIZE) 584 printf("Segment size is optimal for the best" 585 " performance.\n"); 586 } else 587 printf("Warning: something wrong happened!!\n"); 588 589 printf("\nFor the current input parameters (segment size = %u," 590 " maximum segments per SGL = %u):\n", 591 buffer_info.segment_sz, 592 buffer_info.segments_per_buff); 593 printf(" * Total number of buffers: %d\n", 594 buffer_info.total_segments); 595 printf(" * %u buffer(s) %u bytes long, last buffer %u" 596 " byte(s) long\n", 597 buffer_info.total_segments - 1, 598 buffer_info.segment_sz, 599 buffer_info.last_segment_sz); 600 printf(" * Number of ops: %u\n", buffer_info.total_buffs); 601 printf(" * Total memory allocation: %u\n", 602 (buffer_info.total_segments - 1) * buffer_info.segment_sz 603 + buffer_info.last_segment_sz); 604 if (buffer_info.total_buffs > 1) 605 printf(" * %u ops: %u segment(s) in each," 606 " segment size %u\n", 607 buffer_info.total_buffs - 1, 608 buffer_info.segments_per_buff, 609 buffer_info.segment_sz); 610 if (buffer_info.segments_per_last_buff > 1) { 611 printf(" * 1 op %u segments:\n", 612 buffer_info.segments_per_last_buff); 613 printf(" o %u segment size %u\n", 614 buffer_info.segments_per_last_buff - 1, 615 buffer_info.segment_sz); 616 printf(" o last segment size %u\n", 617 buffer_info.last_segment_sz); 618 } else if (buffer_info.segments_per_last_buff == 1) { 619 printf(" * 1 op (the last one): %u segment %u" 620 " byte(s) long\n\n", 621 buffer_info.segments_per_last_buff, 622 buffer_info.last_segment_sz); 623 } 624 printf("\n"); 625 } 626