1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_eal.h> 7 #include <rte_log.h> 8 #include <rte_compressdev.h> 9 10 #include "comp_perf.h" 11 #include "comp_perf_options.h" 12 #include "comp_perf_test_throughput.h" 13 #include "comp_perf_test_cyclecount.h" 14 #include "comp_perf_test_common.h" 15 #include "comp_perf_test_verify.h" 16 17 18 #define DIV_CEIL(a, b) ((a) / (b) + ((a) % (b) != 0)) 19 20 struct cperf_buffer_info { 21 uint16_t total_segments; 22 uint16_t segment_sz; 23 uint16_t last_segment_sz; 24 uint32_t total_buffs; /*number of buffers = number of ops*/ 25 uint16_t segments_per_buff; 26 uint16_t segments_per_last_buff; 27 size_t input_data_sz; 28 }; 29 30 static struct cperf_buffer_info buffer_info; 31 32 int 33 param_range_check(uint16_t size, const struct rte_param_log2_range *range) 34 { 35 unsigned int next_size; 36 37 /* Check lower/upper bounds */ 38 if (size < range->min) 39 return -1; 40 41 if (size > range->max) 42 return -1; 43 44 /* If range is actually only one value, size is correct */ 45 if (range->increment == 0) 46 return 0; 47 48 /* Check if value is one of the supported sizes */ 49 for (next_size = range->min; next_size <= range->max; 50 next_size += range->increment) 51 if (size == next_size) 52 return 0; 53 54 return -1; 55 } 56 57 static uint32_t 58 find_buf_size(uint32_t input_size) 59 { 60 uint32_t i; 61 62 /* From performance point of view the buffer size should be a 63 * power of 2 but also should be enough to store incompressible data 64 */ 65 66 /* We're looking for nearest power of 2 buffer size, which is greater 67 * than input_size 68 */ 69 uint32_t size = 70 !input_size ? MIN_COMPRESSED_BUF_SIZE : (input_size << 1); 71 72 for (i = UINT16_MAX + 1; !(i & size); i >>= 1) 73 ; 74 75 return i > ((UINT16_MAX + 1) >> 1) 76 ? (uint32_t)((float)input_size * EXPANSE_RATIO) 77 : i; 78 } 79 80 void 81 comp_perf_free_memory(struct comp_test_data *test_data, 82 struct cperf_mem_resources *mem) 83 { 84 uint32_t i; 85 86 if (mem->decomp_bufs != NULL) 87 for (i = 0; i < mem->total_bufs; i++) 88 rte_pktmbuf_free(mem->decomp_bufs[i]); 89 90 if (mem->comp_bufs != NULL) 91 for (i = 0; i < mem->total_bufs; i++) 92 rte_pktmbuf_free(mem->comp_bufs[i]); 93 94 rte_free(mem->decomp_bufs); 95 rte_free(mem->comp_bufs); 96 rte_free(mem->decompressed_data); 97 rte_free(mem->compressed_data); 98 rte_mempool_free(mem->op_pool); 99 rte_mempool_free(mem->decomp_buf_pool); 100 rte_mempool_free(mem->comp_buf_pool); 101 102 /* external mbuf support */ 103 if (mem->decomp_memzones != NULL) { 104 for (i = 0; i < test_data->total_segs; i++) 105 rte_memzone_free(mem->decomp_memzones[i]); 106 rte_free(mem->decomp_memzones); 107 } 108 if (mem->comp_memzones != NULL) { 109 for (i = 0; i < test_data->total_segs; i++) 110 rte_memzone_free(mem->comp_memzones[i]); 111 rte_free(mem->comp_memzones); 112 } 113 rte_free(mem->decomp_buf_infos); 114 rte_free(mem->comp_buf_infos); 115 } 116 117 static void 118 comp_perf_extbuf_free_cb(void *addr __rte_unused, void *opaque __rte_unused) 119 { 120 } 121 122 static const struct rte_memzone * 123 comp_perf_make_memzone(const char *name, struct cperf_mem_resources *mem, 124 unsigned int number, size_t size) 125 { 126 unsigned int socket_id = rte_socket_id(); 127 char mz_name[RTE_MEMZONE_NAMESIZE]; 128 const struct rte_memzone *memzone; 129 130 snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_s%u_d%u_q%u_%d", name, 131 socket_id, mem->dev_id, mem->qp_id, number); 132 memzone = rte_memzone_lookup(mz_name); 133 if (memzone != NULL && memzone->len != size) { 134 rte_memzone_free(memzone); 135 memzone = NULL; 136 } 137 if (memzone == NULL) { 138 memzone = rte_memzone_reserve_aligned(mz_name, size, socket_id, 139 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); 140 if (memzone == NULL) 141 RTE_LOG(ERR, USER1, "Can't allocate memory zone %s\n", 142 mz_name); 143 } 144 return memzone; 145 } 146 147 static int 148 comp_perf_allocate_external_mbufs(struct comp_test_data *test_data, 149 struct cperf_mem_resources *mem) 150 { 151 uint32_t i; 152 153 mem->comp_memzones = rte_zmalloc_socket(NULL, 154 test_data->total_segs * sizeof(struct rte_memzone *), 155 0, rte_socket_id()); 156 157 if (mem->comp_memzones == NULL) { 158 RTE_LOG(ERR, USER1, 159 "Memory to hold the compression memzones could not be allocated\n"); 160 return -1; 161 } 162 163 mem->decomp_memzones = rte_zmalloc_socket(NULL, 164 test_data->total_segs * sizeof(struct rte_memzone *), 165 0, rte_socket_id()); 166 167 if (mem->decomp_memzones == NULL) { 168 RTE_LOG(ERR, USER1, 169 "Memory to hold the decompression memzones could not be allocated\n"); 170 return -1; 171 } 172 173 mem->comp_buf_infos = rte_zmalloc_socket(NULL, 174 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 175 0, rte_socket_id()); 176 177 if (mem->comp_buf_infos == NULL) { 178 RTE_LOG(ERR, USER1, 179 "Memory to hold the compression buf infos could not be allocated\n"); 180 return -1; 181 } 182 183 mem->decomp_buf_infos = rte_zmalloc_socket(NULL, 184 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 185 0, rte_socket_id()); 186 187 if (mem->decomp_buf_infos == NULL) { 188 RTE_LOG(ERR, USER1, 189 "Memory to hold the decompression buf infos could not be allocated\n"); 190 return -1; 191 } 192 193 for (i = 0; i < test_data->total_segs; i++) { 194 mem->comp_memzones[i] = comp_perf_make_memzone("comp", mem, 195 i, test_data->out_seg_sz); 196 if (mem->comp_memzones[i] == NULL) { 197 RTE_LOG(ERR, USER1, 198 "Memory to hold the compression memzone could not be allocated\n"); 199 return -1; 200 } 201 202 mem->decomp_memzones[i] = comp_perf_make_memzone("decomp", mem, 203 i, test_data->seg_sz); 204 if (mem->decomp_memzones[i] == NULL) { 205 RTE_LOG(ERR, USER1, 206 "Memory to hold the decompression memzone could not be allocated\n"); 207 return -1; 208 } 209 210 mem->comp_buf_infos[i].free_cb = 211 comp_perf_extbuf_free_cb; 212 mem->comp_buf_infos[i].fcb_opaque = NULL; 213 rte_mbuf_ext_refcnt_set(&mem->comp_buf_infos[i], 1); 214 215 mem->decomp_buf_infos[i].free_cb = 216 comp_perf_extbuf_free_cb; 217 mem->decomp_buf_infos[i].fcb_opaque = NULL; 218 rte_mbuf_ext_refcnt_set(&mem->decomp_buf_infos[i], 1); 219 } 220 221 return 0; 222 } 223 224 int 225 comp_perf_allocate_memory(struct comp_test_data *test_data, 226 struct cperf_mem_resources *mem) 227 { 228 uint16_t comp_mbuf_size; 229 uint16_t decomp_mbuf_size; 230 231 test_data->out_seg_sz = find_buf_size(test_data->seg_sz); 232 233 /* Number of segments for input and output 234 * (compression and decompression) 235 */ 236 test_data->total_segs = DIV_CEIL(test_data->input_data_sz, 237 test_data->seg_sz); 238 239 if (test_data->use_external_mbufs != 0) { 240 if (comp_perf_allocate_external_mbufs(test_data, mem) < 0) 241 return -1; 242 comp_mbuf_size = 0; 243 decomp_mbuf_size = 0; 244 } else { 245 comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 246 decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 247 } 248 249 char pool_name[32] = ""; 250 251 snprintf(pool_name, sizeof(pool_name), "comp_buf_pool_%u_qp_%u", 252 mem->dev_id, mem->qp_id); 253 mem->comp_buf_pool = rte_pktmbuf_pool_create(pool_name, 254 test_data->total_segs, 255 0, 0, 256 comp_mbuf_size, 257 rte_socket_id()); 258 if (mem->comp_buf_pool == NULL) { 259 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 260 return -1; 261 } 262 263 snprintf(pool_name, sizeof(pool_name), "decomp_buf_pool_%u_qp_%u", 264 mem->dev_id, mem->qp_id); 265 mem->decomp_buf_pool = rte_pktmbuf_pool_create(pool_name, 266 test_data->total_segs, 267 0, 0, 268 decomp_mbuf_size, 269 rte_socket_id()); 270 if (mem->decomp_buf_pool == NULL) { 271 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 272 return -1; 273 } 274 275 mem->total_bufs = DIV_CEIL(test_data->total_segs, 276 test_data->max_sgl_segs); 277 278 snprintf(pool_name, sizeof(pool_name), "op_pool_%u_qp_%u", 279 mem->dev_id, mem->qp_id); 280 281 /* one mempool for both src and dst mbufs */ 282 mem->op_pool = rte_comp_op_pool_create(pool_name, 283 mem->total_bufs * 2, 284 0, 0, rte_socket_id()); 285 if (mem->op_pool == NULL) { 286 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n"); 287 return -1; 288 } 289 290 /* 291 * Compressed data might be a bit larger than input data, 292 * if data cannot be compressed 293 */ 294 mem->compressed_data = rte_zmalloc_socket(NULL, 295 RTE_MAX( 296 (size_t) test_data->out_seg_sz * 297 test_data->total_segs, 298 (size_t) MIN_COMPRESSED_BUF_SIZE), 299 0, 300 rte_socket_id()); 301 if (mem->compressed_data == NULL) { 302 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 303 "file could not be allocated\n"); 304 return -1; 305 } 306 307 mem->decompressed_data = rte_zmalloc_socket(NULL, 308 test_data->input_data_sz, 0, 309 rte_socket_id()); 310 if (mem->decompressed_data == NULL) { 311 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 312 "file could not be allocated\n"); 313 return -1; 314 } 315 316 mem->comp_bufs = rte_zmalloc_socket(NULL, 317 mem->total_bufs * sizeof(struct rte_mbuf *), 318 0, rte_socket_id()); 319 if (mem->comp_bufs == NULL) { 320 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs" 321 " could not be allocated\n"); 322 return -1; 323 } 324 325 mem->decomp_bufs = rte_zmalloc_socket(NULL, 326 mem->total_bufs * sizeof(struct rte_mbuf *), 327 0, rte_socket_id()); 328 if (mem->decomp_bufs == NULL) { 329 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs" 330 " could not be allocated\n"); 331 return -1; 332 } 333 334 buffer_info.total_segments = test_data->total_segs; 335 buffer_info.segment_sz = test_data->seg_sz; 336 buffer_info.total_buffs = mem->total_bufs; 337 buffer_info.segments_per_buff = test_data->max_sgl_segs; 338 buffer_info.input_data_sz = test_data->input_data_sz; 339 340 return 0; 341 } 342 343 int 344 prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) 345 { 346 uint32_t remaining_data = test_data->input_data_sz; 347 uint8_t *input_data_ptr = test_data->input_data; 348 size_t data_sz = 0; 349 uint8_t *data_addr; 350 uint32_t i, j; 351 uint16_t segs_per_mbuf = 0; 352 uint32_t cmz = 0; 353 uint32_t dmz = 0; 354 355 for (i = 0; i < mem->total_bufs; i++) { 356 /* Allocate data in input mbuf and copy data from input file */ 357 mem->decomp_bufs[i] = 358 rte_pktmbuf_alloc(mem->decomp_buf_pool); 359 if (mem->decomp_bufs[i] == NULL) { 360 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 361 return -1; 362 } 363 364 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 365 366 if (test_data->use_external_mbufs != 0) { 367 rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i], 368 mem->decomp_memzones[dmz]->addr, 369 mem->decomp_memzones[dmz]->iova, 370 test_data->seg_sz, 371 &mem->decomp_buf_infos[dmz]); 372 dmz++; 373 } 374 375 data_addr = (uint8_t *) rte_pktmbuf_append( 376 mem->decomp_bufs[i], data_sz); 377 if (data_addr == NULL) { 378 RTE_LOG(ERR, USER1, "Could not append data\n"); 379 return -1; 380 } 381 rte_memcpy(data_addr, input_data_ptr, data_sz); 382 383 input_data_ptr += data_sz; 384 remaining_data -= data_sz; 385 386 /* Already one segment in the mbuf */ 387 segs_per_mbuf = 1; 388 389 /* Chain mbufs if needed for input mbufs */ 390 while (segs_per_mbuf < test_data->max_sgl_segs 391 && remaining_data > 0) { 392 struct rte_mbuf *next_seg = 393 rte_pktmbuf_alloc(mem->decomp_buf_pool); 394 395 if (next_seg == NULL) { 396 RTE_LOG(ERR, USER1, 397 "Could not allocate mbuf\n"); 398 return -1; 399 } 400 401 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 402 403 if (test_data->use_external_mbufs != 0) { 404 rte_pktmbuf_attach_extbuf( 405 next_seg, 406 mem->decomp_memzones[dmz]->addr, 407 mem->decomp_memzones[dmz]->iova, 408 test_data->seg_sz, 409 &mem->decomp_buf_infos[dmz]); 410 dmz++; 411 } 412 413 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 414 data_sz); 415 416 if (data_addr == NULL) { 417 RTE_LOG(ERR, USER1, "Could not append data\n"); 418 return -1; 419 } 420 421 rte_memcpy(data_addr, input_data_ptr, data_sz); 422 input_data_ptr += data_sz; 423 remaining_data -= data_sz; 424 425 if (rte_pktmbuf_chain(mem->decomp_bufs[i], 426 next_seg) < 0) { 427 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 428 return -1; 429 } 430 segs_per_mbuf++; 431 } 432 433 /* Allocate data in output mbuf */ 434 mem->comp_bufs[i] = 435 rte_pktmbuf_alloc(mem->comp_buf_pool); 436 if (mem->comp_bufs[i] == NULL) { 437 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 438 return -1; 439 } 440 441 if (test_data->use_external_mbufs != 0) { 442 rte_pktmbuf_attach_extbuf(mem->comp_bufs[i], 443 mem->comp_memzones[cmz]->addr, 444 mem->comp_memzones[cmz]->iova, 445 test_data->out_seg_sz, 446 &mem->comp_buf_infos[cmz]); 447 cmz++; 448 } 449 450 data_addr = (uint8_t *) rte_pktmbuf_append( 451 mem->comp_bufs[i], 452 test_data->out_seg_sz); 453 if (data_addr == NULL) { 454 RTE_LOG(ERR, USER1, "Could not append data\n"); 455 return -1; 456 } 457 458 /* Chain mbufs if needed for output mbufs */ 459 for (j = 1; j < segs_per_mbuf; j++) { 460 struct rte_mbuf *next_seg = 461 rte_pktmbuf_alloc(mem->comp_buf_pool); 462 463 if (next_seg == NULL) { 464 RTE_LOG(ERR, USER1, 465 "Could not allocate mbuf\n"); 466 return -1; 467 } 468 469 if (test_data->use_external_mbufs != 0) { 470 rte_pktmbuf_attach_extbuf( 471 next_seg, 472 mem->comp_memzones[cmz]->addr, 473 mem->comp_memzones[cmz]->iova, 474 test_data->out_seg_sz, 475 &mem->comp_buf_infos[cmz]); 476 cmz++; 477 } 478 479 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 480 test_data->out_seg_sz); 481 if (data_addr == NULL) { 482 RTE_LOG(ERR, USER1, "Could not append data\n"); 483 return -1; 484 } 485 486 if (rte_pktmbuf_chain(mem->comp_bufs[i], 487 next_seg) < 0) { 488 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 489 return -1; 490 } 491 } 492 } 493 494 buffer_info.segments_per_last_buff = segs_per_mbuf; 495 buffer_info.last_segment_sz = data_sz; 496 497 return 0; 498 } 499 500 void 501 print_test_dynamics(const struct comp_test_data *test_data) 502 { 503 uint32_t opt_total_segs = DIV_CEIL(buffer_info.input_data_sz, 504 MAX_SEG_SIZE); 505 506 if (buffer_info.total_buffs > 1) { 507 if (test_data->test == CPERF_TEST_TYPE_THROUGHPUT) { 508 printf("\nWarning: for the current input parameters, number" 509 " of ops is higher than one, which may result" 510 " in sub-optimal performance.\n"); 511 printf("To improve the performance (for the current" 512 " input data) following parameters are" 513 " suggested:\n"); 514 printf(" * Segment size: %d\n", 515 MAX_SEG_SIZE); 516 printf(" * Number of segments: %u\n", 517 opt_total_segs); 518 } 519 } else if (buffer_info.total_buffs == 1) { 520 printf("\nInfo: there is only one op with %u segments -" 521 " the compression ratio is the best.\n", 522 buffer_info.segments_per_last_buff); 523 if (buffer_info.segment_sz < MAX_SEG_SIZE) 524 printf("To reduce compression time, please use" 525 " bigger segment size: %d.\n", 526 MAX_SEG_SIZE); 527 else if (buffer_info.segment_sz == MAX_SEG_SIZE) 528 printf("Segment size is optimal for the best" 529 " performance.\n"); 530 } else 531 printf("Warning: something wrong happened!!\n"); 532 533 printf("\nFor the current input parameters (segment size = %u," 534 " maximum segments per SGL = %u):\n", 535 buffer_info.segment_sz, 536 buffer_info.segments_per_buff); 537 printf(" * Total number of buffers: %d\n", 538 buffer_info.total_segments); 539 printf(" * %u buffer(s) %u bytes long, last buffer %u" 540 " byte(s) long\n", 541 buffer_info.total_segments - 1, 542 buffer_info.segment_sz, 543 buffer_info.last_segment_sz); 544 printf(" * Number of ops: %u\n", buffer_info.total_buffs); 545 printf(" * Total memory allocation: %u\n", 546 (buffer_info.total_segments - 1) * buffer_info.segment_sz 547 + buffer_info.last_segment_sz); 548 if (buffer_info.total_buffs > 1) 549 printf(" * %u ops: %u segment(s) in each," 550 " segment size %u\n", 551 buffer_info.total_buffs - 1, 552 buffer_info.segments_per_buff, 553 buffer_info.segment_sz); 554 if (buffer_info.segments_per_last_buff > 1) { 555 printf(" * 1 op %u segments:\n", 556 buffer_info.segments_per_last_buff); 557 printf(" o %u segment size %u\n", 558 buffer_info.segments_per_last_buff - 1, 559 buffer_info.segment_sz); 560 printf(" o last segment size %u\n", 561 buffer_info.last_segment_sz); 562 } else if (buffer_info.segments_per_last_buff == 1) { 563 printf(" * 1 op (the last one): %u segment %u" 564 " byte(s) long\n\n", 565 buffer_info.segments_per_last_buff, 566 buffer_info.last_segment_sz); 567 } 568 printf("\n"); 569 } 570