1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_eal.h> 7 #include <rte_log.h> 8 #include <rte_compressdev.h> 9 10 #include "comp_perf.h" 11 #include "comp_perf_options.h" 12 #include "comp_perf_test_benchmark.h" 13 #include "comp_perf_test_common.h" 14 #include "comp_perf_test_verify.h" 15 16 17 #define DIV_CEIL(a, b) ((a) / (b) + ((a) % (b) != 0)) 18 19 struct cperf_buffer_info { 20 uint16_t total_segments; 21 uint16_t segment_sz; 22 uint16_t last_segment_sz; 23 uint32_t total_buffs; /*number of buffers = number of ops*/ 24 uint16_t segments_per_buff; 25 uint16_t segments_per_last_buff; 26 size_t input_data_sz; 27 }; 28 29 static struct cperf_buffer_info buffer_info; 30 31 int 32 param_range_check(uint16_t size, const struct rte_param_log2_range *range) 33 { 34 unsigned int next_size; 35 36 /* Check lower/upper bounds */ 37 if (size < range->min) 38 return -1; 39 40 if (size > range->max) 41 return -1; 42 43 /* If range is actually only one value, size is correct */ 44 if (range->increment == 0) 45 return 0; 46 47 /* Check if value is one of the supported sizes */ 48 for (next_size = range->min; next_size <= range->max; 49 next_size += range->increment) 50 if (size == next_size) 51 return 0; 52 53 return -1; 54 } 55 56 static uint32_t 57 find_buf_size(uint32_t input_size) 58 { 59 uint32_t i; 60 61 /* From performance point of view the buffer size should be a 62 * power of 2 but also should be enough to store incompressible data 63 */ 64 65 /* We're looking for nearest power of 2 buffer size, which is greater 66 * than input_size 67 */ 68 uint32_t size = 69 !input_size ? MIN_COMPRESSED_BUF_SIZE : (input_size << 1); 70 71 for (i = UINT16_MAX + 1; !(i & size); i >>= 1) 72 ; 73 74 return i > ((UINT16_MAX + 1) >> 1) 75 ? (uint32_t)((float)input_size * EXPANSE_RATIO) 76 : i; 77 } 78 79 void 80 comp_perf_free_memory(struct comp_test_data *test_data, 81 struct cperf_mem_resources *mem) 82 { 83 uint32_t i; 84 85 if (mem->decomp_bufs != NULL) 86 for (i = 0; i < mem->total_bufs; i++) 87 rte_pktmbuf_free(mem->decomp_bufs[i]); 88 89 if (mem->comp_bufs != NULL) 90 for (i = 0; i < mem->total_bufs; i++) 91 rte_pktmbuf_free(mem->comp_bufs[i]); 92 93 rte_free(mem->decomp_bufs); 94 rte_free(mem->comp_bufs); 95 rte_free(mem->decompressed_data); 96 rte_free(mem->compressed_data); 97 rte_mempool_free(mem->op_pool); 98 rte_mempool_free(mem->decomp_buf_pool); 99 rte_mempool_free(mem->comp_buf_pool); 100 101 /* external mbuf support */ 102 if (mem->decomp_memzones != NULL) { 103 for (i = 0; i < test_data->total_segs; i++) 104 rte_memzone_free(mem->decomp_memzones[i]); 105 rte_free(mem->decomp_memzones); 106 } 107 if (mem->comp_memzones != NULL) { 108 for (i = 0; i < test_data->total_segs; i++) 109 rte_memzone_free(mem->comp_memzones[i]); 110 rte_free(mem->comp_memzones); 111 } 112 rte_free(mem->decomp_buf_infos); 113 rte_free(mem->comp_buf_infos); 114 } 115 116 static void 117 comp_perf_extbuf_free_cb(void *addr __rte_unused, void *opaque __rte_unused) 118 { 119 } 120 121 static const struct rte_memzone * 122 comp_perf_make_memzone(const char *name, struct cperf_mem_resources *mem, 123 unsigned int number, size_t size) 124 { 125 unsigned int socket_id = rte_socket_id(); 126 char mz_name[RTE_MEMZONE_NAMESIZE]; 127 const struct rte_memzone *memzone; 128 129 snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_s%u_d%u_q%u_%d", name, 130 socket_id, mem->dev_id, mem->qp_id, number); 131 memzone = rte_memzone_lookup(mz_name); 132 if (memzone != NULL && memzone->len != size) { 133 rte_memzone_free(memzone); 134 memzone = NULL; 135 } 136 if (memzone == NULL) { 137 memzone = rte_memzone_reserve_aligned(mz_name, size, socket_id, 138 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); 139 if (memzone == NULL) 140 RTE_LOG(ERR, USER1, "Can't allocate memory zone %s\n", 141 mz_name); 142 } 143 return memzone; 144 } 145 146 static int 147 comp_perf_allocate_external_mbufs(struct comp_test_data *test_data, 148 struct cperf_mem_resources *mem) 149 { 150 uint32_t i; 151 152 mem->comp_memzones = rte_zmalloc_socket(NULL, 153 test_data->total_segs * sizeof(struct rte_memzone *), 154 0, rte_socket_id()); 155 156 if (mem->comp_memzones == NULL) { 157 RTE_LOG(ERR, USER1, 158 "Memory to hold the compression memzones could not be allocated\n"); 159 return -1; 160 } 161 162 mem->decomp_memzones = rte_zmalloc_socket(NULL, 163 test_data->total_segs * sizeof(struct rte_memzone *), 164 0, rte_socket_id()); 165 166 if (mem->decomp_memzones == NULL) { 167 RTE_LOG(ERR, USER1, 168 "Memory to hold the decompression memzones could not be allocated\n"); 169 return -1; 170 } 171 172 mem->comp_buf_infos = rte_zmalloc_socket(NULL, 173 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 174 0, rte_socket_id()); 175 176 if (mem->comp_buf_infos == NULL) { 177 RTE_LOG(ERR, USER1, 178 "Memory to hold the compression buf infos could not be allocated\n"); 179 return -1; 180 } 181 182 mem->decomp_buf_infos = rte_zmalloc_socket(NULL, 183 test_data->total_segs * sizeof(struct rte_mbuf_ext_shared_info), 184 0, rte_socket_id()); 185 186 if (mem->decomp_buf_infos == NULL) { 187 RTE_LOG(ERR, USER1, 188 "Memory to hold the decompression buf infos could not be allocated\n"); 189 return -1; 190 } 191 192 for (i = 0; i < test_data->total_segs; i++) { 193 mem->comp_memzones[i] = comp_perf_make_memzone("comp", mem, 194 i, test_data->out_seg_sz); 195 if (mem->comp_memzones[i] == NULL) { 196 RTE_LOG(ERR, USER1, 197 "Memory to hold the compression memzone could not be allocated\n"); 198 return -1; 199 } 200 201 mem->decomp_memzones[i] = comp_perf_make_memzone("decomp", mem, 202 i, test_data->seg_sz); 203 if (mem->decomp_memzones[i] == NULL) { 204 RTE_LOG(ERR, USER1, 205 "Memory to hold the decompression memzone could not be allocated\n"); 206 return -1; 207 } 208 209 mem->comp_buf_infos[i].free_cb = 210 comp_perf_extbuf_free_cb; 211 mem->comp_buf_infos[i].fcb_opaque = NULL; 212 rte_mbuf_ext_refcnt_set(&mem->comp_buf_infos[i], 1); 213 214 mem->decomp_buf_infos[i].free_cb = 215 comp_perf_extbuf_free_cb; 216 mem->decomp_buf_infos[i].fcb_opaque = NULL; 217 rte_mbuf_ext_refcnt_set(&mem->decomp_buf_infos[i], 1); 218 } 219 220 return 0; 221 } 222 223 int 224 comp_perf_allocate_memory(struct comp_test_data *test_data, 225 struct cperf_mem_resources *mem) 226 { 227 uint16_t comp_mbuf_size; 228 uint16_t decomp_mbuf_size; 229 230 test_data->out_seg_sz = find_buf_size(test_data->seg_sz); 231 232 /* Number of segments for input and output 233 * (compression and decompression) 234 */ 235 test_data->total_segs = DIV_CEIL(test_data->input_data_sz, 236 test_data->seg_sz); 237 238 if (test_data->use_external_mbufs != 0) { 239 if (comp_perf_allocate_external_mbufs(test_data, mem) < 0) 240 return -1; 241 comp_mbuf_size = 0; 242 decomp_mbuf_size = 0; 243 } else { 244 comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; 245 decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; 246 } 247 248 char pool_name[32] = ""; 249 250 snprintf(pool_name, sizeof(pool_name), "comp_buf_pool_%u_qp_%u", 251 mem->dev_id, mem->qp_id); 252 mem->comp_buf_pool = rte_pktmbuf_pool_create(pool_name, 253 test_data->total_segs, 254 0, 0, 255 comp_mbuf_size, 256 rte_socket_id()); 257 if (mem->comp_buf_pool == NULL) { 258 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 259 return -1; 260 } 261 262 snprintf(pool_name, sizeof(pool_name), "decomp_buf_pool_%u_qp_%u", 263 mem->dev_id, mem->qp_id); 264 mem->decomp_buf_pool = rte_pktmbuf_pool_create(pool_name, 265 test_data->total_segs, 266 0, 0, 267 decomp_mbuf_size, 268 rte_socket_id()); 269 if (mem->decomp_buf_pool == NULL) { 270 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n"); 271 return -1; 272 } 273 274 mem->total_bufs = DIV_CEIL(test_data->total_segs, 275 test_data->max_sgl_segs); 276 277 snprintf(pool_name, sizeof(pool_name), "op_pool_%u_qp_%u", 278 mem->dev_id, mem->qp_id); 279 mem->op_pool = rte_comp_op_pool_create(pool_name, 280 mem->total_bufs, 281 0, 0, rte_socket_id()); 282 if (mem->op_pool == NULL) { 283 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n"); 284 return -1; 285 } 286 287 /* 288 * Compressed data might be a bit larger than input data, 289 * if data cannot be compressed 290 */ 291 mem->compressed_data = rte_zmalloc_socket(NULL, 292 RTE_MAX( 293 (size_t) test_data->out_seg_sz * 294 test_data->total_segs, 295 (size_t) MIN_COMPRESSED_BUF_SIZE), 296 0, 297 rte_socket_id()); 298 if (mem->compressed_data == NULL) { 299 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 300 "file could not be allocated\n"); 301 return -1; 302 } 303 304 mem->decompressed_data = rte_zmalloc_socket(NULL, 305 test_data->input_data_sz, 0, 306 rte_socket_id()); 307 if (mem->decompressed_data == NULL) { 308 RTE_LOG(ERR, USER1, "Memory to hold the data from the input " 309 "file could not be allocated\n"); 310 return -1; 311 } 312 313 mem->comp_bufs = rte_zmalloc_socket(NULL, 314 mem->total_bufs * sizeof(struct rte_mbuf *), 315 0, rte_socket_id()); 316 if (mem->comp_bufs == NULL) { 317 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs" 318 " could not be allocated\n"); 319 return -1; 320 } 321 322 mem->decomp_bufs = rte_zmalloc_socket(NULL, 323 mem->total_bufs * sizeof(struct rte_mbuf *), 324 0, rte_socket_id()); 325 if (mem->decomp_bufs == NULL) { 326 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs" 327 " could not be allocated\n"); 328 return -1; 329 } 330 331 buffer_info.total_segments = test_data->total_segs; 332 buffer_info.segment_sz = test_data->seg_sz; 333 buffer_info.total_buffs = mem->total_bufs; 334 buffer_info.segments_per_buff = test_data->max_sgl_segs; 335 buffer_info.input_data_sz = test_data->input_data_sz; 336 337 return 0; 338 } 339 340 int 341 prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) 342 { 343 uint32_t remaining_data = test_data->input_data_sz; 344 uint8_t *input_data_ptr = test_data->input_data; 345 size_t data_sz = 0; 346 uint8_t *data_addr; 347 uint32_t i, j; 348 uint16_t segs_per_mbuf = 0; 349 uint32_t cmz = 0; 350 uint32_t dmz = 0; 351 352 for (i = 0; i < mem->total_bufs; i++) { 353 /* Allocate data in input mbuf and copy data from input file */ 354 mem->decomp_bufs[i] = 355 rte_pktmbuf_alloc(mem->decomp_buf_pool); 356 if (mem->decomp_bufs[i] == NULL) { 357 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 358 return -1; 359 } 360 361 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 362 363 if (test_data->use_external_mbufs != 0) { 364 rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i], 365 mem->decomp_memzones[dmz]->addr, 366 mem->decomp_memzones[dmz]->iova, 367 test_data->seg_sz, 368 &mem->decomp_buf_infos[dmz]); 369 dmz++; 370 } 371 372 data_addr = (uint8_t *) rte_pktmbuf_append( 373 mem->decomp_bufs[i], data_sz); 374 if (data_addr == NULL) { 375 RTE_LOG(ERR, USER1, "Could not append data\n"); 376 return -1; 377 } 378 rte_memcpy(data_addr, input_data_ptr, data_sz); 379 380 input_data_ptr += data_sz; 381 remaining_data -= data_sz; 382 383 /* Already one segment in the mbuf */ 384 segs_per_mbuf = 1; 385 386 /* Chain mbufs if needed for input mbufs */ 387 while (segs_per_mbuf < test_data->max_sgl_segs 388 && remaining_data > 0) { 389 struct rte_mbuf *next_seg = 390 rte_pktmbuf_alloc(mem->decomp_buf_pool); 391 392 if (next_seg == NULL) { 393 RTE_LOG(ERR, USER1, 394 "Could not allocate mbuf\n"); 395 return -1; 396 } 397 398 data_sz = RTE_MIN(remaining_data, test_data->seg_sz); 399 400 if (test_data->use_external_mbufs != 0) { 401 rte_pktmbuf_attach_extbuf( 402 next_seg, 403 mem->decomp_memzones[dmz]->addr, 404 mem->decomp_memzones[dmz]->iova, 405 test_data->seg_sz, 406 &mem->decomp_buf_infos[dmz]); 407 dmz++; 408 } 409 410 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 411 data_sz); 412 413 if (data_addr == NULL) { 414 RTE_LOG(ERR, USER1, "Could not append data\n"); 415 return -1; 416 } 417 418 rte_memcpy(data_addr, input_data_ptr, data_sz); 419 input_data_ptr += data_sz; 420 remaining_data -= data_sz; 421 422 if (rte_pktmbuf_chain(mem->decomp_bufs[i], 423 next_seg) < 0) { 424 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 425 return -1; 426 } 427 segs_per_mbuf++; 428 } 429 430 /* Allocate data in output mbuf */ 431 mem->comp_bufs[i] = 432 rte_pktmbuf_alloc(mem->comp_buf_pool); 433 if (mem->comp_bufs[i] == NULL) { 434 RTE_LOG(ERR, USER1, "Could not allocate mbuf\n"); 435 return -1; 436 } 437 438 if (test_data->use_external_mbufs != 0) { 439 rte_pktmbuf_attach_extbuf(mem->comp_bufs[i], 440 mem->comp_memzones[cmz]->addr, 441 mem->comp_memzones[cmz]->iova, 442 test_data->out_seg_sz, 443 &mem->comp_buf_infos[cmz]); 444 cmz++; 445 } 446 447 data_addr = (uint8_t *) rte_pktmbuf_append( 448 mem->comp_bufs[i], 449 test_data->out_seg_sz); 450 if (data_addr == NULL) { 451 RTE_LOG(ERR, USER1, "Could not append data\n"); 452 return -1; 453 } 454 455 /* Chain mbufs if needed for output mbufs */ 456 for (j = 1; j < segs_per_mbuf; j++) { 457 struct rte_mbuf *next_seg = 458 rte_pktmbuf_alloc(mem->comp_buf_pool); 459 460 if (next_seg == NULL) { 461 RTE_LOG(ERR, USER1, 462 "Could not allocate mbuf\n"); 463 return -1; 464 } 465 466 if (test_data->use_external_mbufs != 0) { 467 rte_pktmbuf_attach_extbuf( 468 next_seg, 469 mem->comp_memzones[cmz]->addr, 470 mem->comp_memzones[cmz]->iova, 471 test_data->out_seg_sz, 472 &mem->comp_buf_infos[cmz]); 473 cmz++; 474 } 475 476 data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, 477 test_data->out_seg_sz); 478 if (data_addr == NULL) { 479 RTE_LOG(ERR, USER1, "Could not append data\n"); 480 return -1; 481 } 482 483 if (rte_pktmbuf_chain(mem->comp_bufs[i], 484 next_seg) < 0) { 485 RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); 486 return -1; 487 } 488 } 489 } 490 491 buffer_info.segments_per_last_buff = segs_per_mbuf; 492 buffer_info.last_segment_sz = data_sz; 493 494 return 0; 495 } 496 497 void 498 print_test_dynamics(void) 499 { 500 uint32_t opt_total_segs = DIV_CEIL(buffer_info.input_data_sz, 501 MAX_SEG_SIZE); 502 503 if (buffer_info.total_buffs > 1) { 504 printf("\nWarning: for the current input parameters, number" 505 " of ops is higher than one, which may result" 506 " in sub-optimal performance.\n"); 507 printf("To improve the performance (for the current" 508 " input data) following parameters are" 509 " suggested:\n"); 510 printf(" * Segment size: %d\n", MAX_SEG_SIZE); 511 printf(" * Number of segments: %u\n", opt_total_segs); 512 } else if (buffer_info.total_buffs == 1) { 513 printf("\nInfo: there is only one op with %u segments -" 514 " the compression ratio is the best.\n", 515 buffer_info.segments_per_last_buff); 516 if (buffer_info.segment_sz < MAX_SEG_SIZE) 517 printf("To reduce compression time, please use" 518 " bigger segment size: %d.\n", 519 MAX_SEG_SIZE); 520 else if (buffer_info.segment_sz == MAX_SEG_SIZE) 521 printf("Segment size is optimal for the best" 522 " performance.\n"); 523 } else 524 printf("Warning: something wrong happened!!\n"); 525 526 printf("\nFor the current input parameters (segment size = %u," 527 " maximum segments per SGL = %u):\n", 528 buffer_info.segment_sz, 529 buffer_info.segments_per_buff); 530 printf(" * Total number of buffers: %d\n", 531 buffer_info.total_segments); 532 printf(" * %u buffer(s) %u bytes long, last buffer %u" 533 " byte(s) long\n", 534 buffer_info.total_segments - 1, 535 buffer_info.segment_sz, 536 buffer_info.last_segment_sz); 537 printf(" * Number of ops: %u\n", buffer_info.total_buffs); 538 printf(" * Total memory allocation: %u\n", 539 (buffer_info.total_segments - 1) * buffer_info.segment_sz 540 + buffer_info.last_segment_sz); 541 if (buffer_info.total_buffs > 1) 542 printf(" * %u ops: %u segment(s) in each," 543 " segment size %u\n", 544 buffer_info.total_buffs - 1, 545 buffer_info.segments_per_buff, 546 buffer_info.segment_sz); 547 if (buffer_info.segments_per_last_buff > 1) { 548 printf(" * 1 op %u segments:\n", 549 buffer_info.segments_per_last_buff); 550 printf(" o %u segment size %u\n", 551 buffer_info.segments_per_last_buff - 1, 552 buffer_info.segment_sz); 553 printf(" o last segment size %u\n", 554 buffer_info.last_segment_sz); 555 } else if (buffer_info.segments_per_last_buff == 1) { 556 printf(" * 1 op (the last one): %u segment %u" 557 " byte(s) long\n\n", 558 buffer_info.segments_per_last_buff, 559 buffer_info.last_segment_sz); 560 } 561 printf("\n"); 562 } 563