1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_bus_vdev.h> 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 #include <rte_mbuf.h> 10 #include <rte_compressdev_pmd.h> 11 12 #include "isal_compress_pmd_private.h" 13 14 #define RTE_COMP_ISAL_WINDOW_SIZE 15 15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ 16 #define RTE_COMP_ISAL_LEVEL_ONE 1 17 #define RTE_COMP_ISAL_LEVEL_TWO 2 18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ 19 #define CHKSUM_SZ_CRC 8 20 #define CHKSUM_SZ_ADLER 4 21 22 #define STRINGIFY(s) #s 23 #define ISAL_TOSTRING(maj, min, patch) \ 24 STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch) 25 #define ISAL_VERSION_STRING \ 26 ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION) 27 28 /* Verify and set private xform parameters */ 29 int 30 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, 31 const struct rte_comp_xform *xform) 32 { 33 if (xform == NULL) 34 return -EINVAL; 35 36 /* Set compression private xform variables */ 37 if (xform->type == RTE_COMP_COMPRESS) { 38 /* Set private xform type - COMPRESS/DECOMPRESS */ 39 priv_xform->type = RTE_COMP_COMPRESS; 40 41 /* Set private xform algorithm */ 42 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { 43 if (xform->compress.algo == RTE_COMP_ALGO_NULL) { 44 ISAL_PMD_LOG(ERR, "By-pass not supported\n"); 45 return -ENOTSUP; 46 } 47 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 48 return -ENOTSUP; 49 } 50 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; 51 52 /* Set private xform window size, 32K supported */ 53 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 54 priv_xform->compress.window_size = 55 RTE_COMP_ISAL_WINDOW_SIZE; 56 else { 57 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 58 return -ENOTSUP; 59 } 60 61 /* Set private xform huffman type */ 62 switch (xform->compress.deflate.huffman) { 63 case(RTE_COMP_HUFFMAN_DEFAULT): 64 priv_xform->compress.deflate.huffman = 65 RTE_COMP_HUFFMAN_DEFAULT; 66 break; 67 case(RTE_COMP_HUFFMAN_FIXED): 68 priv_xform->compress.deflate.huffman = 69 RTE_COMP_HUFFMAN_FIXED; 70 break; 71 case(RTE_COMP_HUFFMAN_DYNAMIC): 72 priv_xform->compress.deflate.huffman = 73 RTE_COMP_HUFFMAN_DYNAMIC; 74 break; 75 default: 76 ISAL_PMD_LOG(ERR, "Huffman code not supported\n"); 77 return -ENOTSUP; 78 } 79 80 /* Set private xform checksum */ 81 switch (xform->compress.chksum) { 82 /* Raw deflate by default */ 83 case(RTE_COMP_CHECKSUM_NONE): 84 priv_xform->compress.chksum = IGZIP_DEFLATE; 85 break; 86 case(RTE_COMP_CHECKSUM_CRC32): 87 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR; 88 break; 89 case(RTE_COMP_CHECKSUM_ADLER32): 90 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR; 91 break; 92 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 93 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 94 " supported\n"); 95 return -ENOTSUP; 96 default: 97 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 98 priv_xform->compress.chksum = IGZIP_DEFLATE; 99 break; 100 } 101 102 /* Set private xform level. 103 * Checking compliance with compressdev API, -1 <= level => 9 104 */ 105 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || 106 xform->compress.level > RTE_COMP_LEVEL_MAX) { 107 ISAL_PMD_LOG(ERR, "Compression level out of range\n"); 108 return -EINVAL; 109 } 110 /* Check for Compressdev API level 0, No compression 111 * not supported in ISA-L 112 */ 113 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { 114 ISAL_PMD_LOG(ERR, "No Compression not supported\n"); 115 return -ENOTSUP; 116 } 117 /* If using fixed huffman code, level must be 0 */ 118 else if (priv_xform->compress.deflate.huffman == 119 RTE_COMP_HUFFMAN_FIXED) { 120 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" 121 " fixed huffman code\n"); 122 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; 123 priv_xform->level_buffer_size = 124 ISAL_DEF_LVL0_DEFAULT; 125 } else { 126 /* Mapping API levels to ISA-L levels 1,2 & 3 */ 127 switch (xform->compress.level) { 128 case RTE_COMP_LEVEL_PMD_DEFAULT: 129 /* Default is 1 if not using fixed huffman */ 130 priv_xform->compress.level = 131 RTE_COMP_ISAL_LEVEL_ONE; 132 priv_xform->level_buffer_size = 133 ISAL_DEF_LVL1_DEFAULT; 134 break; 135 case RTE_COMP_LEVEL_MIN: 136 priv_xform->compress.level = 137 RTE_COMP_ISAL_LEVEL_ONE; 138 priv_xform->level_buffer_size = 139 ISAL_DEF_LVL1_DEFAULT; 140 break; 141 case RTE_COMP_ISAL_LEVEL_TWO: 142 priv_xform->compress.level = 143 RTE_COMP_ISAL_LEVEL_TWO; 144 priv_xform->level_buffer_size = 145 ISAL_DEF_LVL2_DEFAULT; 146 break; 147 /* Level 3 or higher requested */ 148 default: 149 /* Check for AVX512, to use ISA-L level 3 */ 150 if (rte_cpu_get_flag_enabled( 151 RTE_CPUFLAG_AVX512F)) { 152 priv_xform->compress.level = 153 RTE_COMP_ISAL_LEVEL_THREE; 154 priv_xform->level_buffer_size = 155 ISAL_DEF_LVL3_DEFAULT; 156 } 157 /* Check for AVX2, to use ISA-L level 3 */ 158 else if (rte_cpu_get_flag_enabled( 159 RTE_CPUFLAG_AVX2)) { 160 priv_xform->compress.level = 161 RTE_COMP_ISAL_LEVEL_THREE; 162 priv_xform->level_buffer_size = 163 ISAL_DEF_LVL3_DEFAULT; 164 } else { 165 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" 166 " 3 or above; Level 3 optimized" 167 " for AVX512 & AVX2 only." 168 " level changed to 2.\n"); 169 priv_xform->compress.level = 170 RTE_COMP_ISAL_LEVEL_TWO; 171 priv_xform->level_buffer_size = 172 ISAL_DEF_LVL2_DEFAULT; 173 } 174 } 175 } 176 } 177 178 /* Set decompression private xform variables */ 179 else if (xform->type == RTE_COMP_DECOMPRESS) { 180 181 /* Set private xform type - COMPRESS/DECOMPRESS */ 182 priv_xform->type = RTE_COMP_DECOMPRESS; 183 184 /* Set private xform algorithm */ 185 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { 186 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { 187 ISAL_PMD_LOG(ERR, "By pass not supported\n"); 188 return -ENOTSUP; 189 } 190 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 191 return -ENOTSUP; 192 } 193 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; 194 195 /* Set private xform checksum */ 196 switch (xform->decompress.chksum) { 197 /* Raw deflate by default */ 198 case(RTE_COMP_CHECKSUM_NONE): 199 priv_xform->decompress.chksum = ISAL_DEFLATE; 200 break; 201 case(RTE_COMP_CHECKSUM_CRC32): 202 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR; 203 break; 204 case(RTE_COMP_CHECKSUM_ADLER32): 205 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR; 206 break; 207 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 208 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 209 " supported\n"); 210 return -ENOTSUP; 211 default: 212 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 213 priv_xform->decompress.chksum = ISAL_DEFLATE; 214 break; 215 } 216 217 /* Set private xform window size, 32K supported */ 218 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 219 priv_xform->decompress.window_size = 220 RTE_COMP_ISAL_WINDOW_SIZE; 221 else { 222 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 223 return -ENOTSUP; 224 } 225 } 226 return 0; 227 } 228 229 /* Compression using chained mbufs for input/output data */ 230 static int 231 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) 232 { 233 int ret; 234 uint32_t remaining_offset; 235 uint32_t remaining_data = op->src.length; 236 struct rte_mbuf *src = op->m_src; 237 struct rte_mbuf *dst = op->m_dst; 238 239 /* check for source/destination offset passing multiple segments 240 * and point compression stream to input/output buffer. 241 */ 242 remaining_offset = op->src.offset; 243 while (remaining_offset >= src->data_len) { 244 remaining_offset -= src->data_len; 245 src = src->next; 246 } 247 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, 248 op->src.length); 249 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 250 remaining_offset); 251 252 remaining_offset = op->dst.offset; 253 while (remaining_offset >= dst->data_len) { 254 remaining_offset -= dst->data_len; 255 dst = dst->next; 256 } 257 qp->stream->avail_out = dst->data_len - remaining_offset; 258 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 259 remaining_offset); 260 261 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 262 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n"); 263 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 264 return -1; 265 } 266 267 while (qp->stream->internal_state.state != ZSTATE_END) { 268 /* Last segment of data */ 269 if (remaining_data <= src->data_len) 270 qp->stream->end_of_stream = 1; 271 272 /* Execute compression operation */ 273 ret = isal_deflate(qp->stream); 274 275 remaining_data = op->src.length - qp->stream->total_in; 276 277 if (ret != COMP_OK) { 278 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 279 op->status = RTE_COMP_OP_STATUS_ERROR; 280 return ret; 281 } 282 283 if (qp->stream->avail_in == 0 && 284 qp->stream->total_in != op->src.length) { 285 if (src->next != NULL) { 286 src = src->next; 287 qp->stream->next_in = 288 rte_pktmbuf_mtod(src, uint8_t *); 289 qp->stream->avail_in = 290 RTE_MIN(remaining_data, src->data_len); 291 } else { 292 ISAL_PMD_LOG(ERR, 293 "Not enough input buffer segments\n"); 294 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 295 return -1; 296 } 297 } 298 299 if (qp->stream->avail_out == 0 && 300 qp->stream->internal_state.state != ZSTATE_END) { 301 if (dst->next != NULL) { 302 dst = dst->next; 303 qp->stream->next_out = 304 rte_pktmbuf_mtod(dst, uint8_t *); 305 qp->stream->avail_out = dst->data_len; 306 } else { 307 ISAL_PMD_LOG(ERR, 308 "Not enough output buffer segments\n"); 309 op->status = 310 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 311 return -1; 312 } 313 } 314 } 315 316 return 0; 317 } 318 319 /* Decompression using chained mbufs for input/output data */ 320 static int 321 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) 322 { 323 int ret; 324 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; 325 uint32_t remaining_data = op->src.length; 326 struct rte_mbuf *src = op->m_src; 327 struct rte_mbuf *dst = op->m_dst; 328 329 /* check for offset passing multiple segments 330 * and point decompression state to input/output buffer 331 */ 332 src_remaining_offset = op->src.offset; 333 while (src_remaining_offset >= src->data_len) { 334 src_remaining_offset -= src->data_len; 335 src = src->next; 336 } 337 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, 338 op->src.length); 339 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 340 src_remaining_offset); 341 342 dst_remaining_offset = op->dst.offset; 343 while (dst_remaining_offset >= dst->data_len) { 344 dst_remaining_offset -= dst->data_len; 345 dst = dst->next; 346 } 347 qp->state->avail_out = dst->data_len - dst_remaining_offset; 348 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 349 dst_remaining_offset); 350 351 while (qp->state->block_state != ISAL_BLOCK_FINISH) { 352 353 ret = isal_inflate(qp->state); 354 355 /* Check for first segment, offset needs to be accounted for */ 356 if (remaining_data == op->src.length) { 357 consumed_data = src->data_len - src_remaining_offset; 358 } else 359 consumed_data = src->data_len; 360 361 if (qp->state->avail_in == 0 362 && op->consumed != op->src.length) { 363 op->consumed += consumed_data; 364 remaining_data -= consumed_data; 365 366 if (src->next != NULL) { 367 src = src->next; 368 qp->state->next_in = 369 rte_pktmbuf_mtod(src, uint8_t *); 370 qp->state->avail_in = 371 RTE_MIN(remaining_data, src->data_len); 372 } 373 } 374 375 if (ret == ISAL_OUT_OVERFLOW) { 376 ISAL_PMD_LOG(ERR, "Decompression operation ran " 377 "out of space, but can be recovered.\n%d bytes " 378 "consumed\t%d bytes produced\n", 379 consumed_data, qp->state->total_out); 380 op->status = 381 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; 382 return ret; 383 } else if (ret < 0) { 384 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 385 op->status = RTE_COMP_OP_STATUS_ERROR; 386 return ret; 387 } 388 389 if (qp->state->avail_out == 0 && 390 qp->state->block_state != ISAL_BLOCK_FINISH) { 391 if (dst->next != NULL) { 392 dst = dst->next; 393 qp->state->next_out = 394 rte_pktmbuf_mtod(dst, uint8_t *); 395 qp->state->avail_out = dst->data_len; 396 } else { 397 ISAL_PMD_LOG(ERR, 398 "Not enough output buffer segments\n"); 399 op->status = 400 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 401 return -1; 402 } 403 } 404 } 405 406 return 0; 407 } 408 409 /* Stateless Compression Function */ 410 static int 411 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 412 struct isal_priv_xform *priv_xform) 413 { 414 int ret = 0; 415 op->status = RTE_COMP_OP_STATUS_SUCCESS; 416 417 /* Required due to init clearing level_buf */ 418 uint8_t *temp_level_buf = qp->stream->level_buf; 419 420 /* Initialize compression stream */ 421 isal_deflate_init(qp->stream); 422 423 qp->stream->level_buf = temp_level_buf; 424 425 /* Set Checksum flag */ 426 qp->stream->gzip_flag = priv_xform->compress.chksum; 427 428 /* Stateless operation, input will be consumed in one go */ 429 qp->stream->flush = NO_FLUSH; 430 431 /* set compression level & intermediate level buffer size */ 432 qp->stream->level = priv_xform->compress.level; 433 qp->stream->level_buf_size = priv_xform->level_buffer_size; 434 435 /* Set op huffman code */ 436 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) 437 isal_deflate_set_hufftables(qp->stream, NULL, 438 IGZIP_HUFFTABLE_STATIC); 439 else if (priv_xform->compress.deflate.huffman == 440 RTE_COMP_HUFFMAN_DEFAULT) 441 isal_deflate_set_hufftables(qp->stream, NULL, 442 IGZIP_HUFFTABLE_DEFAULT); 443 /* Dynamically change the huffman code to suit the input data */ 444 else if (priv_xform->compress.deflate.huffman == 445 RTE_COMP_HUFFMAN_DYNAMIC) 446 isal_deflate_set_hufftables(qp->stream, NULL, 447 IGZIP_HUFFTABLE_DEFAULT); 448 449 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 450 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 451 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 452 return -1; 453 } 454 455 if (op->dst.offset >= op->m_dst->pkt_len) { 456 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" 457 " for offset provided.\n"); 458 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 459 return -1; 460 } 461 462 /* Chained mbufs */ 463 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 464 ret = chained_mbuf_compression(op, qp); 465 if (ret < 0) 466 return ret; 467 } else { 468 /* Linear buffer */ 469 qp->stream->end_of_stream = 1; /* All input consumed in one */ 470 /* Point compression stream to input buffer */ 471 qp->stream->avail_in = op->src.length; 472 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, 473 uint8_t *, op->src.offset); 474 475 /* Point compression stream to output buffer */ 476 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; 477 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 478 uint8_t *, op->dst.offset); 479 480 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 481 ISAL_PMD_LOG(ERR, "Invalid source or destination" 482 " buffers\n"); 483 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 484 return -1; 485 } 486 487 /* Execute compression operation */ 488 ret = isal_deflate_stateless(qp->stream); 489 490 /* Check that output buffer did not run out of space */ 491 if (ret == STATELESS_OVERFLOW) { 492 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 493 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 494 return ret; 495 } 496 497 /* Check that input buffer has been fully consumed */ 498 if (qp->stream->avail_in != (uint32_t)0) { 499 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 500 " entirely\n"); 501 op->status = RTE_COMP_OP_STATUS_ERROR; 502 return -1; 503 } 504 505 if (ret != COMP_OK) { 506 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 507 op->status = RTE_COMP_OP_STATUS_ERROR; 508 return ret; 509 } 510 } 511 512 op->consumed = qp->stream->total_in; 513 if (qp->stream->gzip_flag == IGZIP_DEFLATE) { 514 op->produced = qp->stream->total_out; 515 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) { 516 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER; 517 op->output_chksum = qp->stream->internal_state.crc + 1; 518 } else { 519 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC; 520 op->output_chksum = qp->stream->internal_state.crc; 521 } 522 523 return ret; 524 } 525 526 /* Stateless Decompression Function */ 527 static int 528 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 529 struct isal_priv_xform *priv_xform) 530 { 531 int ret = 0; 532 533 op->status = RTE_COMP_OP_STATUS_SUCCESS; 534 535 /* Initialize decompression state */ 536 isal_inflate_init(qp->state); 537 538 /* Set Checksum flag */ 539 qp->state->crc_flag = priv_xform->decompress.chksum; 540 541 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 542 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 543 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 544 return -1; 545 } 546 547 if (op->dst.offset >= op->m_dst->pkt_len) { 548 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " 549 "offset provided.\n"); 550 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 551 return -1; 552 } 553 554 /* Chained mbufs */ 555 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 556 ret = chained_mbuf_decompression(op, qp); 557 if (ret != 0) 558 return ret; 559 } else { 560 /* Linear buffer */ 561 /* Point decompression state to input buffer */ 562 qp->state->avail_in = op->src.length; 563 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, 564 uint8_t *, op->src.offset); 565 566 /* Point decompression state to output buffer */ 567 qp->state->avail_out = op->m_dst->data_len - op->dst.offset; 568 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 569 uint8_t *, op->dst.offset); 570 571 if (unlikely(!qp->state->next_in || !qp->state->next_out)) { 572 ISAL_PMD_LOG(ERR, "Invalid source or destination" 573 " buffers\n"); 574 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 575 return -1; 576 } 577 578 /* Execute decompression operation */ 579 ret = isal_inflate_stateless(qp->state); 580 581 if (ret == ISAL_OUT_OVERFLOW) { 582 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 583 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 584 return ret; 585 } 586 587 /* Check that input buffer has been fully consumed */ 588 if (qp->state->avail_in != (uint32_t)0) { 589 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 590 " entirely\n"); 591 op->status = RTE_COMP_OP_STATUS_ERROR; 592 return -1; 593 } 594 595 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) { 596 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 597 op->status = RTE_COMP_OP_STATUS_ERROR; 598 return ret; 599 } 600 op->consumed = op->src.length - qp->state->avail_in; 601 } 602 op->produced = qp->state->total_out; 603 op->output_chksum = qp->state->crc; 604 605 return ret; 606 } 607 608 /* Process compression/decompression operation */ 609 static int 610 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, 611 struct isal_priv_xform *priv_xform) 612 { 613 switch (priv_xform->type) { 614 case RTE_COMP_COMPRESS: 615 process_isal_deflate(op, qp, priv_xform); 616 break; 617 case RTE_COMP_DECOMPRESS: 618 process_isal_inflate(op, qp, priv_xform); 619 break; 620 default: 621 ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); 622 return -ENOTSUP; 623 } 624 return 0; 625 } 626 627 /* Enqueue burst */ 628 static uint16_t 629 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 630 uint16_t nb_ops) 631 { 632 struct isal_comp_qp *qp = queue_pair; 633 uint16_t i; 634 int retval; 635 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); 636 637 for (i = 0; i < num_enq; i++) { 638 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { 639 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 640 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n"); 641 qp->qp_stats.enqueue_err_count++; 642 continue; 643 } 644 retval = process_op(qp, ops[i], ops[i]->private_xform); 645 if (unlikely(retval < 0) || 646 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { 647 qp->qp_stats.enqueue_err_count++; 648 } 649 } 650 651 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, 652 num_enq, NULL); 653 qp->num_free_elements -= retval; 654 qp->qp_stats.enqueued_count += retval; 655 656 return retval; 657 } 658 659 /* Dequeue burst */ 660 static uint16_t 661 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 662 uint16_t nb_ops) 663 { 664 struct isal_comp_qp *qp = queue_pair; 665 uint16_t nb_dequeued; 666 667 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, 668 nb_ops, NULL); 669 qp->num_free_elements += nb_dequeued; 670 qp->qp_stats.dequeued_count += nb_dequeued; 671 672 return nb_dequeued; 673 } 674 675 /* Create ISA-L compression device */ 676 static int 677 compdev_isal_create(const char *name, struct rte_vdev_device *vdev, 678 struct rte_compressdev_pmd_init_params *init_params) 679 { 680 struct rte_compressdev *dev; 681 682 dev = rte_compressdev_pmd_create(name, &vdev->device, 683 sizeof(struct isal_comp_private), init_params); 684 if (dev == NULL) { 685 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); 686 return -EFAULT; 687 } 688 689 dev->dev_ops = isal_compress_pmd_ops; 690 691 /* register rx/tx burst functions for data path */ 692 dev->dequeue_burst = isal_comp_pmd_dequeue_burst; 693 dev->enqueue_burst = isal_comp_pmd_enqueue_burst; 694 695 ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING); 696 697 return 0; 698 } 699 700 /** Remove compression device */ 701 static int 702 compdev_isal_remove_dev(struct rte_vdev_device *vdev) 703 { 704 struct rte_compressdev *compdev; 705 const char *name; 706 707 name = rte_vdev_device_name(vdev); 708 if (name == NULL) 709 return -EINVAL; 710 711 compdev = rte_compressdev_pmd_get_named_dev(name); 712 if (compdev == NULL) 713 return -ENODEV; 714 715 return rte_compressdev_pmd_destroy(compdev); 716 } 717 718 /** Initialise ISA-L compression device */ 719 static int 720 compdev_isal_probe(struct rte_vdev_device *dev) 721 { 722 struct rte_compressdev_pmd_init_params init_params = { 723 "", 724 rte_socket_id(), 725 }; 726 const char *name, *args; 727 int retval; 728 729 name = rte_vdev_device_name(dev); 730 if (name == NULL) 731 return -EINVAL; 732 733 args = rte_vdev_device_args(dev); 734 735 retval = rte_compressdev_pmd_parse_input_args(&init_params, args); 736 if (retval) { 737 ISAL_PMD_LOG(ERR, 738 "Failed to parse initialisation arguments[%s]\n", args); 739 return -EINVAL; 740 } 741 742 return compdev_isal_create(name, dev, &init_params); 743 } 744 745 static struct rte_vdev_driver compdev_isal_pmd_drv = { 746 .probe = compdev_isal_probe, 747 .remove = compdev_isal_remove_dev, 748 }; 749 750 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); 751 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, 752 "socket_id=<int>"); 753 RTE_LOG_REGISTER_DEFAULT(isal_logtype_driver, INFO); 754