1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_bus_vdev.h> 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 #include <rte_mbuf.h> 10 #include <rte_compressdev_pmd.h> 11 12 #include "isal_compress_pmd_private.h" 13 14 #define RTE_COMP_ISAL_WINDOW_SIZE 15 15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ 16 #define RTE_COMP_ISAL_LEVEL_ONE 1 17 #define RTE_COMP_ISAL_LEVEL_TWO 2 18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ 19 #define CHKSUM_SZ_CRC 8 20 #define CHKSUM_SZ_ADLER 4 21 22 #define STRINGIFY(s) #s 23 #define ISAL_TOSTRING(maj, min, patch) \ 24 STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch) 25 #define ISAL_VERSION_STRING \ 26 ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION) 27 28 int isal_logtype_driver; 29 30 /* Verify and set private xform parameters */ 31 int 32 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, 33 const struct rte_comp_xform *xform) 34 { 35 if (xform == NULL) 36 return -EINVAL; 37 38 /* Set compression private xform variables */ 39 if (xform->type == RTE_COMP_COMPRESS) { 40 /* Set private xform type - COMPRESS/DECOMPRESS */ 41 priv_xform->type = RTE_COMP_COMPRESS; 42 43 /* Set private xform algorithm */ 44 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { 45 if (xform->compress.algo == RTE_COMP_ALGO_NULL) { 46 ISAL_PMD_LOG(ERR, "By-pass not supported\n"); 47 return -ENOTSUP; 48 } 49 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 50 return -ENOTSUP; 51 } 52 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; 53 54 /* Set private xform window size, 32K supported */ 55 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 56 priv_xform->compress.window_size = 57 RTE_COMP_ISAL_WINDOW_SIZE; 58 else { 59 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 60 return -ENOTSUP; 61 } 62 63 /* Set private xform huffman type */ 64 switch (xform->compress.deflate.huffman) { 65 case(RTE_COMP_HUFFMAN_DEFAULT): 66 priv_xform->compress.deflate.huffman = 67 RTE_COMP_HUFFMAN_DEFAULT; 68 break; 69 case(RTE_COMP_HUFFMAN_FIXED): 70 priv_xform->compress.deflate.huffman = 71 RTE_COMP_HUFFMAN_FIXED; 72 break; 73 case(RTE_COMP_HUFFMAN_DYNAMIC): 74 priv_xform->compress.deflate.huffman = 75 RTE_COMP_HUFFMAN_DYNAMIC; 76 break; 77 default: 78 ISAL_PMD_LOG(ERR, "Huffman code not supported\n"); 79 return -ENOTSUP; 80 } 81 82 /* Set private xform checksum */ 83 switch (xform->compress.chksum) { 84 /* Raw deflate by default */ 85 case(RTE_COMP_CHECKSUM_NONE): 86 priv_xform->compress.chksum = IGZIP_DEFLATE; 87 break; 88 case(RTE_COMP_CHECKSUM_CRC32): 89 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR; 90 break; 91 case(RTE_COMP_CHECKSUM_ADLER32): 92 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR; 93 break; 94 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 95 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 96 " supported\n"); 97 return -ENOTSUP; 98 default: 99 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 100 priv_xform->compress.chksum = IGZIP_DEFLATE; 101 break; 102 } 103 104 /* Set private xform level. 105 * Checking compliance with compressdev API, -1 <= level => 9 106 */ 107 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || 108 xform->compress.level > RTE_COMP_LEVEL_MAX) { 109 ISAL_PMD_LOG(ERR, "Compression level out of range\n"); 110 return -EINVAL; 111 } 112 /* Check for Compressdev API level 0, No compression 113 * not supported in ISA-L 114 */ 115 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { 116 ISAL_PMD_LOG(ERR, "No Compression not supported\n"); 117 return -ENOTSUP; 118 } 119 /* If using fixed huffman code, level must be 0 */ 120 else if (priv_xform->compress.deflate.huffman == 121 RTE_COMP_HUFFMAN_FIXED) { 122 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" 123 " fixed huffman code\n"); 124 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; 125 priv_xform->level_buffer_size = 126 ISAL_DEF_LVL0_DEFAULT; 127 } else { 128 /* Mapping API levels to ISA-L levels 1,2 & 3 */ 129 switch (xform->compress.level) { 130 case RTE_COMP_LEVEL_PMD_DEFAULT: 131 /* Default is 1 if not using fixed huffman */ 132 priv_xform->compress.level = 133 RTE_COMP_ISAL_LEVEL_ONE; 134 priv_xform->level_buffer_size = 135 ISAL_DEF_LVL1_DEFAULT; 136 break; 137 case RTE_COMP_LEVEL_MIN: 138 priv_xform->compress.level = 139 RTE_COMP_ISAL_LEVEL_ONE; 140 priv_xform->level_buffer_size = 141 ISAL_DEF_LVL1_DEFAULT; 142 break; 143 case RTE_COMP_ISAL_LEVEL_TWO: 144 priv_xform->compress.level = 145 RTE_COMP_ISAL_LEVEL_TWO; 146 priv_xform->level_buffer_size = 147 ISAL_DEF_LVL2_DEFAULT; 148 break; 149 /* Level 3 or higher requested */ 150 default: 151 /* Check for AVX512, to use ISA-L level 3 */ 152 if (rte_cpu_get_flag_enabled( 153 RTE_CPUFLAG_AVX512F)) { 154 priv_xform->compress.level = 155 RTE_COMP_ISAL_LEVEL_THREE; 156 priv_xform->level_buffer_size = 157 ISAL_DEF_LVL3_DEFAULT; 158 } 159 /* Check for AVX2, to use ISA-L level 3 */ 160 else if (rte_cpu_get_flag_enabled( 161 RTE_CPUFLAG_AVX2)) { 162 priv_xform->compress.level = 163 RTE_COMP_ISAL_LEVEL_THREE; 164 priv_xform->level_buffer_size = 165 ISAL_DEF_LVL3_DEFAULT; 166 } else { 167 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" 168 " 3 or above; Level 3 optimized" 169 " for AVX512 & AVX2 only." 170 " level changed to 2.\n"); 171 priv_xform->compress.level = 172 RTE_COMP_ISAL_LEVEL_TWO; 173 priv_xform->level_buffer_size = 174 ISAL_DEF_LVL2_DEFAULT; 175 } 176 } 177 } 178 } 179 180 /* Set decompression private xform variables */ 181 else if (xform->type == RTE_COMP_DECOMPRESS) { 182 183 /* Set private xform type - COMPRESS/DECOMPRESS */ 184 priv_xform->type = RTE_COMP_DECOMPRESS; 185 186 /* Set private xform algorithm */ 187 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { 188 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { 189 ISAL_PMD_LOG(ERR, "By pass not supported\n"); 190 return -ENOTSUP; 191 } 192 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 193 return -ENOTSUP; 194 } 195 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; 196 197 /* Set private xform checksum */ 198 switch (xform->decompress.chksum) { 199 /* Raw deflate by default */ 200 case(RTE_COMP_CHECKSUM_NONE): 201 priv_xform->decompress.chksum = ISAL_DEFLATE; 202 break; 203 case(RTE_COMP_CHECKSUM_CRC32): 204 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR; 205 break; 206 case(RTE_COMP_CHECKSUM_ADLER32): 207 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR; 208 break; 209 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 210 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 211 " supported\n"); 212 return -ENOTSUP; 213 default: 214 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 215 priv_xform->decompress.chksum = ISAL_DEFLATE; 216 break; 217 } 218 219 /* Set private xform window size, 32K supported */ 220 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 221 priv_xform->decompress.window_size = 222 RTE_COMP_ISAL_WINDOW_SIZE; 223 else { 224 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 225 return -ENOTSUP; 226 } 227 } 228 return 0; 229 } 230 231 /* Compression using chained mbufs for input/output data */ 232 static int 233 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) 234 { 235 int ret; 236 uint32_t remaining_offset; 237 uint32_t remaining_data = op->src.length; 238 struct rte_mbuf *src = op->m_src; 239 struct rte_mbuf *dst = op->m_dst; 240 241 /* check for source/destination offset passing multiple segments 242 * and point compression stream to input/output buffer. 243 */ 244 remaining_offset = op->src.offset; 245 while (remaining_offset >= src->data_len) { 246 remaining_offset -= src->data_len; 247 src = src->next; 248 } 249 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, 250 op->src.length); 251 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 252 remaining_offset); 253 254 remaining_offset = op->dst.offset; 255 while (remaining_offset >= dst->data_len) { 256 remaining_offset -= dst->data_len; 257 dst = dst->next; 258 } 259 qp->stream->avail_out = dst->data_len - remaining_offset; 260 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 261 remaining_offset); 262 263 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 264 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n"); 265 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 266 return -1; 267 } 268 269 while (qp->stream->internal_state.state != ZSTATE_END) { 270 /* Last segment of data */ 271 if (remaining_data <= src->data_len) 272 qp->stream->end_of_stream = 1; 273 274 /* Execute compression operation */ 275 ret = isal_deflate(qp->stream); 276 277 remaining_data = op->src.length - qp->stream->total_in; 278 279 if (ret != COMP_OK) { 280 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 281 op->status = RTE_COMP_OP_STATUS_ERROR; 282 return ret; 283 } 284 285 if (qp->stream->avail_in == 0 && 286 qp->stream->total_in != op->src.length) { 287 if (src->next != NULL) { 288 src = src->next; 289 qp->stream->next_in = 290 rte_pktmbuf_mtod(src, uint8_t *); 291 qp->stream->avail_in = 292 RTE_MIN(remaining_data, src->data_len); 293 } else { 294 ISAL_PMD_LOG(ERR, 295 "Not enough input buffer segments\n"); 296 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 297 return -1; 298 } 299 } 300 301 if (qp->stream->avail_out == 0 && 302 qp->stream->internal_state.state != ZSTATE_END) { 303 if (dst->next != NULL) { 304 dst = dst->next; 305 qp->stream->next_out = 306 rte_pktmbuf_mtod(dst, uint8_t *); 307 qp->stream->avail_out = dst->data_len; 308 } else { 309 ISAL_PMD_LOG(ERR, 310 "Not enough output buffer segments\n"); 311 op->status = 312 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 313 return -1; 314 } 315 } 316 } 317 318 return 0; 319 } 320 321 /* Decompression using chained mbufs for input/output data */ 322 static int 323 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) 324 { 325 int ret; 326 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; 327 uint32_t remaining_data = op->src.length; 328 struct rte_mbuf *src = op->m_src; 329 struct rte_mbuf *dst = op->m_dst; 330 331 /* check for offset passing multiple segments 332 * and point decompression state to input/output buffer 333 */ 334 src_remaining_offset = op->src.offset; 335 while (src_remaining_offset >= src->data_len) { 336 src_remaining_offset -= src->data_len; 337 src = src->next; 338 } 339 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, 340 op->src.length); 341 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 342 src_remaining_offset); 343 344 dst_remaining_offset = op->dst.offset; 345 while (dst_remaining_offset >= dst->data_len) { 346 dst_remaining_offset -= dst->data_len; 347 dst = dst->next; 348 } 349 qp->state->avail_out = dst->data_len - dst_remaining_offset; 350 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 351 dst_remaining_offset); 352 353 while (qp->state->block_state != ISAL_BLOCK_FINISH) { 354 355 ret = isal_inflate(qp->state); 356 357 /* Check for first segment, offset needs to be accounted for */ 358 if (remaining_data == op->src.length) { 359 consumed_data = src->data_len - src_remaining_offset; 360 } else 361 consumed_data = src->data_len; 362 363 if (qp->state->avail_in == 0 364 && op->consumed != op->src.length) { 365 op->consumed += consumed_data; 366 remaining_data -= consumed_data; 367 368 if (src->next != NULL) { 369 src = src->next; 370 qp->state->next_in = 371 rte_pktmbuf_mtod(src, uint8_t *); 372 qp->state->avail_in = 373 RTE_MIN(remaining_data, src->data_len); 374 } 375 } 376 377 if (ret == ISAL_OUT_OVERFLOW) { 378 ISAL_PMD_LOG(ERR, "Decompression operation ran " 379 "out of space, but can be recovered.\n%d bytes " 380 "consumed\t%d bytes produced\n", 381 consumed_data, qp->state->total_out); 382 op->status = 383 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; 384 return ret; 385 } else if (ret < 0) { 386 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 387 op->status = RTE_COMP_OP_STATUS_ERROR; 388 return ret; 389 } 390 391 if (qp->state->avail_out == 0 && 392 qp->state->block_state != ISAL_BLOCK_FINISH) { 393 if (dst->next != NULL) { 394 dst = dst->next; 395 qp->state->next_out = 396 rte_pktmbuf_mtod(dst, uint8_t *); 397 qp->state->avail_out = dst->data_len; 398 } else { 399 ISAL_PMD_LOG(ERR, 400 "Not enough output buffer segments\n"); 401 op->status = 402 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 403 return -1; 404 } 405 } 406 } 407 408 return 0; 409 } 410 411 /* Stateless Compression Function */ 412 static int 413 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 414 struct isal_priv_xform *priv_xform) 415 { 416 int ret = 0; 417 op->status = RTE_COMP_OP_STATUS_SUCCESS; 418 419 /* Required due to init clearing level_buf */ 420 uint8_t *temp_level_buf = qp->stream->level_buf; 421 422 /* Initialize compression stream */ 423 isal_deflate_init(qp->stream); 424 425 qp->stream->level_buf = temp_level_buf; 426 427 /* Set Checksum flag */ 428 qp->stream->gzip_flag = priv_xform->compress.chksum; 429 430 /* Stateless operation, input will be consumed in one go */ 431 qp->stream->flush = NO_FLUSH; 432 433 /* set compression level & intermediate level buffer size */ 434 qp->stream->level = priv_xform->compress.level; 435 qp->stream->level_buf_size = priv_xform->level_buffer_size; 436 437 /* Set op huffman code */ 438 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) 439 isal_deflate_set_hufftables(qp->stream, NULL, 440 IGZIP_HUFFTABLE_STATIC); 441 else if (priv_xform->compress.deflate.huffman == 442 RTE_COMP_HUFFMAN_DEFAULT) 443 isal_deflate_set_hufftables(qp->stream, NULL, 444 IGZIP_HUFFTABLE_DEFAULT); 445 /* Dynamically change the huffman code to suit the input data */ 446 else if (priv_xform->compress.deflate.huffman == 447 RTE_COMP_HUFFMAN_DYNAMIC) 448 isal_deflate_set_hufftables(qp->stream, NULL, 449 IGZIP_HUFFTABLE_DEFAULT); 450 451 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 452 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 453 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 454 return -1; 455 } 456 457 if (op->dst.offset >= op->m_dst->pkt_len) { 458 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" 459 " for offset provided.\n"); 460 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 461 return -1; 462 } 463 464 /* Chained mbufs */ 465 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 466 ret = chained_mbuf_compression(op, qp); 467 if (ret < 0) 468 return ret; 469 } else { 470 /* Linear buffer */ 471 qp->stream->end_of_stream = 1; /* All input consumed in one */ 472 /* Point compression stream to input buffer */ 473 qp->stream->avail_in = op->src.length; 474 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, 475 uint8_t *, op->src.offset); 476 477 /* Point compression stream to output buffer */ 478 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; 479 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 480 uint8_t *, op->dst.offset); 481 482 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 483 ISAL_PMD_LOG(ERR, "Invalid source or destination" 484 " buffers\n"); 485 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 486 return -1; 487 } 488 489 /* Execute compression operation */ 490 ret = isal_deflate_stateless(qp->stream); 491 492 /* Check that output buffer did not run out of space */ 493 if (ret == STATELESS_OVERFLOW) { 494 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 495 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 496 return ret; 497 } 498 499 /* Check that input buffer has been fully consumed */ 500 if (qp->stream->avail_in != (uint32_t)0) { 501 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 502 " entirely\n"); 503 op->status = RTE_COMP_OP_STATUS_ERROR; 504 return -1; 505 } 506 507 if (ret != COMP_OK) { 508 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 509 op->status = RTE_COMP_OP_STATUS_ERROR; 510 return ret; 511 } 512 } 513 514 op->consumed = qp->stream->total_in; 515 if (qp->stream->gzip_flag == IGZIP_DEFLATE) { 516 op->produced = qp->stream->total_out; 517 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) { 518 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER; 519 op->output_chksum = qp->stream->internal_state.crc + 1; 520 } else { 521 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC; 522 op->output_chksum = qp->stream->internal_state.crc; 523 } 524 525 return ret; 526 } 527 528 /* Stateless Decompression Function */ 529 static int 530 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 531 struct isal_priv_xform *priv_xform) 532 { 533 int ret = 0; 534 535 op->status = RTE_COMP_OP_STATUS_SUCCESS; 536 537 /* Initialize decompression state */ 538 isal_inflate_init(qp->state); 539 540 /* Set Checksum flag */ 541 qp->state->crc_flag = priv_xform->decompress.chksum; 542 543 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 544 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 545 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 546 return -1; 547 } 548 549 if (op->dst.offset >= op->m_dst->pkt_len) { 550 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " 551 "offset provided.\n"); 552 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 553 return -1; 554 } 555 556 /* Chained mbufs */ 557 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 558 ret = chained_mbuf_decompression(op, qp); 559 if (ret != 0) 560 return ret; 561 } else { 562 /* Linear buffer */ 563 /* Point decompression state to input buffer */ 564 qp->state->avail_in = op->src.length; 565 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, 566 uint8_t *, op->src.offset); 567 568 /* Point decompression state to output buffer */ 569 qp->state->avail_out = op->m_dst->data_len - op->dst.offset; 570 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 571 uint8_t *, op->dst.offset); 572 573 if (unlikely(!qp->state->next_in || !qp->state->next_out)) { 574 ISAL_PMD_LOG(ERR, "Invalid source or destination" 575 " buffers\n"); 576 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 577 return -1; 578 } 579 580 /* Execute decompression operation */ 581 ret = isal_inflate_stateless(qp->state); 582 583 if (ret == ISAL_OUT_OVERFLOW) { 584 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 585 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 586 return ret; 587 } 588 589 /* Check that input buffer has been fully consumed */ 590 if (qp->state->avail_in != (uint32_t)0) { 591 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 592 " entirely\n"); 593 op->status = RTE_COMP_OP_STATUS_ERROR; 594 return -1; 595 } 596 597 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) { 598 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 599 op->status = RTE_COMP_OP_STATUS_ERROR; 600 return ret; 601 } 602 op->consumed = op->src.length - qp->state->avail_in; 603 } 604 op->produced = qp->state->total_out; 605 op->output_chksum = qp->state->crc; 606 607 return ret; 608 } 609 610 /* Process compression/decompression operation */ 611 static int 612 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, 613 struct isal_priv_xform *priv_xform) 614 { 615 switch (priv_xform->type) { 616 case RTE_COMP_COMPRESS: 617 process_isal_deflate(op, qp, priv_xform); 618 break; 619 case RTE_COMP_DECOMPRESS: 620 process_isal_inflate(op, qp, priv_xform); 621 break; 622 default: 623 ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); 624 return -ENOTSUP; 625 } 626 return 0; 627 } 628 629 /* Enqueue burst */ 630 static uint16_t 631 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 632 uint16_t nb_ops) 633 { 634 struct isal_comp_qp *qp = queue_pair; 635 uint16_t i; 636 int retval; 637 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); 638 639 for (i = 0; i < num_enq; i++) { 640 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { 641 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 642 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n"); 643 qp->qp_stats.enqueue_err_count++; 644 continue; 645 } 646 retval = process_op(qp, ops[i], ops[i]->private_xform); 647 if (unlikely(retval < 0) || 648 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { 649 qp->qp_stats.enqueue_err_count++; 650 } 651 } 652 653 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, 654 num_enq, NULL); 655 qp->num_free_elements -= retval; 656 qp->qp_stats.enqueued_count += retval; 657 658 return retval; 659 } 660 661 /* Dequeue burst */ 662 static uint16_t 663 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 664 uint16_t nb_ops) 665 { 666 struct isal_comp_qp *qp = queue_pair; 667 uint16_t nb_dequeued; 668 669 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, 670 nb_ops, NULL); 671 qp->num_free_elements += nb_dequeued; 672 qp->qp_stats.dequeued_count += nb_dequeued; 673 674 return nb_dequeued; 675 } 676 677 /* Create ISA-L compression device */ 678 static int 679 compdev_isal_create(const char *name, struct rte_vdev_device *vdev, 680 struct rte_compressdev_pmd_init_params *init_params) 681 { 682 struct rte_compressdev *dev; 683 684 dev = rte_compressdev_pmd_create(name, &vdev->device, 685 sizeof(struct isal_comp_private), init_params); 686 if (dev == NULL) { 687 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); 688 return -EFAULT; 689 } 690 691 dev->dev_ops = isal_compress_pmd_ops; 692 693 /* register rx/tx burst functions for data path */ 694 dev->dequeue_burst = isal_comp_pmd_dequeue_burst; 695 dev->enqueue_burst = isal_comp_pmd_enqueue_burst; 696 697 ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING); 698 699 return 0; 700 } 701 702 /** Remove compression device */ 703 static int 704 compdev_isal_remove_dev(struct rte_vdev_device *vdev) 705 { 706 struct rte_compressdev *compdev; 707 const char *name; 708 709 name = rte_vdev_device_name(vdev); 710 if (name == NULL) 711 return -EINVAL; 712 713 compdev = rte_compressdev_pmd_get_named_dev(name); 714 if (compdev == NULL) 715 return -ENODEV; 716 717 return rte_compressdev_pmd_destroy(compdev); 718 } 719 720 /** Initialise ISA-L compression device */ 721 static int 722 compdev_isal_probe(struct rte_vdev_device *dev) 723 { 724 struct rte_compressdev_pmd_init_params init_params = { 725 "", 726 rte_socket_id(), 727 }; 728 const char *name, *args; 729 int retval; 730 731 name = rte_vdev_device_name(dev); 732 if (name == NULL) 733 return -EINVAL; 734 735 args = rte_vdev_device_args(dev); 736 737 retval = rte_compressdev_pmd_parse_input_args(&init_params, args); 738 if (retval) { 739 ISAL_PMD_LOG(ERR, 740 "Failed to parse initialisation arguments[%s]\n", args); 741 return -EINVAL; 742 } 743 744 return compdev_isal_create(name, dev, &init_params); 745 } 746 747 static struct rte_vdev_driver compdev_isal_pmd_drv = { 748 .probe = compdev_isal_probe, 749 .remove = compdev_isal_remove_dev, 750 }; 751 752 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); 753 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, 754 "socket_id=<int>"); 755 756 RTE_INIT(isal_init_log) 757 { 758 isal_logtype_driver = rte_log_register("pmd.compress.isal"); 759 if (isal_logtype_driver >= 0) 760 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO); 761 } 762