1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <bus_vdev_driver.h> 7 #include <rte_common.h> 8 #include <rte_cpuflags.h> 9 #include <rte_malloc.h> 10 #include <rte_mbuf.h> 11 #include <rte_compressdev_pmd.h> 12 13 #include "isal_compress_pmd_private.h" 14 15 #define RTE_COMP_ISAL_WINDOW_SIZE 15 16 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ 17 #define RTE_COMP_ISAL_LEVEL_ONE 1 18 #define RTE_COMP_ISAL_LEVEL_TWO 2 19 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ 20 #define CHKSUM_SZ_CRC 8 21 #define CHKSUM_SZ_ADLER 4 22 23 #define STRINGIFY(s) #s 24 #define ISAL_TOSTRING(maj, min, patch) \ 25 STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch) 26 #define ISAL_VERSION_STRING \ 27 ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION) 28 29 /* Verify and set private xform parameters */ 30 int 31 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, 32 const struct rte_comp_xform *xform) 33 { 34 if (xform == NULL) 35 return -EINVAL; 36 37 /* Set compression private xform variables */ 38 if (xform->type == RTE_COMP_COMPRESS) { 39 /* Set private xform type - COMPRESS/DECOMPRESS */ 40 priv_xform->type = RTE_COMP_COMPRESS; 41 42 /* Set private xform algorithm */ 43 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { 44 if (xform->compress.algo == RTE_COMP_ALGO_NULL) { 45 ISAL_PMD_LOG(ERR, "By-pass not supported"); 46 return -ENOTSUP; 47 } 48 ISAL_PMD_LOG(ERR, "Algorithm not supported"); 49 return -ENOTSUP; 50 } 51 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; 52 53 /* Set private xform window size, 32K supported */ 54 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 55 priv_xform->compress.window_size = 56 RTE_COMP_ISAL_WINDOW_SIZE; 57 else { 58 ISAL_PMD_LOG(ERR, "Window size not supported"); 59 return -ENOTSUP; 60 } 61 62 /* Set private xform huffman type */ 63 switch (xform->compress.deflate.huffman) { 64 case(RTE_COMP_HUFFMAN_DEFAULT): 65 priv_xform->compress.deflate.huffman = 66 RTE_COMP_HUFFMAN_DEFAULT; 67 break; 68 case(RTE_COMP_HUFFMAN_FIXED): 69 priv_xform->compress.deflate.huffman = 70 RTE_COMP_HUFFMAN_FIXED; 71 break; 72 case(RTE_COMP_HUFFMAN_DYNAMIC): 73 priv_xform->compress.deflate.huffman = 74 RTE_COMP_HUFFMAN_DYNAMIC; 75 break; 76 default: 77 ISAL_PMD_LOG(ERR, "Huffman code not supported"); 78 return -ENOTSUP; 79 } 80 81 /* Set private xform checksum */ 82 switch (xform->compress.chksum) { 83 /* Raw deflate by default */ 84 case(RTE_COMP_CHECKSUM_NONE): 85 priv_xform->compress.chksum = IGZIP_DEFLATE; 86 break; 87 case(RTE_COMP_CHECKSUM_CRC32): 88 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR; 89 break; 90 case(RTE_COMP_CHECKSUM_ADLER32): 91 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR; 92 break; 93 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 94 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 95 " supported"); 96 return -ENOTSUP; 97 default: 98 ISAL_PMD_LOG(ERR, "Checksum type not supported"); 99 priv_xform->compress.chksum = IGZIP_DEFLATE; 100 break; 101 } 102 103 /* Set private xform level. 104 * Checking compliance with compressdev API, -1 <= level => 9 105 */ 106 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || 107 xform->compress.level > RTE_COMP_LEVEL_MAX) { 108 ISAL_PMD_LOG(ERR, "Compression level out of range"); 109 return -EINVAL; 110 } 111 /* Check for Compressdev API level 0, No compression 112 * not supported in ISA-L 113 */ 114 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { 115 ISAL_PMD_LOG(ERR, "No Compression not supported"); 116 return -ENOTSUP; 117 } 118 /* If using fixed huffman code, level must be 0 */ 119 else if (priv_xform->compress.deflate.huffman == 120 RTE_COMP_HUFFMAN_FIXED) { 121 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" 122 " fixed huffman code"); 123 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; 124 priv_xform->level_buffer_size = 125 ISAL_DEF_LVL0_DEFAULT; 126 } else { 127 /* Mapping API levels to ISA-L levels 1,2 & 3 */ 128 switch (xform->compress.level) { 129 case RTE_COMP_LEVEL_PMD_DEFAULT: 130 /* Default is 1 if not using fixed huffman */ 131 priv_xform->compress.level = 132 RTE_COMP_ISAL_LEVEL_ONE; 133 priv_xform->level_buffer_size = 134 ISAL_DEF_LVL1_DEFAULT; 135 break; 136 case RTE_COMP_LEVEL_MIN: 137 priv_xform->compress.level = 138 RTE_COMP_ISAL_LEVEL_ONE; 139 priv_xform->level_buffer_size = 140 ISAL_DEF_LVL1_DEFAULT; 141 break; 142 case RTE_COMP_ISAL_LEVEL_TWO: 143 priv_xform->compress.level = 144 RTE_COMP_ISAL_LEVEL_TWO; 145 priv_xform->level_buffer_size = 146 ISAL_DEF_LVL2_DEFAULT; 147 break; 148 /* Level 3 or higher requested */ 149 default: 150 #ifdef RTE_ARCH_X86 151 /* Check for AVX512, to use ISA-L level 3 */ 152 if (rte_cpu_get_flag_enabled( 153 RTE_CPUFLAG_AVX512F)) { 154 priv_xform->compress.level = 155 RTE_COMP_ISAL_LEVEL_THREE; 156 priv_xform->level_buffer_size = 157 ISAL_DEF_LVL3_DEFAULT; 158 } 159 /* Check for AVX2, to use ISA-L level 3 */ 160 else if (rte_cpu_get_flag_enabled( 161 RTE_CPUFLAG_AVX2)) { 162 priv_xform->compress.level = 163 RTE_COMP_ISAL_LEVEL_THREE; 164 priv_xform->level_buffer_size = 165 ISAL_DEF_LVL3_DEFAULT; 166 } else 167 #endif 168 { 169 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" 170 " 3 or above; Level 3 optimized" 171 " for AVX512 & AVX2 only." 172 " level changed to 2."); 173 priv_xform->compress.level = 174 RTE_COMP_ISAL_LEVEL_TWO; 175 priv_xform->level_buffer_size = 176 ISAL_DEF_LVL2_DEFAULT; 177 } 178 } 179 } 180 } 181 182 /* Set decompression private xform variables */ 183 else if (xform->type == RTE_COMP_DECOMPRESS) { 184 185 /* Set private xform type - COMPRESS/DECOMPRESS */ 186 priv_xform->type = RTE_COMP_DECOMPRESS; 187 188 /* Set private xform algorithm */ 189 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { 190 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { 191 ISAL_PMD_LOG(ERR, "By pass not supported"); 192 return -ENOTSUP; 193 } 194 ISAL_PMD_LOG(ERR, "Algorithm not supported"); 195 return -ENOTSUP; 196 } 197 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; 198 199 /* Set private xform checksum */ 200 switch (xform->decompress.chksum) { 201 /* Raw deflate by default */ 202 case(RTE_COMP_CHECKSUM_NONE): 203 priv_xform->decompress.chksum = ISAL_DEFLATE; 204 break; 205 case(RTE_COMP_CHECKSUM_CRC32): 206 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR; 207 break; 208 case(RTE_COMP_CHECKSUM_ADLER32): 209 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR; 210 break; 211 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 212 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 213 " supported"); 214 return -ENOTSUP; 215 default: 216 ISAL_PMD_LOG(ERR, "Checksum type not supported"); 217 priv_xform->decompress.chksum = ISAL_DEFLATE; 218 break; 219 } 220 221 /* Set private xform window size, 32K supported */ 222 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 223 priv_xform->decompress.window_size = 224 RTE_COMP_ISAL_WINDOW_SIZE; 225 else { 226 ISAL_PMD_LOG(ERR, "Window size not supported"); 227 return -ENOTSUP; 228 } 229 } 230 return 0; 231 } 232 233 /* Compression using chained mbufs for input/output data */ 234 static int 235 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) 236 { 237 int ret; 238 uint32_t remaining_offset; 239 uint32_t remaining_data = op->src.length; 240 struct rte_mbuf *src = op->m_src; 241 struct rte_mbuf *dst = op->m_dst; 242 243 /* check for source/destination offset passing multiple segments 244 * and point compression stream to input/output buffer. 245 */ 246 remaining_offset = op->src.offset; 247 while (remaining_offset >= src->data_len) { 248 remaining_offset -= src->data_len; 249 src = src->next; 250 } 251 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, 252 op->src.length); 253 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 254 remaining_offset); 255 256 remaining_offset = op->dst.offset; 257 while (remaining_offset >= dst->data_len) { 258 remaining_offset -= dst->data_len; 259 dst = dst->next; 260 } 261 qp->stream->avail_out = dst->data_len - remaining_offset; 262 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 263 remaining_offset); 264 265 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 266 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer"); 267 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 268 return -1; 269 } 270 271 while (qp->stream->internal_state.state != ZSTATE_END) { 272 /* Last segment of data */ 273 if (remaining_data <= src->data_len) 274 qp->stream->end_of_stream = 1; 275 276 /* Execute compression operation */ 277 ret = isal_deflate(qp->stream); 278 279 remaining_data = op->src.length - qp->stream->total_in; 280 281 if (ret != COMP_OK) { 282 ISAL_PMD_LOG(ERR, "Compression operation failed"); 283 op->status = RTE_COMP_OP_STATUS_ERROR; 284 return ret; 285 } 286 287 if (qp->stream->avail_in == 0 && 288 qp->stream->total_in != op->src.length) { 289 if (src->next != NULL) { 290 src = src->next; 291 qp->stream->next_in = 292 rte_pktmbuf_mtod(src, uint8_t *); 293 qp->stream->avail_in = 294 RTE_MIN(remaining_data, src->data_len); 295 } else { 296 ISAL_PMD_LOG(ERR, 297 "Not enough input buffer segments"); 298 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 299 return -1; 300 } 301 } 302 303 if (qp->stream->avail_out == 0 && 304 qp->stream->internal_state.state != ZSTATE_END) { 305 if (dst->next != NULL) { 306 dst = dst->next; 307 qp->stream->next_out = 308 rte_pktmbuf_mtod(dst, uint8_t *); 309 qp->stream->avail_out = dst->data_len; 310 } else { 311 ISAL_PMD_LOG(ERR, 312 "Not enough output buffer segments"); 313 op->status = 314 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 315 return -1; 316 } 317 } 318 } 319 320 return 0; 321 } 322 323 /* Decompression using chained mbufs for input/output data */ 324 static int 325 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) 326 { 327 int ret; 328 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; 329 uint32_t remaining_data = op->src.length; 330 struct rte_mbuf *src = op->m_src; 331 struct rte_mbuf *dst = op->m_dst; 332 333 /* check for offset passing multiple segments 334 * and point decompression state to input/output buffer 335 */ 336 src_remaining_offset = op->src.offset; 337 while (src_remaining_offset >= src->data_len) { 338 src_remaining_offset -= src->data_len; 339 src = src->next; 340 } 341 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, 342 op->src.length); 343 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 344 src_remaining_offset); 345 346 dst_remaining_offset = op->dst.offset; 347 while (dst_remaining_offset >= dst->data_len) { 348 dst_remaining_offset -= dst->data_len; 349 dst = dst->next; 350 } 351 qp->state->avail_out = dst->data_len - dst_remaining_offset; 352 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 353 dst_remaining_offset); 354 355 while (qp->state->block_state != ISAL_BLOCK_FINISH) { 356 357 ret = isal_inflate(qp->state); 358 359 /* Check for first segment, offset needs to be accounted for */ 360 if (remaining_data == op->src.length) { 361 consumed_data = src->data_len - src_remaining_offset; 362 } else 363 consumed_data = src->data_len; 364 365 if (qp->state->avail_in == 0 366 && op->consumed != op->src.length) { 367 op->consumed += consumed_data; 368 remaining_data -= consumed_data; 369 370 if (src->next != NULL) { 371 src = src->next; 372 qp->state->next_in = 373 rte_pktmbuf_mtod(src, uint8_t *); 374 qp->state->avail_in = 375 RTE_MIN(remaining_data, src->data_len); 376 } 377 } 378 379 if (ret == ISAL_OUT_OVERFLOW) { 380 ISAL_PMD_LOG(ERR, "Decompression operation ran " 381 "out of space, but can be recovered.%d bytes " 382 "consumed\t%d bytes produced", 383 consumed_data, qp->state->total_out); 384 op->status = 385 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; 386 return ret; 387 } else if (ret < 0) { 388 ISAL_PMD_LOG(ERR, "Decompression operation failed"); 389 op->status = RTE_COMP_OP_STATUS_ERROR; 390 return ret; 391 } 392 393 if (qp->state->avail_out == 0 && 394 qp->state->block_state != ISAL_BLOCK_FINISH) { 395 if (dst->next != NULL) { 396 dst = dst->next; 397 qp->state->next_out = 398 rte_pktmbuf_mtod(dst, uint8_t *); 399 qp->state->avail_out = dst->data_len; 400 } else { 401 ISAL_PMD_LOG(ERR, 402 "Not enough output buffer segments"); 403 op->status = 404 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 405 return -1; 406 } 407 } 408 } 409 410 return 0; 411 } 412 413 /* Stateless Compression Function */ 414 static int 415 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 416 struct isal_priv_xform *priv_xform) 417 { 418 int ret = 0; 419 op->status = RTE_COMP_OP_STATUS_SUCCESS; 420 421 /* Required due to init clearing level_buf */ 422 uint8_t *temp_level_buf = qp->stream->level_buf; 423 424 /* Initialize compression stream */ 425 isal_deflate_init(qp->stream); 426 427 qp->stream->level_buf = temp_level_buf; 428 429 /* Set Checksum flag */ 430 qp->stream->gzip_flag = priv_xform->compress.chksum; 431 432 /* Stateless operation, input will be consumed in one go */ 433 qp->stream->flush = NO_FLUSH; 434 435 /* set compression level & intermediate level buffer size */ 436 qp->stream->level = priv_xform->compress.level; 437 qp->stream->level_buf_size = priv_xform->level_buffer_size; 438 439 /* Set op huffman code */ 440 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) 441 isal_deflate_set_hufftables(qp->stream, NULL, 442 IGZIP_HUFFTABLE_STATIC); 443 else if (priv_xform->compress.deflate.huffman == 444 RTE_COMP_HUFFMAN_DEFAULT) 445 isal_deflate_set_hufftables(qp->stream, NULL, 446 IGZIP_HUFFTABLE_DEFAULT); 447 /* Dynamically change the huffman code to suit the input data */ 448 else if (priv_xform->compress.deflate.huffman == 449 RTE_COMP_HUFFMAN_DYNAMIC) 450 isal_deflate_set_hufftables(qp->stream, NULL, 451 IGZIP_HUFFTABLE_DEFAULT); 452 453 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 454 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough."); 455 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 456 return -1; 457 } 458 459 if (op->dst.offset >= op->m_dst->pkt_len) { 460 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" 461 " for offset provided."); 462 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 463 return -1; 464 } 465 466 /* Chained mbufs */ 467 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 468 ret = chained_mbuf_compression(op, qp); 469 if (ret < 0) 470 return ret; 471 } else { 472 /* Linear buffer */ 473 qp->stream->end_of_stream = 1; /* All input consumed in one */ 474 /* Point compression stream to input buffer */ 475 qp->stream->avail_in = op->src.length; 476 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, 477 uint8_t *, op->src.offset); 478 479 /* Point compression stream to output buffer */ 480 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; 481 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 482 uint8_t *, op->dst.offset); 483 484 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 485 ISAL_PMD_LOG(ERR, "Invalid source or destination" 486 " buffers"); 487 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 488 return -1; 489 } 490 491 /* Execute compression operation */ 492 ret = isal_deflate_stateless(qp->stream); 493 494 /* Check that output buffer did not run out of space */ 495 if (ret == STATELESS_OVERFLOW) { 496 ISAL_PMD_LOG(ERR, "Output buffer not big enough"); 497 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 498 return ret; 499 } 500 501 /* Check that input buffer has been fully consumed */ 502 if (qp->stream->avail_in != (uint32_t)0) { 503 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 504 " entirely"); 505 op->status = RTE_COMP_OP_STATUS_ERROR; 506 return -1; 507 } 508 509 if (ret != COMP_OK) { 510 ISAL_PMD_LOG(ERR, "Compression operation failed"); 511 op->status = RTE_COMP_OP_STATUS_ERROR; 512 return ret; 513 } 514 } 515 516 op->consumed = qp->stream->total_in; 517 if (qp->stream->gzip_flag == IGZIP_DEFLATE) { 518 op->produced = qp->stream->total_out; 519 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) { 520 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER; 521 op->output_chksum = qp->stream->internal_state.crc + 1; 522 } else { 523 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC; 524 op->output_chksum = qp->stream->internal_state.crc; 525 } 526 527 return ret; 528 } 529 530 /* Stateless Decompression Function */ 531 static int 532 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 533 struct isal_priv_xform *priv_xform) 534 { 535 int ret = 0; 536 537 op->status = RTE_COMP_OP_STATUS_SUCCESS; 538 539 /* Initialize decompression state */ 540 isal_inflate_init(qp->state); 541 542 /* Set Checksum flag */ 543 qp->state->crc_flag = priv_xform->decompress.chksum; 544 545 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 546 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough."); 547 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 548 return -1; 549 } 550 551 if (op->dst.offset >= op->m_dst->pkt_len) { 552 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " 553 "offset provided."); 554 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 555 return -1; 556 } 557 558 /* Chained mbufs */ 559 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 560 ret = chained_mbuf_decompression(op, qp); 561 if (ret != 0) 562 return ret; 563 } else { 564 /* Linear buffer */ 565 /* Point decompression state to input buffer */ 566 qp->state->avail_in = op->src.length; 567 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, 568 uint8_t *, op->src.offset); 569 570 /* Point decompression state to output buffer */ 571 qp->state->avail_out = op->m_dst->data_len - op->dst.offset; 572 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 573 uint8_t *, op->dst.offset); 574 575 if (unlikely(!qp->state->next_in || !qp->state->next_out)) { 576 ISAL_PMD_LOG(ERR, "Invalid source or destination" 577 " buffers"); 578 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 579 return -1; 580 } 581 582 /* Execute decompression operation */ 583 ret = isal_inflate_stateless(qp->state); 584 585 if (ret == ISAL_OUT_OVERFLOW) { 586 ISAL_PMD_LOG(ERR, "Output buffer not big enough"); 587 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 588 return ret; 589 } 590 591 /* Check that input buffer has been fully consumed */ 592 if (qp->state->avail_in != (uint32_t)0) { 593 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 594 " entirely"); 595 op->status = RTE_COMP_OP_STATUS_ERROR; 596 return -1; 597 } 598 599 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) { 600 ISAL_PMD_LOG(ERR, "Decompression operation failed"); 601 op->status = RTE_COMP_OP_STATUS_ERROR; 602 return ret; 603 } 604 op->consumed = op->src.length - qp->state->avail_in; 605 } 606 op->produced = qp->state->total_out; 607 op->output_chksum = qp->state->crc; 608 609 return ret; 610 } 611 612 /* Process compression/decompression operation */ 613 static int 614 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, 615 struct isal_priv_xform *priv_xform) 616 { 617 switch (priv_xform->type) { 618 case RTE_COMP_COMPRESS: 619 process_isal_deflate(op, qp, priv_xform); 620 break; 621 case RTE_COMP_DECOMPRESS: 622 process_isal_inflate(op, qp, priv_xform); 623 break; 624 default: 625 ISAL_PMD_LOG(ERR, "Operation Not Supported"); 626 return -ENOTSUP; 627 } 628 return 0; 629 } 630 631 /* Enqueue burst */ 632 static uint16_t 633 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 634 uint16_t nb_ops) 635 { 636 struct isal_comp_qp *qp = queue_pair; 637 uint16_t i; 638 int retval; 639 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); 640 641 for (i = 0; i < num_enq; i++) { 642 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { 643 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 644 ISAL_PMD_LOG(ERR, "Stateful operation not Supported"); 645 qp->qp_stats.enqueue_err_count++; 646 continue; 647 } 648 retval = process_op(qp, ops[i], ops[i]->private_xform); 649 if (unlikely(retval < 0) || 650 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { 651 qp->qp_stats.enqueue_err_count++; 652 } 653 } 654 655 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, 656 num_enq, NULL); 657 qp->num_free_elements -= retval; 658 qp->qp_stats.enqueued_count += retval; 659 660 return retval; 661 } 662 663 /* Dequeue burst */ 664 static uint16_t 665 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 666 uint16_t nb_ops) 667 { 668 struct isal_comp_qp *qp = queue_pair; 669 uint16_t nb_dequeued; 670 671 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, 672 nb_ops, NULL); 673 qp->num_free_elements += nb_dequeued; 674 qp->qp_stats.dequeued_count += nb_dequeued; 675 676 return nb_dequeued; 677 } 678 679 /* Create ISA-L compression device */ 680 static int 681 compdev_isal_create(const char *name, struct rte_vdev_device *vdev, 682 struct rte_compressdev_pmd_init_params *init_params) 683 { 684 struct rte_compressdev *dev; 685 686 dev = rte_compressdev_pmd_create(name, &vdev->device, 687 sizeof(struct isal_comp_private), init_params); 688 if (dev == NULL) { 689 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); 690 return -EFAULT; 691 } 692 693 dev->dev_ops = isal_compress_pmd_ops; 694 695 /* register rx/tx burst functions for data path */ 696 dev->dequeue_burst = isal_comp_pmd_dequeue_burst; 697 dev->enqueue_burst = isal_comp_pmd_enqueue_burst; 698 699 ISAL_PMD_LOG(INFO, "ISA-L library version used: "ISAL_VERSION_STRING); 700 701 return 0; 702 } 703 704 /** Remove compression device */ 705 static int 706 compdev_isal_remove_dev(struct rte_vdev_device *vdev) 707 { 708 struct rte_compressdev *compdev; 709 const char *name; 710 711 name = rte_vdev_device_name(vdev); 712 if (name == NULL) 713 return -EINVAL; 714 715 compdev = rte_compressdev_pmd_get_named_dev(name); 716 if (compdev == NULL) 717 return -ENODEV; 718 719 return rte_compressdev_pmd_destroy(compdev); 720 } 721 722 /** Initialise ISA-L compression device */ 723 static int 724 compdev_isal_probe(struct rte_vdev_device *dev) 725 { 726 struct rte_compressdev_pmd_init_params init_params = { 727 "", 728 rte_socket_id(), 729 }; 730 const char *name, *args; 731 int retval; 732 733 name = rte_vdev_device_name(dev); 734 if (name == NULL) 735 return -EINVAL; 736 737 args = rte_vdev_device_args(dev); 738 739 retval = rte_compressdev_pmd_parse_input_args(&init_params, args); 740 if (retval) { 741 ISAL_PMD_LOG(ERR, 742 "Failed to parse initialisation arguments[%s]", args); 743 return -EINVAL; 744 } 745 746 return compdev_isal_create(name, dev, &init_params); 747 } 748 749 static struct rte_vdev_driver compdev_isal_pmd_drv = { 750 .probe = compdev_isal_probe, 751 .remove = compdev_isal_remove_dev, 752 }; 753 754 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); 755 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, 756 "socket_id=<int>"); 757 RTE_LOG_REGISTER_DEFAULT(isal_logtype_driver, INFO); 758