1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_bus_vdev.h> 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 #include <rte_mbuf.h> 10 #include <rte_compressdev_pmd.h> 11 12 #include "isal_compress_pmd_private.h" 13 14 #define RTE_COMP_ISAL_WINDOW_SIZE 15 15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ 16 #define RTE_COMP_ISAL_LEVEL_ONE 1 17 #define RTE_COMP_ISAL_LEVEL_TWO 2 18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ 19 #define CHKSUM_SZ_CRC 8 20 #define CHKSUM_SZ_ADLER 4 21 22 int isal_logtype_driver; 23 24 /* Verify and set private xform parameters */ 25 int 26 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, 27 const struct rte_comp_xform *xform) 28 { 29 if (xform == NULL) 30 return -EINVAL; 31 32 /* Set compression private xform variables */ 33 if (xform->type == RTE_COMP_COMPRESS) { 34 /* Set private xform type - COMPRESS/DECOMPRESS */ 35 priv_xform->type = RTE_COMP_COMPRESS; 36 37 /* Set private xform algorithm */ 38 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { 39 if (xform->compress.algo == RTE_COMP_ALGO_NULL) { 40 ISAL_PMD_LOG(ERR, "By-pass not supported\n"); 41 return -ENOTSUP; 42 } 43 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 44 return -ENOTSUP; 45 } 46 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; 47 48 /* Set private xform window size, 32K supported */ 49 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 50 priv_xform->compress.window_size = 51 RTE_COMP_ISAL_WINDOW_SIZE; 52 else { 53 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 54 return -ENOTSUP; 55 } 56 57 /* Set private xform huffman type */ 58 switch (xform->compress.deflate.huffman) { 59 case(RTE_COMP_HUFFMAN_DEFAULT): 60 priv_xform->compress.deflate.huffman = 61 RTE_COMP_HUFFMAN_DEFAULT; 62 break; 63 case(RTE_COMP_HUFFMAN_FIXED): 64 priv_xform->compress.deflate.huffman = 65 RTE_COMP_HUFFMAN_FIXED; 66 break; 67 case(RTE_COMP_HUFFMAN_DYNAMIC): 68 priv_xform->compress.deflate.huffman = 69 RTE_COMP_HUFFMAN_DYNAMIC; 70 break; 71 default: 72 ISAL_PMD_LOG(ERR, "Huffman code not supported\n"); 73 return -ENOTSUP; 74 } 75 76 /* Set private xform checksum */ 77 switch (xform->compress.chksum) { 78 /* Raw deflate by default */ 79 case(RTE_COMP_CHECKSUM_NONE): 80 priv_xform->compress.chksum = IGZIP_DEFLATE; 81 break; 82 case(RTE_COMP_CHECKSUM_CRC32): 83 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR; 84 break; 85 case(RTE_COMP_CHECKSUM_ADLER32): 86 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR; 87 break; 88 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 89 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 90 " supported\n"); 91 return -ENOTSUP; 92 default: 93 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 94 priv_xform->compress.chksum = IGZIP_DEFLATE; 95 break; 96 } 97 98 /* Set private xform level. 99 * Checking compliance with compressdev API, -1 <= level => 9 100 */ 101 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || 102 xform->compress.level > RTE_COMP_LEVEL_MAX) { 103 ISAL_PMD_LOG(ERR, "Compression level out of range\n"); 104 return -EINVAL; 105 } 106 /* Check for Compressdev API level 0, No compression 107 * not supported in ISA-L 108 */ 109 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { 110 ISAL_PMD_LOG(ERR, "No Compression not supported\n"); 111 return -ENOTSUP; 112 } 113 /* If using fixed huffman code, level must be 0 */ 114 else if (priv_xform->compress.deflate.huffman == 115 RTE_COMP_HUFFMAN_FIXED) { 116 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" 117 " fixed huffman code\n"); 118 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; 119 priv_xform->level_buffer_size = 120 ISAL_DEF_LVL0_DEFAULT; 121 } else { 122 /* Mapping API levels to ISA-L levels 1,2 & 3 */ 123 switch (xform->compress.level) { 124 case RTE_COMP_LEVEL_PMD_DEFAULT: 125 /* Default is 1 if not using fixed huffman */ 126 priv_xform->compress.level = 127 RTE_COMP_ISAL_LEVEL_ONE; 128 priv_xform->level_buffer_size = 129 ISAL_DEF_LVL1_DEFAULT; 130 break; 131 case RTE_COMP_LEVEL_MIN: 132 priv_xform->compress.level = 133 RTE_COMP_ISAL_LEVEL_ONE; 134 priv_xform->level_buffer_size = 135 ISAL_DEF_LVL1_DEFAULT; 136 break; 137 case RTE_COMP_ISAL_LEVEL_TWO: 138 priv_xform->compress.level = 139 RTE_COMP_ISAL_LEVEL_TWO; 140 priv_xform->level_buffer_size = 141 ISAL_DEF_LVL2_DEFAULT; 142 break; 143 /* Level 3 or higher requested */ 144 default: 145 /* Check for AVX512, to use ISA-L level 3 */ 146 if (rte_cpu_get_flag_enabled( 147 RTE_CPUFLAG_AVX512F)) { 148 priv_xform->compress.level = 149 RTE_COMP_ISAL_LEVEL_THREE; 150 priv_xform->level_buffer_size = 151 ISAL_DEF_LVL3_DEFAULT; 152 } 153 /* Check for AVX2, to use ISA-L level 3 */ 154 else if (rte_cpu_get_flag_enabled( 155 RTE_CPUFLAG_AVX2)) { 156 priv_xform->compress.level = 157 RTE_COMP_ISAL_LEVEL_THREE; 158 priv_xform->level_buffer_size = 159 ISAL_DEF_LVL3_DEFAULT; 160 } else { 161 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" 162 " 3 or above; Level 3 optimized" 163 " for AVX512 & AVX2 only." 164 " level changed to 2.\n"); 165 priv_xform->compress.level = 166 RTE_COMP_ISAL_LEVEL_TWO; 167 priv_xform->level_buffer_size = 168 ISAL_DEF_LVL2_DEFAULT; 169 } 170 } 171 } 172 } 173 174 /* Set decompression private xform variables */ 175 else if (xform->type == RTE_COMP_DECOMPRESS) { 176 177 /* Set private xform type - COMPRESS/DECOMPRESS */ 178 priv_xform->type = RTE_COMP_DECOMPRESS; 179 180 /* Set private xform algorithm */ 181 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { 182 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { 183 ISAL_PMD_LOG(ERR, "By pass not supported\n"); 184 return -ENOTSUP; 185 } 186 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 187 return -ENOTSUP; 188 } 189 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; 190 191 /* Set private xform checksum */ 192 switch (xform->decompress.chksum) { 193 /* Raw deflate by default */ 194 case(RTE_COMP_CHECKSUM_NONE): 195 priv_xform->decompress.chksum = ISAL_DEFLATE; 196 break; 197 case(RTE_COMP_CHECKSUM_CRC32): 198 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR; 199 break; 200 case(RTE_COMP_CHECKSUM_ADLER32): 201 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR; 202 break; 203 case(RTE_COMP_CHECKSUM_CRC32_ADLER32): 204 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" 205 " supported\n"); 206 return -ENOTSUP; 207 default: 208 ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); 209 priv_xform->decompress.chksum = ISAL_DEFLATE; 210 break; 211 } 212 213 /* Set private xform window size, 32K supported */ 214 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 215 priv_xform->decompress.window_size = 216 RTE_COMP_ISAL_WINDOW_SIZE; 217 else { 218 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 219 return -ENOTSUP; 220 } 221 } 222 return 0; 223 } 224 225 /* Compression using chained mbufs for input/output data */ 226 static int 227 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) 228 { 229 int ret; 230 uint32_t remaining_offset; 231 uint32_t remaining_data = op->src.length; 232 struct rte_mbuf *src = op->m_src; 233 struct rte_mbuf *dst = op->m_dst; 234 235 /* check for source/destination offset passing multiple segments 236 * and point compression stream to input/output buffer. 237 */ 238 remaining_offset = op->src.offset; 239 while (remaining_offset >= src->data_len) { 240 remaining_offset -= src->data_len; 241 src = src->next; 242 } 243 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, 244 op->src.length); 245 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 246 remaining_offset); 247 248 remaining_offset = op->dst.offset; 249 while (remaining_offset >= dst->data_len) { 250 remaining_offset -= dst->data_len; 251 dst = dst->next; 252 } 253 qp->stream->avail_out = dst->data_len - remaining_offset; 254 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 255 remaining_offset); 256 257 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 258 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n"); 259 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 260 return -1; 261 } 262 263 while (qp->stream->internal_state.state != ZSTATE_END) { 264 /* Last segment of data */ 265 if (remaining_data <= src->data_len) 266 qp->stream->end_of_stream = 1; 267 268 /* Execute compression operation */ 269 ret = isal_deflate(qp->stream); 270 271 remaining_data = op->src.length - qp->stream->total_in; 272 273 if (ret != COMP_OK) { 274 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 275 op->status = RTE_COMP_OP_STATUS_ERROR; 276 return ret; 277 } 278 279 if (qp->stream->avail_in == 0 && 280 qp->stream->total_in != op->src.length) { 281 if (src->next != NULL) { 282 src = src->next; 283 qp->stream->next_in = 284 rte_pktmbuf_mtod(src, uint8_t *); 285 qp->stream->avail_in = 286 RTE_MIN(remaining_data, src->data_len); 287 } else { 288 ISAL_PMD_LOG(ERR, 289 "Not enough input buffer segments\n"); 290 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 291 return -1; 292 } 293 } 294 295 if (qp->stream->avail_out == 0 && 296 qp->stream->internal_state.state != ZSTATE_END) { 297 if (dst->next != NULL) { 298 dst = dst->next; 299 qp->stream->next_out = 300 rte_pktmbuf_mtod(dst, uint8_t *); 301 qp->stream->avail_out = dst->data_len; 302 } else { 303 ISAL_PMD_LOG(ERR, 304 "Not enough output buffer segments\n"); 305 op->status = 306 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 307 return -1; 308 } 309 } 310 } 311 312 return 0; 313 } 314 315 /* Decompression using chained mbufs for input/output data */ 316 static int 317 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) 318 { 319 int ret; 320 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; 321 uint32_t remaining_data = op->src.length; 322 struct rte_mbuf *src = op->m_src; 323 struct rte_mbuf *dst = op->m_dst; 324 325 /* check for offset passing multiple segments 326 * and point decompression state to input/output buffer 327 */ 328 src_remaining_offset = op->src.offset; 329 while (src_remaining_offset >= src->data_len) { 330 src_remaining_offset -= src->data_len; 331 src = src->next; 332 } 333 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, 334 op->src.length); 335 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, 336 src_remaining_offset); 337 338 dst_remaining_offset = op->dst.offset; 339 while (dst_remaining_offset >= dst->data_len) { 340 dst_remaining_offset -= dst->data_len; 341 dst = dst->next; 342 } 343 qp->state->avail_out = dst->data_len - dst_remaining_offset; 344 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, 345 dst_remaining_offset); 346 347 while (qp->state->block_state != ISAL_BLOCK_FINISH) { 348 349 ret = isal_inflate(qp->state); 350 351 if (ret != ISAL_DECOMP_OK) { 352 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 353 op->status = RTE_COMP_OP_STATUS_ERROR; 354 return ret; 355 } 356 357 /* Check for first segment, offset needs to be accounted for */ 358 if (remaining_data == op->src.length) { 359 consumed_data = src->data_len - src_remaining_offset; 360 } else 361 consumed_data = src->data_len; 362 363 if (qp->state->avail_in == 0 364 && op->consumed != op->src.length) { 365 op->consumed += consumed_data; 366 remaining_data -= consumed_data; 367 368 if (src->next != NULL) { 369 src = src->next; 370 qp->state->next_in = 371 rte_pktmbuf_mtod(src, uint8_t *); 372 qp->state->avail_in = 373 RTE_MIN(remaining_data, src->data_len); 374 } 375 } 376 377 if (qp->state->avail_out == 0 && 378 qp->state->block_state != ISAL_BLOCK_FINISH) { 379 if (dst->next != NULL) { 380 dst = dst->next; 381 qp->state->next_out = 382 rte_pktmbuf_mtod(dst, uint8_t *); 383 qp->state->avail_out = dst->data_len; 384 } else { 385 ISAL_PMD_LOG(ERR, 386 "Not enough output buffer segments\n"); 387 op->status = 388 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 389 return -1; 390 } 391 } 392 } 393 394 return 0; 395 } 396 397 /* Stateless Compression Function */ 398 static int 399 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 400 struct isal_priv_xform *priv_xform) 401 { 402 int ret = 0; 403 op->status = RTE_COMP_OP_STATUS_SUCCESS; 404 405 /* Required due to init clearing level_buf */ 406 uint8_t *temp_level_buf = qp->stream->level_buf; 407 408 /* Initialize compression stream */ 409 isal_deflate_stateless_init(qp->stream); 410 411 qp->stream->level_buf = temp_level_buf; 412 413 /* Set Checksum flag */ 414 qp->stream->gzip_flag = priv_xform->compress.chksum; 415 416 /* Stateless operation, input will be consumed in one go */ 417 qp->stream->flush = NO_FLUSH; 418 419 /* set compression level & intermediate level buffer size */ 420 qp->stream->level = priv_xform->compress.level; 421 qp->stream->level_buf_size = priv_xform->level_buffer_size; 422 423 /* Set op huffman code */ 424 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) 425 isal_deflate_set_hufftables(qp->stream, NULL, 426 IGZIP_HUFFTABLE_STATIC); 427 else if (priv_xform->compress.deflate.huffman == 428 RTE_COMP_HUFFMAN_DEFAULT) 429 isal_deflate_set_hufftables(qp->stream, NULL, 430 IGZIP_HUFFTABLE_DEFAULT); 431 /* Dynamically change the huffman code to suit the input data */ 432 else if (priv_xform->compress.deflate.huffman == 433 RTE_COMP_HUFFMAN_DYNAMIC) 434 isal_deflate_set_hufftables(qp->stream, NULL, 435 IGZIP_HUFFTABLE_DEFAULT); 436 437 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 438 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 439 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 440 return -1; 441 } 442 443 if (op->dst.offset >= op->m_dst->pkt_len) { 444 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" 445 " for offset provided.\n"); 446 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 447 return -1; 448 } 449 450 /* Chained mbufs */ 451 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 452 ret = chained_mbuf_compression(op, qp); 453 if (ret < 0) 454 return ret; 455 } else { 456 /* Linear buffer */ 457 qp->stream->end_of_stream = 1; /* All input consumed in one */ 458 /* Point compression stream to input buffer */ 459 qp->stream->avail_in = op->src.length; 460 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, 461 uint8_t *, op->src.offset); 462 463 /* Point compression stream to output buffer */ 464 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; 465 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 466 uint8_t *, op->dst.offset); 467 468 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 469 ISAL_PMD_LOG(ERR, "Invalid source or destination" 470 " buffers\n"); 471 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 472 return -1; 473 } 474 475 /* Execute compression operation */ 476 ret = isal_deflate_stateless(qp->stream); 477 478 /* Check that output buffer did not run out of space */ 479 if (ret == STATELESS_OVERFLOW) { 480 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 481 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 482 return ret; 483 } 484 485 /* Check that input buffer has been fully consumed */ 486 if (qp->stream->avail_in != (uint32_t)0) { 487 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 488 " entirely\n"); 489 op->status = RTE_COMP_OP_STATUS_ERROR; 490 return -1; 491 } 492 493 if (ret != COMP_OK) { 494 ISAL_PMD_LOG(ERR, "Compression operation failed\n"); 495 op->status = RTE_COMP_OP_STATUS_ERROR; 496 return ret; 497 } 498 } 499 500 op->consumed = qp->stream->total_in; 501 if (qp->stream->gzip_flag == IGZIP_DEFLATE) { 502 op->produced = qp->stream->total_out; 503 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) { 504 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER; 505 op->output_chksum = qp->stream->internal_state.crc + 1; 506 } else { 507 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC; 508 op->output_chksum = qp->stream->internal_state.crc; 509 } 510 511 isal_deflate_reset(qp->stream); 512 513 return ret; 514 } 515 516 /* Stateless Decompression Function */ 517 static int 518 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 519 struct isal_priv_xform *priv_xform) 520 { 521 int ret = 0; 522 523 op->status = RTE_COMP_OP_STATUS_SUCCESS; 524 525 /* Initialize decompression state */ 526 isal_inflate_init(qp->state); 527 528 /* Set Checksum flag */ 529 qp->state->crc_flag = priv_xform->decompress.chksum; 530 531 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { 532 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); 533 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 534 return -1; 535 } 536 537 if (op->dst.offset >= op->m_dst->pkt_len) { 538 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " 539 "offset provided.\n"); 540 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 541 return -1; 542 } 543 544 /* Chained mbufs */ 545 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { 546 ret = chained_mbuf_decompression(op, qp); 547 if (ret != 0) 548 return ret; 549 } else { 550 /* Linear buffer */ 551 /* Point decompression state to input buffer */ 552 qp->state->avail_in = op->src.length; 553 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, 554 uint8_t *, op->src.offset); 555 556 /* Point decompression state to output buffer */ 557 qp->state->avail_out = op->m_dst->data_len - op->dst.offset; 558 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, 559 uint8_t *, op->dst.offset); 560 561 if (unlikely(!qp->state->next_in || !qp->state->next_out)) { 562 ISAL_PMD_LOG(ERR, "Invalid source or destination" 563 " buffers\n"); 564 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 565 return -1; 566 } 567 568 /* Execute decompression operation */ 569 ret = isal_inflate_stateless(qp->state); 570 571 if (ret == ISAL_OUT_OVERFLOW) { 572 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 573 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 574 return ret; 575 } 576 577 /* Check that input buffer has been fully consumed */ 578 if (qp->state->avail_in != (uint32_t)0) { 579 ISAL_PMD_LOG(ERR, "Input buffer could not be read" 580 " entirely\n"); 581 op->status = RTE_COMP_OP_STATUS_ERROR; 582 return -1; 583 } 584 585 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) { 586 ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); 587 op->status = RTE_COMP_OP_STATUS_ERROR; 588 return ret; 589 } 590 op->consumed = op->src.length - qp->state->avail_in; 591 } 592 op->produced = qp->state->total_out; 593 op->output_chksum = qp->state->crc; 594 595 isal_inflate_reset(qp->state); 596 597 return ret; 598 } 599 600 /* Process compression/decompression operation */ 601 static int 602 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, 603 struct isal_priv_xform *priv_xform) 604 { 605 switch (priv_xform->type) { 606 case RTE_COMP_COMPRESS: 607 process_isal_deflate(op, qp, priv_xform); 608 break; 609 case RTE_COMP_DECOMPRESS: 610 process_isal_inflate(op, qp, priv_xform); 611 break; 612 default: 613 ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); 614 return -ENOTSUP; 615 } 616 return 0; 617 } 618 619 /* Enqueue burst */ 620 static uint16_t 621 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 622 uint16_t nb_ops) 623 { 624 struct isal_comp_qp *qp = queue_pair; 625 uint16_t i; 626 int retval; 627 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); 628 629 for (i = 0; i < num_enq; i++) { 630 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { 631 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 632 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n"); 633 qp->qp_stats.enqueue_err_count++; 634 continue; 635 } 636 retval = process_op(qp, ops[i], ops[i]->private_xform); 637 if (unlikely(retval < 0) || 638 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { 639 qp->qp_stats.enqueue_err_count++; 640 } 641 } 642 643 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, 644 num_enq, NULL); 645 qp->num_free_elements -= retval; 646 qp->qp_stats.enqueued_count += retval; 647 648 return retval; 649 } 650 651 /* Dequeue burst */ 652 static uint16_t 653 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 654 uint16_t nb_ops) 655 { 656 struct isal_comp_qp *qp = queue_pair; 657 uint16_t nb_dequeued; 658 659 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, 660 nb_ops, NULL); 661 qp->num_free_elements += nb_dequeued; 662 qp->qp_stats.dequeued_count += nb_dequeued; 663 664 return nb_dequeued; 665 } 666 667 /* Create ISA-L compression device */ 668 static int 669 compdev_isal_create(const char *name, struct rte_vdev_device *vdev, 670 struct rte_compressdev_pmd_init_params *init_params) 671 { 672 struct rte_compressdev *dev; 673 674 dev = rte_compressdev_pmd_create(name, &vdev->device, 675 sizeof(struct isal_comp_private), init_params); 676 if (dev == NULL) { 677 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); 678 return -EFAULT; 679 } 680 681 dev->dev_ops = isal_compress_pmd_ops; 682 683 /* register rx/tx burst functions for data path */ 684 dev->dequeue_burst = isal_comp_pmd_dequeue_burst; 685 dev->enqueue_burst = isal_comp_pmd_enqueue_burst; 686 687 return 0; 688 } 689 690 /** Remove compression device */ 691 static int 692 compdev_isal_remove_dev(struct rte_vdev_device *vdev) 693 { 694 struct rte_compressdev *compdev; 695 const char *name; 696 697 name = rte_vdev_device_name(vdev); 698 if (name == NULL) 699 return -EINVAL; 700 701 compdev = rte_compressdev_pmd_get_named_dev(name); 702 if (compdev == NULL) 703 return -ENODEV; 704 705 return rte_compressdev_pmd_destroy(compdev); 706 } 707 708 /** Initialise ISA-L compression device */ 709 static int 710 compdev_isal_probe(struct rte_vdev_device *dev) 711 { 712 struct rte_compressdev_pmd_init_params init_params = { 713 "", 714 rte_socket_id(), 715 }; 716 const char *name, *args; 717 int retval; 718 719 name = rte_vdev_device_name(dev); 720 if (name == NULL) 721 return -EINVAL; 722 723 args = rte_vdev_device_args(dev); 724 725 retval = rte_compressdev_pmd_parse_input_args(&init_params, args); 726 if (retval) { 727 ISAL_PMD_LOG(ERR, 728 "Failed to parse initialisation arguments[%s]\n", args); 729 return -EINVAL; 730 } 731 732 return compdev_isal_create(name, dev, &init_params); 733 } 734 735 static struct rte_vdev_driver compdev_isal_pmd_drv = { 736 .probe = compdev_isal_probe, 737 .remove = compdev_isal_remove_dev, 738 }; 739 740 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); 741 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, 742 "socket_id=<int>"); 743 744 RTE_INIT(isal_init_log) 745 { 746 isal_logtype_driver = rte_log_register("pmd.compress.isal"); 747 if (isal_logtype_driver >= 0) 748 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO); 749 } 750