1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_bus_vdev.h> 7 #include <rte_common.h> 8 #include <rte_malloc.h> 9 #include <rte_mbuf.h> 10 #include <rte_compressdev_pmd.h> 11 12 #include "isal_compress_pmd_private.h" 13 14 #define RTE_COMP_ISAL_WINDOW_SIZE 15 15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */ 16 #define RTE_COMP_ISAL_LEVEL_ONE 1 17 #define RTE_COMP_ISAL_LEVEL_TWO 2 18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ 19 20 int isal_logtype_driver; 21 22 /* Verify and set private xform parameters */ 23 int 24 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, 25 const struct rte_comp_xform *xform) 26 { 27 if (xform == NULL) 28 return -EINVAL; 29 30 /* Set compression private xform variables */ 31 if (xform->type == RTE_COMP_COMPRESS) { 32 /* Set private xform type - COMPRESS/DECOMPRESS */ 33 priv_xform->type = RTE_COMP_COMPRESS; 34 35 /* Set private xform algorithm */ 36 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) { 37 if (xform->compress.algo == RTE_COMP_ALGO_NULL) { 38 ISAL_PMD_LOG(ERR, "By-pass not supported\n"); 39 return -ENOTSUP; 40 } 41 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 42 return -ENOTSUP; 43 } 44 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; 45 46 /* Set private xform checksum - raw deflate by default */ 47 if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { 48 ISAL_PMD_LOG(ERR, "Checksum not supported\n"); 49 return -ENOTSUP; 50 } 51 52 /* Set private xform window size, 32K supported */ 53 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 54 priv_xform->compress.window_size = 55 RTE_COMP_ISAL_WINDOW_SIZE; 56 else { 57 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 58 return -ENOTSUP; 59 } 60 61 /* Set private xform huffman type */ 62 switch (xform->compress.deflate.huffman) { 63 case(RTE_COMP_HUFFMAN_DEFAULT): 64 priv_xform->compress.deflate.huffman = 65 RTE_COMP_HUFFMAN_DEFAULT; 66 break; 67 case(RTE_COMP_HUFFMAN_FIXED): 68 priv_xform->compress.deflate.huffman = 69 RTE_COMP_HUFFMAN_FIXED; 70 break; 71 case(RTE_COMP_HUFFMAN_DYNAMIC): 72 priv_xform->compress.deflate.huffman = 73 RTE_COMP_HUFFMAN_DYNAMIC; 74 break; 75 default: 76 ISAL_PMD_LOG(ERR, "Huffman code not supported\n"); 77 return -ENOTSUP; 78 } 79 80 /* Set private xform level. 81 * Checking compliance with compressdev API, -1 <= level => 9 82 */ 83 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT || 84 xform->compress.level > RTE_COMP_LEVEL_MAX) { 85 ISAL_PMD_LOG(ERR, "Compression level out of range\n"); 86 return -EINVAL; 87 } 88 /* Check for Compressdev API level 0, No compression 89 * not supported in ISA-L 90 */ 91 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) { 92 ISAL_PMD_LOG(ERR, "No Compression not supported\n"); 93 return -ENOTSUP; 94 } 95 /* If using fixed huffman code, level must be 0 */ 96 else if (priv_xform->compress.deflate.huffman == 97 RTE_COMP_HUFFMAN_FIXED) { 98 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a" 99 " fixed huffman code\n"); 100 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO; 101 priv_xform->level_buffer_size = 102 ISAL_DEF_LVL0_DEFAULT; 103 } else { 104 /* Mapping API levels to ISA-L levels 1,2 & 3 */ 105 switch (xform->compress.level) { 106 case RTE_COMP_LEVEL_PMD_DEFAULT: 107 /* Default is 1 if not using fixed huffman */ 108 priv_xform->compress.level = 109 RTE_COMP_ISAL_LEVEL_ONE; 110 priv_xform->level_buffer_size = 111 ISAL_DEF_LVL1_DEFAULT; 112 break; 113 case RTE_COMP_LEVEL_MIN: 114 priv_xform->compress.level = 115 RTE_COMP_ISAL_LEVEL_ONE; 116 priv_xform->level_buffer_size = 117 ISAL_DEF_LVL1_DEFAULT; 118 break; 119 case RTE_COMP_ISAL_LEVEL_TWO: 120 priv_xform->compress.level = 121 RTE_COMP_ISAL_LEVEL_TWO; 122 priv_xform->level_buffer_size = 123 ISAL_DEF_LVL2_DEFAULT; 124 break; 125 /* Level 3 or higher requested */ 126 default: 127 /* Check for AVX512, to use ISA-L level 3 */ 128 if (rte_cpu_get_flag_enabled( 129 RTE_CPUFLAG_AVX512F)) { 130 priv_xform->compress.level = 131 RTE_COMP_ISAL_LEVEL_THREE; 132 priv_xform->level_buffer_size = 133 ISAL_DEF_LVL3_DEFAULT; 134 } 135 /* Check for AVX2, to use ISA-L level 3 */ 136 else if (rte_cpu_get_flag_enabled( 137 RTE_CPUFLAG_AVX2)) { 138 priv_xform->compress.level = 139 RTE_COMP_ISAL_LEVEL_THREE; 140 priv_xform->level_buffer_size = 141 ISAL_DEF_LVL3_DEFAULT; 142 } else { 143 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level" 144 " 3 or above; Level 3 optimized" 145 " for AVX512 & AVX2 only." 146 " level changed to 2.\n"); 147 priv_xform->compress.level = 148 RTE_COMP_ISAL_LEVEL_TWO; 149 priv_xform->level_buffer_size = 150 ISAL_DEF_LVL2_DEFAULT; 151 } 152 } 153 } 154 } 155 156 /* Set decompression private xform variables */ 157 else if (xform->type == RTE_COMP_DECOMPRESS) { 158 159 /* Set private xform type - COMPRESS/DECOMPRESS */ 160 priv_xform->type = RTE_COMP_DECOMPRESS; 161 162 /* Set private xform algorithm */ 163 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) { 164 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) { 165 ISAL_PMD_LOG(ERR, "By pass not supported\n"); 166 return -ENOTSUP; 167 } 168 ISAL_PMD_LOG(ERR, "Algorithm not supported\n"); 169 return -ENOTSUP; 170 } 171 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; 172 173 /* Set private xform checksum - raw deflate by default */ 174 if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { 175 ISAL_PMD_LOG(ERR, "Checksum not supported\n"); 176 return -ENOTSUP; 177 } 178 179 /* Set private xform window size, 32K supported */ 180 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) 181 priv_xform->decompress.window_size = 182 RTE_COMP_ISAL_WINDOW_SIZE; 183 else { 184 ISAL_PMD_LOG(ERR, "Window size not supported\n"); 185 return -ENOTSUP; 186 } 187 } 188 return 0; 189 } 190 191 /* Stateless Compression Function */ 192 static int 193 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, 194 struct isal_priv_xform *priv_xform) 195 { 196 int ret = 0; 197 op->status = RTE_COMP_OP_STATUS_SUCCESS; 198 199 /* Required due to init clearing level_buf */ 200 uint8_t *temp_level_buf = qp->stream->level_buf; 201 202 /* Initialize compression stream */ 203 isal_deflate_stateless_init(qp->stream); 204 205 qp->stream->level_buf = temp_level_buf; 206 207 /* Stateless operation, input will be consumed in one go */ 208 qp->stream->flush = NO_FLUSH; 209 210 /* set op level & intermediate level buffer */ 211 qp->stream->level = priv_xform->compress.level; 212 qp->stream->level_buf_size = priv_xform->level_buffer_size; 213 214 /* Point compression stream structure to input/output buffers */ 215 qp->stream->avail_in = op->src.length; 216 qp->stream->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *); 217 qp->stream->avail_out = op->m_dst->data_len; 218 qp->stream->next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *); 219 qp->stream->end_of_stream = 1; /* All input consumed in one go */ 220 221 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { 222 ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n"); 223 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 224 return -1; 225 } 226 227 /* Set op huffman code */ 228 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) 229 isal_deflate_set_hufftables(qp->stream, NULL, 230 IGZIP_HUFFTABLE_STATIC); 231 else if (priv_xform->compress.deflate.huffman == 232 RTE_COMP_HUFFMAN_DEFAULT) 233 isal_deflate_set_hufftables(qp->stream, NULL, 234 IGZIP_HUFFTABLE_DEFAULT); 235 /* Dynamically change the huffman code to suit the input data */ 236 else if (priv_xform->compress.deflate.huffman == 237 RTE_COMP_HUFFMAN_DYNAMIC) 238 isal_deflate_set_hufftables(qp->stream, NULL, 239 IGZIP_HUFFTABLE_DEFAULT); 240 241 /* Execute compression operation */ 242 ret = isal_deflate_stateless(qp->stream); 243 244 /* Check that output buffer did not run out of space */ 245 if (ret == STATELESS_OVERFLOW) { 246 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 247 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 248 return ret; 249 } 250 251 /* Check that input buffer has been fully consumed */ 252 if (qp->stream->avail_in != (uint32_t)0) { 253 ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n"); 254 op->status = RTE_COMP_OP_STATUS_ERROR; 255 return -1; 256 } 257 258 if (ret != COMP_OK) { 259 op->status = RTE_COMP_OP_STATUS_ERROR; 260 return ret; 261 } 262 263 op->consumed = qp->stream->total_in; 264 op->produced = qp->stream->total_out; 265 266 return ret; 267 } 268 269 /* Stateless Decompression Function */ 270 static int 271 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp) 272 { 273 int ret = 0; 274 275 op->status = RTE_COMP_OP_STATUS_SUCCESS; 276 277 /* Initialize decompression state */ 278 isal_inflate_init(qp->state); 279 280 /* Point decompression state structure to input/output buffers */ 281 qp->state->avail_in = op->src.length; 282 qp->state->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *); 283 qp->state->avail_out = op->m_dst->data_len; 284 qp->state->next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *); 285 286 if (unlikely(!qp->state->next_in || !qp->state->next_out)) { 287 ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n"); 288 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 289 return -1; 290 } 291 292 /* Execute decompression operation */ 293 ret = isal_inflate_stateless(qp->state); 294 295 if (ret == ISAL_OUT_OVERFLOW) { 296 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); 297 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 298 return ret; 299 } 300 301 /* Check that input buffer has been fully consumed */ 302 if (qp->state->avail_in != (uint32_t)0) { 303 ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n"); 304 op->status = RTE_COMP_OP_STATUS_ERROR; 305 return -1; 306 } 307 308 if (ret != ISAL_DECOMP_OK) { 309 op->status = RTE_COMP_OP_STATUS_ERROR; 310 return ret; 311 } 312 313 op->consumed = op->src.length - qp->state->avail_in; 314 op->produced = qp->state->total_out; 315 316 return ret; 317 } 318 319 /* Process compression/decompression operation */ 320 static int 321 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, 322 struct isal_priv_xform *priv_xform) 323 { 324 switch (priv_xform->type) { 325 case RTE_COMP_COMPRESS: 326 process_isal_deflate(op, qp, priv_xform); 327 break; 328 case RTE_COMP_DECOMPRESS: 329 process_isal_inflate(op, qp); 330 break; 331 default: 332 ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); 333 return -ENOTSUP; 334 } 335 return 0; 336 } 337 338 /* Enqueue burst */ 339 static uint16_t 340 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 341 uint16_t nb_ops) 342 { 343 struct isal_comp_qp *qp = queue_pair; 344 uint16_t i; 345 int retval; 346 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops); 347 348 for (i = 0; i < num_enq; i++) { 349 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) { 350 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 351 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n"); 352 qp->qp_stats.enqueue_err_count++; 353 continue; 354 } 355 retval = process_op(qp, ops[i], ops[i]->private_xform); 356 if (unlikely(retval < 0) || 357 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) { 358 qp->qp_stats.enqueue_err_count++; 359 } 360 } 361 362 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops, 363 num_enq, NULL); 364 qp->num_free_elements -= retval; 365 qp->qp_stats.enqueued_count += retval; 366 367 return retval; 368 } 369 370 /* Dequeue burst */ 371 static uint16_t 372 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 373 uint16_t nb_ops) 374 { 375 struct isal_comp_qp *qp = queue_pair; 376 uint16_t nb_dequeued; 377 378 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops, 379 nb_ops, NULL); 380 qp->num_free_elements += nb_dequeued; 381 qp->qp_stats.dequeued_count += nb_dequeued; 382 383 return nb_dequeued; 384 } 385 386 /* Create ISA-L compression device */ 387 static int 388 compdev_isal_create(const char *name, struct rte_vdev_device *vdev, 389 struct rte_compressdev_pmd_init_params *init_params) 390 { 391 struct rte_compressdev *dev; 392 393 dev = rte_compressdev_pmd_create(name, &vdev->device, 394 sizeof(struct isal_comp_private), init_params); 395 if (dev == NULL) { 396 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev"); 397 return -EFAULT; 398 } 399 400 dev->dev_ops = isal_compress_pmd_ops; 401 402 /* register rx/tx burst functions for data path */ 403 dev->dequeue_burst = isal_comp_pmd_dequeue_burst; 404 dev->enqueue_burst = isal_comp_pmd_enqueue_burst; 405 406 return 0; 407 } 408 409 /** Remove compression device */ 410 static int 411 compdev_isal_remove_dev(struct rte_vdev_device *vdev) 412 { 413 struct rte_compressdev *compdev; 414 const char *name; 415 416 name = rte_vdev_device_name(vdev); 417 if (name == NULL) 418 return -EINVAL; 419 420 compdev = rte_compressdev_pmd_get_named_dev(name); 421 if (compdev == NULL) 422 return -ENODEV; 423 424 return rte_compressdev_pmd_destroy(compdev); 425 } 426 427 /** Initialise ISA-L compression device */ 428 static int 429 compdev_isal_probe(struct rte_vdev_device *dev) 430 { 431 struct rte_compressdev_pmd_init_params init_params = { 432 "", 433 rte_socket_id(), 434 }; 435 const char *name, *args; 436 int retval; 437 438 name = rte_vdev_device_name(dev); 439 if (name == NULL) 440 return -EINVAL; 441 442 args = rte_vdev_device_args(dev); 443 444 retval = rte_compressdev_pmd_parse_input_args(&init_params, args); 445 if (retval) { 446 ISAL_PMD_LOG(ERR, 447 "Failed to parse initialisation arguments[%s]\n", args); 448 return -EINVAL; 449 } 450 451 return compdev_isal_create(name, dev, &init_params); 452 } 453 454 static struct rte_vdev_driver compdev_isal_pmd_drv = { 455 .probe = compdev_isal_probe, 456 .remove = compdev_isal_remove_dev, 457 }; 458 459 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); 460 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, 461 "socket_id=<int>"); 462 463 RTE_INIT(isal_init_log); 464 465 static void 466 isal_init_log(void) 467 { 468 isal_logtype_driver = rte_log_register("comp_isal"); 469 if (isal_logtype_driver >= 0) 470 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO); 471 } 472