1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #include <string.h> 6 7 #include <rte_byteorder.h> 8 #include <rte_common.h> 9 #include <rte_cpuflags.h> 10 #include <rte_malloc.h> 11 12 #include "otx_zip.h" 13 14 static const struct rte_compressdev_capabilities 15 octtx_zip_pmd_capabilities[] = { 16 { .algo = RTE_COMP_ALGO_DEFLATE, 17 /* Deflate */ 18 .comp_feature_flags = RTE_COMP_FF_HUFFMAN_FIXED | 19 RTE_COMP_FF_HUFFMAN_DYNAMIC, 20 /* Non sharable Priv XFORM and Stateless */ 21 .window_size = { 22 .min = 1, 23 .max = 14, 24 .increment = 1 25 /* size supported 2^1 to 2^14 */ 26 }, 27 }, 28 RTE_COMP_END_OF_CAPABILITIES_LIST() 29 }; 30 31 /* 32 * Reset session to default state for next set of stateless operation 33 */ 34 static inline void 35 reset_stream(struct zip_stream *z_stream) 36 { 37 union zip_inst_s *inst = (union zip_inst_s *)(z_stream->inst); 38 39 inst->s.bf = 1; 40 inst->s.ef = 0; 41 } 42 43 int 44 zip_process_op(struct rte_comp_op *op, 45 struct zipvf_qp *qp, 46 struct zip_stream *zstrm) 47 { 48 union zip_inst_s *inst = zstrm->inst; 49 volatile union zip_zres_s *zresult = NULL; 50 51 52 if ((op->m_src->nb_segs > 1) || (op->m_dst->nb_segs > 1) || 53 (op->src.offset > rte_pktmbuf_pkt_len(op->m_src)) || 54 (op->dst.offset > rte_pktmbuf_pkt_len(op->m_dst))) { 55 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 56 ZIP_PMD_ERR("Segmented packet is not supported\n"); 57 return 0; 58 } 59 60 zipvf_prepare_cmd_stateless(op, zstrm); 61 62 zresult = (union zip_zres_s *)zstrm->bufs[RES_BUF]; 63 zresult->s.compcode = 0; 64 65 #ifdef ZIP_DBG 66 zip_dump_instruction(inst); 67 #endif 68 69 /* Submit zip command */ 70 zipvf_push_command(qp, (void *)inst); 71 72 /* Check and Process results in sync mode */ 73 do { 74 } while (!zresult->s.compcode); 75 76 if (zresult->s.compcode == ZIP_COMP_E_SUCCESS) { 77 op->status = RTE_COMP_OP_STATUS_SUCCESS; 78 } else { 79 /* FATAL error cannot do anything */ 80 ZIP_PMD_ERR("operation failed with error code:%d\n", 81 zresult->s.compcode); 82 if (zresult->s.compcode == ZIP_COMP_E_DSTOP) 83 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 84 else 85 op->status = RTE_COMP_OP_STATUS_ERROR; 86 } 87 88 ZIP_PMD_INFO("written %d\n", zresult->s.totalbyteswritten); 89 90 /* Update op stats */ 91 switch (op->status) { 92 case RTE_COMP_OP_STATUS_SUCCESS: 93 op->consumed = zresult->s.totalbytesread; 94 /* Fall-through */ 95 case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED: 96 op->produced = zresult->s.totalbyteswritten; 97 break; 98 default: 99 ZIP_PMD_ERR("stats not updated for status:%d\n", 100 op->status); 101 break; 102 } 103 /* zstream is reset irrespective of result */ 104 reset_stream(zstrm); 105 106 zresult->s.compcode = ZIP_COMP_E_NOTDONE; 107 return 0; 108 } 109 110 /** Parse xform parameters and setup a stream */ 111 static int 112 zip_set_stream_parameters(struct rte_compressdev *dev, 113 const struct rte_comp_xform *xform, 114 struct zip_stream *z_stream) 115 { 116 int ret; 117 union zip_inst_s *inst; 118 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 119 void *res; 120 121 /* Allocate resources required by a stream */ 122 ret = rte_mempool_get_bulk(vf->zip_mp, 123 z_stream->bufs, MAX_BUFS_PER_STREAM); 124 if (ret < 0) 125 return -1; 126 127 /* get one command buffer from pool and set up */ 128 inst = (union zip_inst_s *)z_stream->bufs[CMD_BUF]; 129 res = z_stream->bufs[RES_BUF]; 130 131 memset(inst->u, 0, sizeof(inst->u)); 132 133 /* set bf for only first ops of stream */ 134 inst->s.bf = 1; 135 136 if (xform->type == RTE_COMP_COMPRESS) { 137 inst->s.op = ZIP_OP_E_COMP; 138 139 switch (xform->compress.deflate.huffman) { 140 case RTE_COMP_HUFFMAN_DEFAULT: 141 inst->s.cc = ZIP_CC_DEFAULT; 142 break; 143 case RTE_COMP_HUFFMAN_FIXED: 144 inst->s.cc = ZIP_CC_FIXED_HUFF; 145 break; 146 case RTE_COMP_HUFFMAN_DYNAMIC: 147 inst->s.cc = ZIP_CC_DYN_HUFF; 148 break; 149 default: 150 ret = -1; 151 goto err; 152 } 153 154 switch (xform->compress.level) { 155 case RTE_COMP_LEVEL_MIN: 156 inst->s.ss = ZIP_COMP_E_LEVEL_MIN; 157 break; 158 case RTE_COMP_LEVEL_MAX: 159 inst->s.ss = ZIP_COMP_E_LEVEL_MAX; 160 break; 161 case RTE_COMP_LEVEL_NONE: 162 ZIP_PMD_ERR("Compression level not supported"); 163 ret = -1; 164 goto err; 165 default: 166 /* for any value between min and max , choose 167 * PMD default. 168 */ 169 inst->s.ss = ZIP_COMP_E_LEVEL_MED; /** PMD default **/ 170 break; 171 } 172 } else if (xform->type == RTE_COMP_DECOMPRESS) { 173 inst->s.op = ZIP_OP_E_DECOMP; 174 /* from HRM, 175 * For DEFLATE decompression, [CC] must be 0x0. 176 * For decompression, [SS] must be 0x0 177 */ 178 inst->s.cc = 0; 179 /* Speed bit should not be set for decompression */ 180 inst->s.ss = 0; 181 /* decompression context is supported only for STATEFUL 182 * operations. Currently we support STATELESS ONLY so 183 * skip setting of ctx pointer 184 */ 185 186 } else { 187 ZIP_PMD_ERR("\nxform type not supported"); 188 ret = -1; 189 goto err; 190 } 191 192 inst->s.res_ptr_addr.s.addr = rte_mempool_virt2iova(res); 193 inst->s.res_ptr_ctl.s.length = 0; 194 195 z_stream->inst = inst; 196 z_stream->func = zip_process_op; 197 198 return 0; 199 200 err: 201 rte_mempool_put_bulk(vf->zip_mp, 202 (void *)&(z_stream->bufs[0]), 203 MAX_BUFS_PER_STREAM); 204 205 return ret; 206 } 207 208 /** Configure device */ 209 static int 210 zip_pmd_config(struct rte_compressdev *dev, 211 struct rte_compressdev_config *config) 212 { 213 int nb_streams; 214 char res_pool[RTE_MEMZONE_NAMESIZE]; 215 struct zip_vf *vf; 216 struct rte_mempool *zip_buf_mp; 217 218 if (!config || !dev) 219 return -EIO; 220 221 vf = (struct zip_vf *)(dev->data->dev_private); 222 223 /* create pool with maximum numbers of resources 224 * required by streams 225 */ 226 227 /* use common pool for non-shareable priv_xform and stream */ 228 nb_streams = config->max_nb_priv_xforms + config->max_nb_streams; 229 230 snprintf(res_pool, RTE_MEMZONE_NAMESIZE, "octtx_zip_res_pool%u", 231 dev->data->dev_id); 232 233 /** TBD Should we use the per core object cache for stream resources */ 234 zip_buf_mp = rte_mempool_create( 235 res_pool, 236 nb_streams * MAX_BUFS_PER_STREAM, 237 ZIP_BUF_SIZE, 238 0, 239 0, 240 NULL, 241 NULL, 242 NULL, 243 NULL, 244 SOCKET_ID_ANY, 245 0); 246 247 if (zip_buf_mp == NULL) { 248 ZIP_PMD_ERR( 249 "Failed to create buf mempool octtx_zip_res_pool%u", 250 dev->data->dev_id); 251 return -1; 252 } 253 254 vf->zip_mp = zip_buf_mp; 255 256 return 0; 257 } 258 259 /** Start device */ 260 static int 261 zip_pmd_start(__rte_unused struct rte_compressdev *dev) 262 { 263 return 0; 264 } 265 266 /** Stop device */ 267 static void 268 zip_pmd_stop(__rte_unused struct rte_compressdev *dev) 269 { 270 271 } 272 273 /** Close device */ 274 static int 275 zip_pmd_close(struct rte_compressdev *dev) 276 { 277 if (dev == NULL) 278 return -1; 279 280 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 281 rte_mempool_free(vf->zip_mp); 282 283 return 0; 284 } 285 286 /** Get device statistics */ 287 static void 288 zip_pmd_stats_get(struct rte_compressdev *dev, 289 struct rte_compressdev_stats *stats) 290 { 291 int qp_id; 292 293 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 294 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 295 296 stats->enqueued_count += qp->qp_stats.enqueued_count; 297 stats->dequeued_count += qp->qp_stats.dequeued_count; 298 299 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 300 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 301 } 302 } 303 304 /** Reset device statistics */ 305 static void 306 zip_pmd_stats_reset(struct rte_compressdev *dev) 307 { 308 int qp_id; 309 310 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 311 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 312 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 313 } 314 } 315 316 /** Get device info */ 317 static void 318 zip_pmd_info_get(struct rte_compressdev *dev, 319 struct rte_compressdev_info *dev_info) 320 { 321 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 322 323 if (dev_info != NULL) { 324 dev_info->driver_name = dev->device->driver->name; 325 dev_info->feature_flags = dev->feature_flags; 326 dev_info->capabilities = octtx_zip_pmd_capabilities; 327 dev_info->max_nb_queue_pairs = vf->max_nb_queue_pairs; 328 } 329 } 330 331 /** Release queue pair */ 332 static int 333 zip_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 334 { 335 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 336 337 if (qp != NULL) { 338 zipvf_q_term(qp); 339 340 rte_ring_free(qp->processed_pkts); 341 342 rte_free(qp); 343 dev->data->queue_pairs[qp_id] = NULL; 344 } 345 return 0; 346 } 347 348 /** Create a ring to place process packets on */ 349 static struct rte_ring * 350 zip_pmd_qp_create_processed_pkts_ring(struct zipvf_qp *qp, 351 unsigned int ring_size, int socket_id) 352 { 353 struct rte_ring *r; 354 355 r = rte_ring_lookup(qp->name); 356 if (r) { 357 if (rte_ring_get_size(r) >= ring_size) { 358 ZIP_PMD_INFO("Reusing existing ring %s for processed" 359 " packets", qp->name); 360 return r; 361 } 362 363 ZIP_PMD_ERR("Unable to reuse existing ring %s for processed" 364 " packets", qp->name); 365 return NULL; 366 } 367 368 return rte_ring_create(qp->name, ring_size, socket_id, 369 RING_F_EXACT_SZ); 370 } 371 372 /** Setup a queue pair */ 373 static int 374 zip_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 375 uint32_t max_inflight_ops, int socket_id) 376 { 377 struct zipvf_qp *qp = NULL; 378 struct zip_vf *vf; 379 char *name; 380 int ret; 381 382 if (!dev) 383 return -1; 384 385 vf = (struct zip_vf *) (dev->data->dev_private); 386 387 /* Free memory prior to re-allocation if needed. */ 388 if (dev->data->queue_pairs[qp_id] != NULL) { 389 ZIP_PMD_INFO("Using existing queue pair %d ", qp_id); 390 return 0; 391 } 392 393 name = rte_malloc(NULL, RTE_COMPRESSDEV_NAME_MAX_LEN, 0); 394 snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, 395 "zip_pmd_%u_qp_%u", 396 dev->data->dev_id, qp_id); 397 398 /* Allocate the queue pair data structure. */ 399 qp = rte_zmalloc_socket(name, sizeof(*qp), 400 RTE_CACHE_LINE_SIZE, socket_id); 401 if (qp == NULL) 402 return (-ENOMEM); 403 404 qp->name = name; 405 406 /* Create completion queue up to max_inflight_ops */ 407 qp->processed_pkts = zip_pmd_qp_create_processed_pkts_ring(qp, 408 max_inflight_ops, socket_id); 409 if (qp->processed_pkts == NULL) 410 goto qp_setup_cleanup; 411 412 qp->id = qp_id; 413 qp->vf = vf; 414 415 ret = zipvf_q_init(qp); 416 if (ret < 0) 417 goto qp_setup_cleanup; 418 419 dev->data->queue_pairs[qp_id] = qp; 420 421 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 422 return 0; 423 424 qp_setup_cleanup: 425 rte_ring_free(qp->processed_pkts); 426 rte_free(qp); 427 return -1; 428 } 429 430 static int 431 zip_pmd_stream_create(struct rte_compressdev *dev, 432 const struct rte_comp_xform *xform, void **stream) 433 { 434 int ret; 435 struct zip_stream *strm = NULL; 436 437 strm = rte_malloc(NULL, 438 sizeof(struct zip_stream), 0); 439 440 if (strm == NULL) 441 return (-ENOMEM); 442 443 ret = zip_set_stream_parameters(dev, xform, strm); 444 if (ret < 0) { 445 ZIP_PMD_ERR("failed configure xform parameters"); 446 rte_free(strm); 447 return ret; 448 } 449 *stream = strm; 450 return 0; 451 } 452 453 static int 454 zip_pmd_stream_free(struct rte_compressdev *dev, void *stream) 455 { 456 struct zip_vf *vf = (struct zip_vf *) (dev->data->dev_private); 457 struct zip_stream *z_stream; 458 459 if (stream == NULL) 460 return 0; 461 462 z_stream = (struct zip_stream *)stream; 463 464 /* Free resources back to pool */ 465 rte_mempool_put_bulk(vf->zip_mp, 466 (void *)&(z_stream->bufs[0]), 467 MAX_BUFS_PER_STREAM); 468 469 /* Zero out the whole structure */ 470 memset(stream, 0, sizeof(struct zip_stream)); 471 rte_free(stream); 472 473 return 0; 474 } 475 476 477 static uint16_t 478 zip_pmd_enqueue_burst_sync(void *queue_pair, 479 struct rte_comp_op **ops, uint16_t nb_ops) 480 { 481 struct zipvf_qp *qp = queue_pair; 482 struct rte_comp_op *op; 483 struct zip_stream *zstrm; 484 int i, ret = 0; 485 uint16_t enqd = 0; 486 487 for (i = 0; i < nb_ops; i++) { 488 op = ops[i]; 489 490 if (op->op_type == RTE_COMP_OP_STATEFUL) { 491 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 492 } else { 493 /* process stateless ops */ 494 zstrm = (struct zip_stream *)op->private_xform; 495 if (unlikely(zstrm == NULL)) 496 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 497 else 498 ret = zstrm->func(op, qp, zstrm); 499 } 500 501 /* Whatever is out of op, put it into completion queue with 502 * its status 503 */ 504 if (!ret) 505 ret = rte_ring_enqueue(qp->processed_pkts, (void *)op); 506 507 if (unlikely(ret < 0)) { 508 /* increment count if failed to enqueue op*/ 509 qp->qp_stats.enqueue_err_count++; 510 } else { 511 qp->qp_stats.enqueued_count++; 512 enqd++; 513 } 514 } 515 return enqd; 516 } 517 518 static uint16_t 519 zip_pmd_dequeue_burst_sync(void *queue_pair, 520 struct rte_comp_op **ops, uint16_t nb_ops) 521 { 522 struct zipvf_qp *qp = queue_pair; 523 524 unsigned int nb_dequeued = 0; 525 526 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, 527 (void **)ops, nb_ops, NULL); 528 qp->qp_stats.dequeued_count += nb_dequeued; 529 530 return nb_dequeued; 531 } 532 533 static struct rte_compressdev_ops octtx_zip_pmd_ops = { 534 .dev_configure = zip_pmd_config, 535 .dev_start = zip_pmd_start, 536 .dev_stop = zip_pmd_stop, 537 .dev_close = zip_pmd_close, 538 539 .stats_get = zip_pmd_stats_get, 540 .stats_reset = zip_pmd_stats_reset, 541 542 .dev_infos_get = zip_pmd_info_get, 543 544 .queue_pair_setup = zip_pmd_qp_setup, 545 .queue_pair_release = zip_pmd_qp_release, 546 547 .private_xform_create = zip_pmd_stream_create, 548 .private_xform_free = zip_pmd_stream_free, 549 .stream_create = NULL, 550 .stream_free = NULL 551 }; 552 553 static int 554 zip_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 555 struct rte_pci_device *pci_dev) 556 { 557 int ret = 0; 558 char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 559 struct rte_compressdev *compressdev; 560 struct rte_compressdev_pmd_init_params init_params = { 561 "", 562 rte_socket_id(), 563 }; 564 565 ZIP_PMD_INFO("vendor_id=0x%x device_id=0x%x", 566 (unsigned int)pci_dev->id.vendor_id, 567 (unsigned int)pci_dev->id.device_id); 568 569 rte_pci_device_name(&pci_dev->addr, compressdev_name, 570 sizeof(compressdev_name)); 571 572 compressdev = rte_compressdev_pmd_create(compressdev_name, 573 &pci_dev->device, sizeof(struct zip_vf), &init_params); 574 if (compressdev == NULL) { 575 ZIP_PMD_ERR("driver %s: create failed", init_params.name); 576 return -ENODEV; 577 } 578 579 /* 580 * create only if proc_type is primary. 581 */ 582 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 583 /* create vf dev with given pmd dev id */ 584 ret = zipvf_create(compressdev); 585 if (ret < 0) { 586 ZIP_PMD_ERR("Device creation failed"); 587 rte_compressdev_pmd_destroy(compressdev); 588 return ret; 589 } 590 } 591 592 compressdev->dev_ops = &octtx_zip_pmd_ops; 593 /* register rx/tx burst functions for data path */ 594 compressdev->dequeue_burst = zip_pmd_dequeue_burst_sync; 595 compressdev->enqueue_burst = zip_pmd_enqueue_burst_sync; 596 compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; 597 return ret; 598 } 599 600 static int 601 zip_pci_remove(struct rte_pci_device *pci_dev) 602 { 603 struct rte_compressdev *compressdev; 604 char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 605 606 if (pci_dev == NULL) { 607 ZIP_PMD_ERR(" Invalid PCI Device\n"); 608 return -EINVAL; 609 } 610 rte_pci_device_name(&pci_dev->addr, compressdev_name, 611 sizeof(compressdev_name)); 612 613 compressdev = rte_compressdev_pmd_get_named_dev(compressdev_name); 614 if (compressdev == NULL) 615 return -ENODEV; 616 617 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 618 if (zipvf_destroy(compressdev) < 0) 619 return -ENODEV; 620 } 621 return rte_compressdev_pmd_destroy(compressdev); 622 } 623 624 static struct rte_pci_id pci_id_octtx_zipvf_table[] = { 625 { 626 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 627 PCI_DEVICE_ID_OCTEONTX_ZIPVF), 628 }, 629 { 630 .device_id = 0 631 }, 632 }; 633 634 /** 635 * Structure that represents a PCI driver 636 */ 637 static struct rte_pci_driver octtx_zip_pmd = { 638 .id_table = pci_id_octtx_zipvf_table, 639 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 640 .probe = zip_pci_probe, 641 .remove = zip_pci_remove, 642 }; 643 644 RTE_PMD_REGISTER_PCI(COMPRESSDEV_NAME_ZIP_PMD, octtx_zip_pmd); 645 RTE_PMD_REGISTER_PCI_TABLE(COMPRESSDEV_NAME_ZIP_PMD, pci_id_octtx_zipvf_table); 646 RTE_LOG_REGISTER_DEFAULT(octtx_zip_logtype_driver, INFO); 647