1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #include <string.h> 6 7 #include <rte_byteorder.h> 8 #include <rte_common.h> 9 #include <rte_cpuflags.h> 10 #include <rte_malloc.h> 11 12 #include "otx_zip.h" 13 14 static const struct rte_compressdev_capabilities 15 octtx_zip_pmd_capabilities[] = { 16 { .algo = RTE_COMP_ALGO_DEFLATE, 17 /* Deflate */ 18 .comp_feature_flags = RTE_COMP_FF_HUFFMAN_FIXED | 19 RTE_COMP_FF_HUFFMAN_DYNAMIC, 20 /* Non sharable Priv XFORM and Stateless */ 21 .window_size = { 22 .min = 1, 23 .max = 14, 24 .increment = 1 25 /* size supported 2^1 to 2^14 */ 26 }, 27 }, 28 RTE_COMP_END_OF_CAPABILITIES_LIST() 29 }; 30 31 /* 32 * Reset session to default state for next set of stateless operation 33 */ 34 static inline void 35 reset_stream(struct zip_stream *z_stream) 36 { 37 union zip_inst_s *inst = (union zip_inst_s *)(z_stream->inst); 38 39 inst->s.bf = 1; 40 inst->s.ef = 0; 41 } 42 43 int 44 zip_process_op(struct rte_comp_op *op, 45 struct zipvf_qp *qp, 46 struct zip_stream *zstrm) 47 { 48 union zip_inst_s *inst = zstrm->inst; 49 volatile union zip_zres_s *zresult = NULL; 50 51 52 if ((op->m_src->nb_segs > 1) || (op->m_dst->nb_segs > 1) || 53 (op->src.offset > rte_pktmbuf_pkt_len(op->m_src)) || 54 (op->dst.offset > rte_pktmbuf_pkt_len(op->m_dst))) { 55 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 56 ZIP_PMD_ERR("Segmented packet is not supported\n"); 57 return 0; 58 } 59 60 zipvf_prepare_cmd_stateless(op, zstrm); 61 62 zresult = (union zip_zres_s *)zstrm->bufs[RES_BUF]; 63 zresult->s.compcode = 0; 64 65 #ifdef ZIP_DBG 66 zip_dump_instruction(inst); 67 #endif 68 69 /* Submit zip command */ 70 zipvf_push_command(qp, (void *)inst); 71 72 /* Check and Process results in sync mode */ 73 do { 74 } while (!zresult->s.compcode); 75 76 if (zresult->s.compcode == ZIP_COMP_E_SUCCESS) { 77 op->status = RTE_COMP_OP_STATUS_SUCCESS; 78 } else { 79 /* FATAL error cannot do anything */ 80 ZIP_PMD_ERR("operation failed with error code:%d\n", 81 zresult->s.compcode); 82 if (zresult->s.compcode == ZIP_COMP_E_DSTOP) 83 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; 84 else 85 op->status = RTE_COMP_OP_STATUS_ERROR; 86 } 87 88 ZIP_PMD_INFO("written %d\n", zresult->s.totalbyteswritten); 89 90 /* Update op stats */ 91 switch (op->status) { 92 case RTE_COMP_OP_STATUS_SUCCESS: 93 op->consumed = zresult->s.totalbytesread; 94 /* Fall-through */ 95 case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED: 96 op->produced = zresult->s.totalbyteswritten; 97 break; 98 default: 99 ZIP_PMD_ERR("stats not updated for status:%d\n", 100 op->status); 101 break; 102 } 103 /* zstream is reset irrespective of result */ 104 reset_stream(zstrm); 105 106 zresult->s.compcode = ZIP_COMP_E_NOTDONE; 107 return 0; 108 } 109 110 /** Parse xform parameters and setup a stream */ 111 static int 112 zip_set_stream_parameters(struct rte_compressdev *dev, 113 const struct rte_comp_xform *xform, 114 struct zip_stream *z_stream) 115 { 116 int ret; 117 union zip_inst_s *inst; 118 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 119 void *res; 120 121 /* Allocate resources required by a stream */ 122 ret = rte_mempool_get_bulk(vf->zip_mp, 123 z_stream->bufs, MAX_BUFS_PER_STREAM); 124 if (ret < 0) 125 return -1; 126 127 /* get one command buffer from pool and set up */ 128 inst = (union zip_inst_s *)z_stream->bufs[CMD_BUF]; 129 res = z_stream->bufs[RES_BUF]; 130 131 memset(inst->u, 0, sizeof(inst->u)); 132 133 /* set bf for only first ops of stream */ 134 inst->s.bf = 1; 135 136 if (xform->type == RTE_COMP_COMPRESS) { 137 inst->s.op = ZIP_OP_E_COMP; 138 139 switch (xform->compress.deflate.huffman) { 140 case RTE_COMP_HUFFMAN_DEFAULT: 141 inst->s.cc = ZIP_CC_DEFAULT; 142 break; 143 case RTE_COMP_HUFFMAN_FIXED: 144 inst->s.cc = ZIP_CC_FIXED_HUFF; 145 break; 146 case RTE_COMP_HUFFMAN_DYNAMIC: 147 inst->s.cc = ZIP_CC_DYN_HUFF; 148 break; 149 default: 150 ret = -1; 151 goto err; 152 } 153 154 switch (xform->compress.level) { 155 case RTE_COMP_LEVEL_MIN: 156 inst->s.ss = ZIP_COMP_E_LEVEL_MIN; 157 break; 158 case RTE_COMP_LEVEL_MAX: 159 inst->s.ss = ZIP_COMP_E_LEVEL_MAX; 160 break; 161 case RTE_COMP_LEVEL_NONE: 162 ZIP_PMD_ERR("Compression level not supported"); 163 ret = -1; 164 goto err; 165 default: 166 /* for any value between min and max , choose 167 * PMD default. 168 */ 169 inst->s.ss = ZIP_COMP_E_LEVEL_MED; /** PMD default **/ 170 break; 171 } 172 } else if (xform->type == RTE_COMP_DECOMPRESS) { 173 inst->s.op = ZIP_OP_E_DECOMP; 174 /* from HRM, 175 * For DEFLATE decompression, [CC] must be 0x0. 176 * For decompression, [SS] must be 0x0 177 */ 178 inst->s.cc = 0; 179 /* Speed bit should not be set for decompression */ 180 inst->s.ss = 0; 181 /* decompression context is supported only for STATEFUL 182 * operations. Currently we support STATELESS ONLY so 183 * skip setting of ctx pointer 184 */ 185 186 } else { 187 ZIP_PMD_ERR("\nxform type not supported"); 188 ret = -1; 189 goto err; 190 } 191 192 inst->s.res_ptr_addr.s.addr = rte_mempool_virt2iova(res); 193 inst->s.res_ptr_ctl.s.length = 0; 194 195 z_stream->inst = inst; 196 z_stream->func = zip_process_op; 197 198 return 0; 199 200 err: 201 rte_mempool_put_bulk(vf->zip_mp, 202 (void *)&(z_stream->bufs[0]), 203 MAX_BUFS_PER_STREAM); 204 205 return ret; 206 } 207 208 /** Configure device */ 209 static int 210 zip_pmd_config(struct rte_compressdev *dev, 211 struct rte_compressdev_config *config) 212 { 213 int nb_streams; 214 char res_pool[RTE_MEMZONE_NAMESIZE]; 215 struct zip_vf *vf; 216 struct rte_mempool *zip_buf_mp; 217 218 if (!config || !dev) 219 return -EIO; 220 221 vf = (struct zip_vf *)(dev->data->dev_private); 222 223 /* create pool with maximum numbers of resources 224 * required by streams 225 */ 226 227 /* use common pool for non-shareable priv_xform and stream */ 228 nb_streams = config->max_nb_priv_xforms + config->max_nb_streams; 229 230 snprintf(res_pool, RTE_MEMZONE_NAMESIZE, "octtx_zip_res_pool%u", 231 dev->data->dev_id); 232 233 /** TBD Should we use the per core object cache for stream resources */ 234 zip_buf_mp = rte_mempool_create( 235 res_pool, 236 nb_streams * MAX_BUFS_PER_STREAM, 237 ZIP_BUF_SIZE, 238 0, 239 0, 240 NULL, 241 NULL, 242 NULL, 243 NULL, 244 SOCKET_ID_ANY, 245 0); 246 247 if (zip_buf_mp == NULL) { 248 ZIP_PMD_ERR( 249 "Failed to create buf mempool octtx_zip_res_pool%u", 250 dev->data->dev_id); 251 return -1; 252 } 253 254 vf->zip_mp = zip_buf_mp; 255 256 return 0; 257 } 258 259 /** Start device */ 260 static int 261 zip_pmd_start(__rte_unused struct rte_compressdev *dev) 262 { 263 return 0; 264 } 265 266 /** Stop device */ 267 static void 268 zip_pmd_stop(__rte_unused struct rte_compressdev *dev) 269 { 270 271 } 272 273 /** Close device */ 274 static int 275 zip_pmd_close(struct rte_compressdev *dev) 276 { 277 if (dev == NULL) 278 return -1; 279 280 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 281 rte_mempool_free(vf->zip_mp); 282 283 return 0; 284 } 285 286 /** Get device statistics */ 287 static void 288 zip_pmd_stats_get(struct rte_compressdev *dev, 289 struct rte_compressdev_stats *stats) 290 { 291 int qp_id; 292 293 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 294 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 295 296 stats->enqueued_count += qp->qp_stats.enqueued_count; 297 stats->dequeued_count += qp->qp_stats.dequeued_count; 298 299 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 300 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 301 } 302 } 303 304 /** Reset device statistics */ 305 static void 306 zip_pmd_stats_reset(struct rte_compressdev *dev) 307 { 308 int qp_id; 309 310 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 311 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 312 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 313 } 314 } 315 316 /** Get device info */ 317 static void 318 zip_pmd_info_get(struct rte_compressdev *dev, 319 struct rte_compressdev_info *dev_info) 320 { 321 struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private; 322 323 if (dev_info != NULL) { 324 dev_info->driver_name = dev->device->driver->name; 325 dev_info->feature_flags = dev->feature_flags; 326 dev_info->capabilities = octtx_zip_pmd_capabilities; 327 dev_info->max_nb_queue_pairs = vf->max_nb_queue_pairs; 328 } 329 } 330 331 /** Release queue pair */ 332 static int 333 zip_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 334 { 335 struct zipvf_qp *qp = dev->data->queue_pairs[qp_id]; 336 337 if (qp != NULL) { 338 zipvf_q_term(qp); 339 340 if (qp->processed_pkts) 341 rte_ring_free(qp->processed_pkts); 342 343 rte_free(qp); 344 dev->data->queue_pairs[qp_id] = NULL; 345 } 346 return 0; 347 } 348 349 /** Create a ring to place process packets on */ 350 static struct rte_ring * 351 zip_pmd_qp_create_processed_pkts_ring(struct zipvf_qp *qp, 352 unsigned int ring_size, int socket_id) 353 { 354 struct rte_ring *r; 355 356 r = rte_ring_lookup(qp->name); 357 if (r) { 358 if (rte_ring_get_size(r) >= ring_size) { 359 ZIP_PMD_INFO("Reusing existing ring %s for processed" 360 " packets", qp->name); 361 return r; 362 } 363 364 ZIP_PMD_ERR("Unable to reuse existing ring %s for processed" 365 " packets", qp->name); 366 return NULL; 367 } 368 369 return rte_ring_create(qp->name, ring_size, socket_id, 370 RING_F_EXACT_SZ); 371 } 372 373 /** Setup a queue pair */ 374 static int 375 zip_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 376 uint32_t max_inflight_ops, int socket_id) 377 { 378 struct zipvf_qp *qp = NULL; 379 struct zip_vf *vf; 380 char *name; 381 int ret; 382 383 if (!dev) 384 return -1; 385 386 vf = (struct zip_vf *) (dev->data->dev_private); 387 388 /* Free memory prior to re-allocation if needed. */ 389 if (dev->data->queue_pairs[qp_id] != NULL) { 390 ZIP_PMD_INFO("Using existing queue pair %d ", qp_id); 391 return 0; 392 } 393 394 name = rte_malloc(NULL, RTE_COMPRESSDEV_NAME_MAX_LEN, 0); 395 snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, 396 "zip_pmd_%u_qp_%u", 397 dev->data->dev_id, qp_id); 398 399 /* Allocate the queue pair data structure. */ 400 qp = rte_zmalloc_socket(name, sizeof(*qp), 401 RTE_CACHE_LINE_SIZE, socket_id); 402 if (qp == NULL) 403 return (-ENOMEM); 404 405 qp->name = name; 406 407 /* Create completion queue up to max_inflight_ops */ 408 qp->processed_pkts = zip_pmd_qp_create_processed_pkts_ring(qp, 409 max_inflight_ops, socket_id); 410 if (qp->processed_pkts == NULL) 411 goto qp_setup_cleanup; 412 413 qp->id = qp_id; 414 qp->vf = vf; 415 416 ret = zipvf_q_init(qp); 417 if (ret < 0) 418 goto qp_setup_cleanup; 419 420 dev->data->queue_pairs[qp_id] = qp; 421 422 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 423 return 0; 424 425 qp_setup_cleanup: 426 if (qp->processed_pkts) 427 rte_ring_free(qp->processed_pkts); 428 if (qp) 429 rte_free(qp); 430 return -1; 431 } 432 433 static int 434 zip_pmd_stream_create(struct rte_compressdev *dev, 435 const struct rte_comp_xform *xform, void **stream) 436 { 437 int ret; 438 struct zip_stream *strm = NULL; 439 440 strm = rte_malloc(NULL, 441 sizeof(struct zip_stream), 0); 442 443 if (strm == NULL) 444 return (-ENOMEM); 445 446 ret = zip_set_stream_parameters(dev, xform, strm); 447 if (ret < 0) { 448 ZIP_PMD_ERR("failed configure xform parameters"); 449 rte_free(strm); 450 return ret; 451 } 452 *stream = strm; 453 return 0; 454 } 455 456 static int 457 zip_pmd_stream_free(struct rte_compressdev *dev, void *stream) 458 { 459 struct zip_vf *vf = (struct zip_vf *) (dev->data->dev_private); 460 struct zip_stream *z_stream; 461 462 if (stream == NULL) 463 return 0; 464 465 z_stream = (struct zip_stream *)stream; 466 467 /* Free resources back to pool */ 468 rte_mempool_put_bulk(vf->zip_mp, 469 (void *)&(z_stream->bufs[0]), 470 MAX_BUFS_PER_STREAM); 471 472 /* Zero out the whole structure */ 473 memset(stream, 0, sizeof(struct zip_stream)); 474 rte_free(stream); 475 476 return 0; 477 } 478 479 480 static uint16_t 481 zip_pmd_enqueue_burst_sync(void *queue_pair, 482 struct rte_comp_op **ops, uint16_t nb_ops) 483 { 484 struct zipvf_qp *qp = queue_pair; 485 struct rte_comp_op *op; 486 struct zip_stream *zstrm; 487 int i, ret = 0; 488 uint16_t enqd = 0; 489 490 for (i = 0; i < nb_ops; i++) { 491 op = ops[i]; 492 493 if (op->op_type == RTE_COMP_OP_STATEFUL) { 494 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 495 } else { 496 /* process stateless ops */ 497 zstrm = (struct zip_stream *)op->private_xform; 498 if (unlikely(zstrm == NULL)) 499 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; 500 else 501 ret = zstrm->func(op, qp, zstrm); 502 } 503 504 /* Whatever is out of op, put it into completion queue with 505 * its status 506 */ 507 if (!ret) 508 ret = rte_ring_enqueue(qp->processed_pkts, (void *)op); 509 510 if (unlikely(ret < 0)) { 511 /* increment count if failed to enqueue op*/ 512 qp->qp_stats.enqueue_err_count++; 513 } else { 514 qp->qp_stats.enqueued_count++; 515 enqd++; 516 } 517 } 518 return enqd; 519 } 520 521 static uint16_t 522 zip_pmd_dequeue_burst_sync(void *queue_pair, 523 struct rte_comp_op **ops, uint16_t nb_ops) 524 { 525 struct zipvf_qp *qp = queue_pair; 526 527 unsigned int nb_dequeued = 0; 528 529 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, 530 (void **)ops, nb_ops, NULL); 531 qp->qp_stats.dequeued_count += nb_dequeued; 532 533 return nb_dequeued; 534 } 535 536 static struct rte_compressdev_ops octtx_zip_pmd_ops = { 537 .dev_configure = zip_pmd_config, 538 .dev_start = zip_pmd_start, 539 .dev_stop = zip_pmd_stop, 540 .dev_close = zip_pmd_close, 541 542 .stats_get = zip_pmd_stats_get, 543 .stats_reset = zip_pmd_stats_reset, 544 545 .dev_infos_get = zip_pmd_info_get, 546 547 .queue_pair_setup = zip_pmd_qp_setup, 548 .queue_pair_release = zip_pmd_qp_release, 549 550 .private_xform_create = zip_pmd_stream_create, 551 .private_xform_free = zip_pmd_stream_free, 552 .stream_create = NULL, 553 .stream_free = NULL 554 }; 555 556 static int 557 zip_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 558 struct rte_pci_device *pci_dev) 559 { 560 int ret = 0; 561 char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 562 struct rte_compressdev *compressdev; 563 struct rte_compressdev_pmd_init_params init_params = { 564 "", 565 rte_socket_id(), 566 }; 567 568 ZIP_PMD_INFO("vendor_id=0x%x device_id=0x%x", 569 (unsigned int)pci_dev->id.vendor_id, 570 (unsigned int)pci_dev->id.device_id); 571 572 rte_pci_device_name(&pci_dev->addr, compressdev_name, 573 sizeof(compressdev_name)); 574 575 compressdev = rte_compressdev_pmd_create(compressdev_name, 576 &pci_dev->device, sizeof(struct zip_vf), &init_params); 577 if (compressdev == NULL) { 578 ZIP_PMD_ERR("driver %s: create failed", init_params.name); 579 return -ENODEV; 580 } 581 582 /* 583 * create only if proc_type is primary. 584 */ 585 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 586 /* create vf dev with given pmd dev id */ 587 ret = zipvf_create(compressdev); 588 if (ret < 0) { 589 ZIP_PMD_ERR("Device creation failed"); 590 rte_compressdev_pmd_destroy(compressdev); 591 return ret; 592 } 593 } 594 595 compressdev->dev_ops = &octtx_zip_pmd_ops; 596 /* register rx/tx burst functions for data path */ 597 compressdev->dequeue_burst = zip_pmd_dequeue_burst_sync; 598 compressdev->enqueue_burst = zip_pmd_enqueue_burst_sync; 599 compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; 600 return ret; 601 } 602 603 static int 604 zip_pci_remove(struct rte_pci_device *pci_dev) 605 { 606 struct rte_compressdev *compressdev; 607 char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 608 609 if (pci_dev == NULL) { 610 ZIP_PMD_ERR(" Invalid PCI Device\n"); 611 return -EINVAL; 612 } 613 rte_pci_device_name(&pci_dev->addr, compressdev_name, 614 sizeof(compressdev_name)); 615 616 compressdev = rte_compressdev_pmd_get_named_dev(compressdev_name); 617 if (compressdev == NULL) 618 return -ENODEV; 619 620 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 621 if (zipvf_destroy(compressdev) < 0) 622 return -ENODEV; 623 } 624 return rte_compressdev_pmd_destroy(compressdev); 625 } 626 627 static struct rte_pci_id pci_id_octtx_zipvf_table[] = { 628 { 629 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 630 PCI_DEVICE_ID_OCTEONTX_ZIPVF), 631 }, 632 { 633 .device_id = 0 634 }, 635 }; 636 637 /** 638 * Structure that represents a PCI driver 639 */ 640 static struct rte_pci_driver octtx_zip_pmd = { 641 .id_table = pci_id_octtx_zipvf_table, 642 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 643 .probe = zip_pci_probe, 644 .remove = zip_pci_remove, 645 }; 646 647 RTE_PMD_REGISTER_PCI(COMPRESSDEV_NAME_ZIP_PMD, octtx_zip_pmd); 648 RTE_PMD_REGISTER_PCI_TABLE(COMPRESSDEV_NAME_ZIP_PMD, pci_id_octtx_zipvf_table); 649 RTE_LOG_REGISTER(octtx_zip_logtype_driver, pmd.compress.octeontx, INFO); 650