1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_common.h> 7 #include <rte_compressdev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "isal_compress_pmd_private.h" 11 12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { 13 { 14 .algo = RTE_COMP_ALGO_DEFLATE, 15 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | 16 RTE_COMP_FF_OOP_SGL_IN_LB_OUT | 17 RTE_COMP_FF_OOP_LB_IN_SGL_OUT | 18 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 19 RTE_COMP_FF_HUFFMAN_FIXED | 20 RTE_COMP_FF_HUFFMAN_DYNAMIC | 21 RTE_COMP_FF_CRC32_CHECKSUM | 22 RTE_COMP_FF_ADLER32_CHECKSUM, 23 .window_size = { 24 .min = 15, 25 .max = 15, 26 .increment = 0 27 }, 28 }, 29 RTE_COMP_END_OF_CAPABILITIES_LIST() 30 }; 31 32 /** Configure device */ 33 static int 34 isal_comp_pmd_config(struct rte_compressdev *dev, 35 struct rte_compressdev_config *config) 36 { 37 int ret = 0; 38 unsigned int n; 39 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 40 unsigned int elt_size = sizeof(struct isal_priv_xform); 41 struct isal_comp_private *internals = dev->data->dev_private; 42 43 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", 44 dev->data->dev_id); 45 if (n > sizeof(mp_name)) { 46 ISAL_PMD_LOG(ERR, 47 "Unable to create unique name for xform mempool"); 48 return -ENOMEM; 49 } 50 51 internals->priv_xform_mp = rte_mempool_lookup(mp_name); 52 53 if (internals->priv_xform_mp != NULL) { 54 if (((internals->priv_xform_mp)->elt_size != elt_size) || 55 ((internals->priv_xform_mp)->size < 56 config->max_nb_priv_xforms)) { 57 58 ISAL_PMD_LOG(ERR, "%s mempool already exists with different" 59 " initialization parameters", mp_name); 60 internals->priv_xform_mp = NULL; 61 return -ENOMEM; 62 } 63 } else { /* First time configuration */ 64 internals->priv_xform_mp = rte_mempool_create( 65 mp_name, /* mempool name */ 66 /* number of elements*/ 67 config->max_nb_priv_xforms, 68 elt_size, /* element size*/ 69 0, /* Cache size*/ 70 0, /* private data size */ 71 NULL, /* obj initialization constructor */ 72 NULL, /* obj initialization constructor arg */ 73 NULL, /**< obj constructor*/ 74 NULL, /* obj constructor arg */ 75 config->socket_id, /* socket id */ 76 0); /* flags */ 77 } 78 79 if (internals->priv_xform_mp == NULL) { 80 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); 81 return -ENOMEM; 82 } 83 84 dev->data->dev_private = internals; 85 86 return ret; 87 } 88 89 /** Start device */ 90 static int 91 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) 92 { 93 return 0; 94 } 95 96 /** Stop device */ 97 static void 98 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) 99 { 100 } 101 102 /** Close device */ 103 static int 104 isal_comp_pmd_close(struct rte_compressdev *dev) 105 { 106 /* Free private data */ 107 struct isal_comp_private *internals = dev->data->dev_private; 108 109 rte_mempool_free(internals->priv_xform_mp); 110 return 0; 111 } 112 113 /** Get device statistics */ 114 static void 115 isal_comp_pmd_stats_get(struct rte_compressdev *dev, 116 struct rte_compressdev_stats *stats) 117 { 118 uint16_t qp_id; 119 120 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 121 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 122 123 stats->enqueued_count += qp->qp_stats.enqueued_count; 124 stats->dequeued_count += qp->qp_stats.dequeued_count; 125 126 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 127 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 128 } 129 } 130 131 /** Get device info */ 132 static void 133 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, 134 struct rte_compressdev_info *dev_info) 135 { 136 if (dev_info != NULL) { 137 dev_info->capabilities = isal_pmd_capabilities; 138 dev_info->feature_flags = RTE_COMPDEV_FF_CPU_AVX512 | 139 RTE_COMPDEV_FF_CPU_AVX2 | 140 RTE_COMPDEV_FF_CPU_AVX | 141 RTE_COMPDEV_FF_CPU_SSE; 142 } 143 } 144 145 /** Reset device statistics */ 146 static void 147 isal_comp_pmd_stats_reset(struct rte_compressdev *dev) 148 { 149 uint16_t qp_id; 150 151 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 152 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 153 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 154 } 155 } 156 157 /** Release queue pair */ 158 static int 159 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 160 { 161 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 162 163 if (qp == NULL) 164 return -EINVAL; 165 166 if (qp->stream != NULL) 167 rte_free(qp->stream); 168 169 if (qp->stream->level_buf != NULL) 170 rte_free(qp->stream->level_buf); 171 172 if (qp->state != NULL) 173 rte_free(qp->state); 174 175 if (qp->processed_pkts != NULL) 176 rte_ring_free(qp->processed_pkts); 177 178 rte_free(qp); 179 dev->data->queue_pairs[qp_id] = NULL; 180 181 return 0; 182 } 183 184 /** Create a ring to place process packets on */ 185 static struct rte_ring * 186 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, 187 unsigned int ring_size, int socket_id) 188 { 189 struct rte_ring *r; 190 191 r = rte_ring_lookup(qp->name); 192 if (r) { 193 if (rte_ring_get_size(r) >= ring_size) { 194 ISAL_PMD_LOG(DEBUG, 195 "Reusing existing ring %s for processed packets", 196 qp->name); 197 return r; 198 } 199 200 ISAL_PMD_LOG(ERR, 201 "Unable to reuse existing ring %s" 202 " for processed packets", 203 qp->name); 204 return NULL; 205 } 206 207 return rte_ring_create(qp->name, ring_size, socket_id, 208 RING_F_SP_ENQ | RING_F_SC_DEQ); 209 } 210 211 /** set a unique name for the queue pair based on its name, dev_id and qp_id */ 212 static int 213 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, 214 struct isal_comp_qp *qp) 215 { 216 unsigned int n = snprintf(qp->name, sizeof(qp->name), 217 "isal_compression_pmd_%u_qp_%u", 218 dev->data->dev_id, qp->id); 219 220 if (n >= sizeof(qp->name)) 221 return -1; 222 223 return 0; 224 } 225 226 /* Setup a queue pair */ 227 static int 228 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 229 uint32_t max_inflight_ops, int socket_id) 230 { 231 struct isal_comp_qp *qp = NULL; 232 int retval; 233 234 /* Free memory prior to re-allocation if needed. */ 235 if (dev->data->queue_pairs[qp_id] != NULL) 236 isal_comp_pmd_qp_release(dev, qp_id); 237 238 /* Allocate the queue pair data structure. */ 239 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), 240 RTE_CACHE_LINE_SIZE, socket_id); 241 if (qp == NULL) { 242 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); 243 return (-ENOMEM); 244 } 245 246 /* Initialize memory for compression stream structure */ 247 qp->stream = rte_zmalloc_socket("Isa-l compression stream ", 248 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, 249 socket_id); 250 251 /* Initialize memory for compression level buffer */ 252 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", 253 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, 254 socket_id); 255 256 /* Initialize memory for decompression state structure */ 257 qp->state = rte_zmalloc_socket("Isa-l decompression state", 258 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, 259 socket_id); 260 261 qp->id = qp_id; 262 dev->data->queue_pairs[qp_id] = qp; 263 264 retval = isal_comp_pmd_qp_set_unique_name(dev, qp); 265 if (retval) { 266 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 267 "compression device"); 268 goto qp_setup_cleanup; 269 } 270 271 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, 272 max_inflight_ops, socket_id); 273 if (qp->processed_pkts == NULL) { 274 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 275 "compression device"); 276 goto qp_setup_cleanup; 277 } 278 279 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); 280 281 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 282 return 0; 283 284 qp_setup_cleanup: 285 if (qp) 286 rte_free(qp); 287 288 return -1; 289 } 290 291 /** Set private xform data*/ 292 static int 293 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, 294 const struct rte_comp_xform *xform, void **priv_xform) 295 { 296 int ret; 297 struct isal_comp_private *internals = dev->data->dev_private; 298 299 if (xform == NULL) { 300 ISAL_PMD_LOG(ERR, "Invalid Xform struct"); 301 return -EINVAL; 302 } 303 304 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { 305 ISAL_PMD_LOG(ERR, 306 "Couldn't get object from private xform mempool"); 307 return -ENOMEM; 308 } 309 310 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); 311 if (ret != 0) { 312 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); 313 314 /* Return private xform to mempool */ 315 rte_mempool_put(internals->priv_xform_mp, priv_xform); 316 return ret; 317 } 318 return 0; 319 } 320 321 /** Clear memory of the private xform so it doesn't leave key material behind */ 322 static int 323 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) 324 { 325 struct isal_comp_private *internals = dev->data->dev_private; 326 327 /* Zero out the whole structure */ 328 if (priv_xform) { 329 memset(priv_xform, 0, sizeof(struct isal_priv_xform)); 330 rte_mempool_put(internals->priv_xform_mp, priv_xform); 331 } 332 return 0; 333 } 334 335 struct rte_compressdev_ops isal_pmd_ops = { 336 .dev_configure = isal_comp_pmd_config, 337 .dev_start = isal_comp_pmd_start, 338 .dev_stop = isal_comp_pmd_stop, 339 .dev_close = isal_comp_pmd_close, 340 341 .stats_get = isal_comp_pmd_stats_get, 342 .stats_reset = isal_comp_pmd_stats_reset, 343 344 .dev_infos_get = isal_comp_pmd_info_get, 345 346 .queue_pair_setup = isal_comp_pmd_qp_setup, 347 .queue_pair_release = isal_comp_pmd_qp_release, 348 349 .private_xform_create = isal_comp_pmd_priv_xform_create, 350 .private_xform_free = isal_comp_pmd_priv_xform_free, 351 }; 352 353 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; 354