1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_common.h> 7 #include <rte_compressdev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "isal_compress_pmd_private.h" 11 12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { 13 { 14 .algo = RTE_COMP_ALGO_DEFLATE, 15 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | 16 RTE_COMP_FF_OOP_SGL_IN_LB_OUT | 17 RTE_COMP_FF_OOP_LB_IN_SGL_OUT | 18 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 19 RTE_COMP_FF_HUFFMAN_FIXED | 20 RTE_COMP_FF_HUFFMAN_DYNAMIC, 21 .window_size = { 22 .min = 15, 23 .max = 15, 24 .increment = 0 25 }, 26 }, 27 RTE_COMP_END_OF_CAPABILITIES_LIST() 28 }; 29 30 /** Configure device */ 31 static int 32 isal_comp_pmd_config(struct rte_compressdev *dev, 33 struct rte_compressdev_config *config) 34 { 35 int ret = 0; 36 unsigned int n; 37 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 38 unsigned int elt_size = sizeof(struct isal_priv_xform); 39 struct isal_comp_private *internals = dev->data->dev_private; 40 41 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", 42 dev->data->dev_id); 43 if (n > sizeof(mp_name)) { 44 ISAL_PMD_LOG(ERR, 45 "Unable to create unique name for xform mempool"); 46 return -ENOMEM; 47 } 48 49 internals->priv_xform_mp = rte_mempool_lookup(mp_name); 50 51 if (internals->priv_xform_mp != NULL) { 52 if (((internals->priv_xform_mp)->elt_size != elt_size) || 53 ((internals->priv_xform_mp)->size < 54 config->max_nb_priv_xforms)) { 55 56 ISAL_PMD_LOG(ERR, "%s mempool already exists with different" 57 " initialization parameters", mp_name); 58 internals->priv_xform_mp = NULL; 59 return -ENOMEM; 60 } 61 } else { /* First time configuration */ 62 internals->priv_xform_mp = rte_mempool_create( 63 mp_name, /* mempool name */ 64 /* number of elements*/ 65 config->max_nb_priv_xforms, 66 elt_size, /* element size*/ 67 0, /* Cache size*/ 68 0, /* private data size */ 69 NULL, /* obj initialization constructor */ 70 NULL, /* obj initialization constructor arg */ 71 NULL, /**< obj constructor*/ 72 NULL, /* obj constructor arg */ 73 config->socket_id, /* socket id */ 74 0); /* flags */ 75 } 76 77 if (internals->priv_xform_mp == NULL) { 78 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); 79 return -ENOMEM; 80 } 81 82 dev->data->dev_private = internals; 83 84 return ret; 85 } 86 87 /** Start device */ 88 static int 89 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) 90 { 91 return 0; 92 } 93 94 /** Stop device */ 95 static void 96 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) 97 { 98 } 99 100 /** Close device */ 101 static int 102 isal_comp_pmd_close(struct rte_compressdev *dev) 103 { 104 /* Free private data */ 105 struct isal_comp_private *internals = dev->data->dev_private; 106 107 rte_mempool_free(internals->priv_xform_mp); 108 return 0; 109 } 110 111 /** Get device statistics */ 112 static void 113 isal_comp_pmd_stats_get(struct rte_compressdev *dev, 114 struct rte_compressdev_stats *stats) 115 { 116 uint16_t qp_id; 117 118 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 119 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 120 121 stats->enqueued_count += qp->qp_stats.enqueued_count; 122 stats->dequeued_count += qp->qp_stats.dequeued_count; 123 124 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 125 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 126 } 127 } 128 129 /** Get device info */ 130 static void 131 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, 132 struct rte_compressdev_info *dev_info) 133 { 134 if (dev_info != NULL) { 135 dev_info->capabilities = isal_pmd_capabilities; 136 dev_info->feature_flags = RTE_COMPDEV_FF_CPU_AVX512 | 137 RTE_COMPDEV_FF_CPU_AVX2 | 138 RTE_COMPDEV_FF_CPU_AVX | 139 RTE_COMPDEV_FF_CPU_SSE; 140 } 141 } 142 143 /** Reset device statistics */ 144 static void 145 isal_comp_pmd_stats_reset(struct rte_compressdev *dev) 146 { 147 uint16_t qp_id; 148 149 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 150 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 151 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 152 } 153 } 154 155 /** Release queue pair */ 156 static int 157 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 158 { 159 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 160 161 if (qp == NULL) 162 return -EINVAL; 163 164 if (qp->stream != NULL) 165 rte_free(qp->stream); 166 167 if (qp->stream->level_buf != NULL) 168 rte_free(qp->stream->level_buf); 169 170 if (qp->state != NULL) 171 rte_free(qp->state); 172 173 if (qp->processed_pkts != NULL) 174 rte_ring_free(qp->processed_pkts); 175 176 rte_free(qp); 177 dev->data->queue_pairs[qp_id] = NULL; 178 179 return 0; 180 } 181 182 /** Create a ring to place process packets on */ 183 static struct rte_ring * 184 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, 185 unsigned int ring_size, int socket_id) 186 { 187 struct rte_ring *r; 188 189 r = rte_ring_lookup(qp->name); 190 if (r) { 191 if (rte_ring_get_size(r) >= ring_size) { 192 ISAL_PMD_LOG(DEBUG, 193 "Reusing existing ring %s for processed packets", 194 qp->name); 195 return r; 196 } 197 198 ISAL_PMD_LOG(ERR, 199 "Unable to reuse existing ring %s" 200 " for processed packets", 201 qp->name); 202 return NULL; 203 } 204 205 return rte_ring_create(qp->name, ring_size, socket_id, 206 RING_F_SP_ENQ | RING_F_SC_DEQ); 207 } 208 209 /** set a unique name for the queue pair based on its name, dev_id and qp_id */ 210 static int 211 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, 212 struct isal_comp_qp *qp) 213 { 214 unsigned int n = snprintf(qp->name, sizeof(qp->name), 215 "isal_compression_pmd_%u_qp_%u", 216 dev->data->dev_id, qp->id); 217 218 if (n >= sizeof(qp->name)) 219 return -1; 220 221 return 0; 222 } 223 224 /* Setup a queue pair */ 225 static int 226 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 227 uint32_t max_inflight_ops, int socket_id) 228 { 229 struct isal_comp_qp *qp = NULL; 230 int retval; 231 232 /* Free memory prior to re-allocation if needed. */ 233 if (dev->data->queue_pairs[qp_id] != NULL) 234 isal_comp_pmd_qp_release(dev, qp_id); 235 236 /* Allocate the queue pair data structure. */ 237 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), 238 RTE_CACHE_LINE_SIZE, socket_id); 239 if (qp == NULL) { 240 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); 241 return (-ENOMEM); 242 } 243 244 /* Initialize memory for compression stream structure */ 245 qp->stream = rte_zmalloc_socket("Isa-l compression stream ", 246 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, 247 socket_id); 248 249 /* Initialize memory for compression level buffer */ 250 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", 251 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, 252 socket_id); 253 254 /* Initialize memory for decompression state structure */ 255 qp->state = rte_zmalloc_socket("Isa-l decompression state", 256 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, 257 socket_id); 258 259 qp->id = qp_id; 260 dev->data->queue_pairs[qp_id] = qp; 261 262 retval = isal_comp_pmd_qp_set_unique_name(dev, qp); 263 if (retval) { 264 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 265 "compression device"); 266 goto qp_setup_cleanup; 267 } 268 269 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, 270 max_inflight_ops, socket_id); 271 if (qp->processed_pkts == NULL) { 272 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 273 "compression device"); 274 goto qp_setup_cleanup; 275 } 276 277 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); 278 279 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 280 return 0; 281 282 qp_setup_cleanup: 283 if (qp) 284 rte_free(qp); 285 286 return -1; 287 } 288 289 /** Set private xform data*/ 290 static int 291 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, 292 const struct rte_comp_xform *xform, void **priv_xform) 293 { 294 int ret; 295 struct isal_comp_private *internals = dev->data->dev_private; 296 297 if (xform == NULL) { 298 ISAL_PMD_LOG(ERR, "Invalid Xform struct"); 299 return -EINVAL; 300 } 301 302 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { 303 ISAL_PMD_LOG(ERR, 304 "Couldn't get object from private xform mempool"); 305 return -ENOMEM; 306 } 307 308 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); 309 if (ret != 0) { 310 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); 311 312 /* Return private xform to mempool */ 313 rte_mempool_put(internals->priv_xform_mp, priv_xform); 314 return ret; 315 } 316 return 0; 317 } 318 319 /** Clear memory of the private xform so it doesn't leave key material behind */ 320 static int 321 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) 322 { 323 struct isal_comp_private *internals = dev->data->dev_private; 324 325 /* Zero out the whole structure */ 326 if (priv_xform) { 327 memset(priv_xform, 0, sizeof(struct isal_priv_xform)); 328 rte_mempool_put(internals->priv_xform_mp, priv_xform); 329 } 330 return 0; 331 } 332 333 struct rte_compressdev_ops isal_pmd_ops = { 334 .dev_configure = isal_comp_pmd_config, 335 .dev_start = isal_comp_pmd_start, 336 .dev_stop = isal_comp_pmd_stop, 337 .dev_close = isal_comp_pmd_close, 338 339 .stats_get = isal_comp_pmd_stats_get, 340 .stats_reset = isal_comp_pmd_stats_reset, 341 342 .dev_infos_get = isal_comp_pmd_info_get, 343 344 .queue_pair_setup = isal_comp_pmd_qp_setup, 345 .queue_pair_release = isal_comp_pmd_qp_release, 346 347 .private_xform_create = isal_comp_pmd_priv_xform_create, 348 .private_xform_free = isal_comp_pmd_priv_xform_free, 349 }; 350 351 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; 352