1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_common.h> 7 #include <rte_compressdev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "isal_compress_pmd_private.h" 11 12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { 13 { 14 .algo = RTE_COMP_ALGO_DEFLATE, 15 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | 16 RTE_COMP_FF_OOP_SGL_IN_LB_OUT | 17 RTE_COMP_FF_OOP_LB_IN_SGL_OUT | 18 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 19 RTE_COMP_FF_HUFFMAN_FIXED | 20 RTE_COMP_FF_HUFFMAN_DYNAMIC | 21 RTE_COMP_FF_CRC32_CHECKSUM | 22 RTE_COMP_FF_ADLER32_CHECKSUM, 23 .window_size = { 24 .min = 15, 25 .max = 15, 26 .increment = 0 27 }, 28 }, 29 RTE_COMP_END_OF_CAPABILITIES_LIST() 30 }; 31 32 /** Configure device */ 33 static int 34 isal_comp_pmd_config(struct rte_compressdev *dev, 35 struct rte_compressdev_config *config) 36 { 37 int ret = 0; 38 unsigned int n; 39 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 40 unsigned int elt_size = sizeof(struct isal_priv_xform); 41 struct isal_comp_private *internals = dev->data->dev_private; 42 43 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", 44 dev->data->dev_id); 45 if (n > sizeof(mp_name)) { 46 ISAL_PMD_LOG(ERR, 47 "Unable to create unique name for xform mempool"); 48 return -ENOMEM; 49 } 50 51 internals->priv_xform_mp = rte_mempool_lookup(mp_name); 52 53 if (internals->priv_xform_mp != NULL) { 54 if (((internals->priv_xform_mp)->elt_size != elt_size) || 55 ((internals->priv_xform_mp)->size < 56 config->max_nb_priv_xforms)) { 57 58 ISAL_PMD_LOG(ERR, "%s mempool already exists with different" 59 " initialization parameters", mp_name); 60 internals->priv_xform_mp = NULL; 61 return -ENOMEM; 62 } 63 } else { /* First time configuration */ 64 internals->priv_xform_mp = rte_mempool_create( 65 mp_name, /* mempool name */ 66 /* number of elements*/ 67 config->max_nb_priv_xforms, 68 elt_size, /* element size*/ 69 0, /* Cache size*/ 70 0, /* private data size */ 71 NULL, /* obj initialization constructor */ 72 NULL, /* obj initialization constructor arg */ 73 NULL, /**< obj constructor*/ 74 NULL, /* obj constructor arg */ 75 config->socket_id, /* socket id */ 76 0); /* flags */ 77 } 78 79 if (internals->priv_xform_mp == NULL) { 80 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); 81 return -ENOMEM; 82 } 83 84 dev->data->dev_private = internals; 85 86 return ret; 87 } 88 89 /** Start device */ 90 static int 91 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) 92 { 93 return 0; 94 } 95 96 /** Stop device */ 97 static void 98 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) 99 { 100 } 101 102 /** Close device */ 103 static int 104 isal_comp_pmd_close(struct rte_compressdev *dev) 105 { 106 /* Free private data */ 107 struct isal_comp_private *internals = dev->data->dev_private; 108 109 rte_mempool_free(internals->priv_xform_mp); 110 return 0; 111 } 112 113 /** Get device statistics */ 114 static void 115 isal_comp_pmd_stats_get(struct rte_compressdev *dev, 116 struct rte_compressdev_stats *stats) 117 { 118 uint16_t qp_id; 119 120 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 121 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 122 123 stats->enqueued_count += qp->qp_stats.enqueued_count; 124 stats->dequeued_count += qp->qp_stats.dequeued_count; 125 126 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 127 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 128 } 129 } 130 131 /** Get device info */ 132 static void 133 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, 134 struct rte_compressdev_info *dev_info) 135 { 136 if (dev_info != NULL) { 137 dev_info->capabilities = isal_pmd_capabilities; 138 139 /* Check CPU for supported vector instruction and set 140 * feature_flags 141 */ 142 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) 143 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512; 144 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) 145 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2; 146 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) 147 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX; 148 else 149 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE; 150 } 151 } 152 153 /** Reset device statistics */ 154 static void 155 isal_comp_pmd_stats_reset(struct rte_compressdev *dev) 156 { 157 uint16_t qp_id; 158 159 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 160 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 161 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 162 } 163 } 164 165 /** Release queue pair */ 166 static int 167 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 168 { 169 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 170 171 if (qp == NULL) 172 return -EINVAL; 173 174 if (qp->stream != NULL) 175 rte_free(qp->stream); 176 177 if (qp->stream->level_buf != NULL) 178 rte_free(qp->stream->level_buf); 179 180 if (qp->state != NULL) 181 rte_free(qp->state); 182 183 if (qp->processed_pkts != NULL) 184 rte_ring_free(qp->processed_pkts); 185 186 rte_free(qp); 187 dev->data->queue_pairs[qp_id] = NULL; 188 189 return 0; 190 } 191 192 /** Create a ring to place process packets on */ 193 static struct rte_ring * 194 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, 195 unsigned int ring_size, int socket_id) 196 { 197 struct rte_ring *r; 198 199 r = rte_ring_lookup(qp->name); 200 if (r) { 201 if (rte_ring_get_size(r) >= ring_size) { 202 ISAL_PMD_LOG(DEBUG, 203 "Reusing existing ring %s for processed packets", 204 qp->name); 205 return r; 206 } 207 208 ISAL_PMD_LOG(ERR, 209 "Unable to reuse existing ring %s" 210 " for processed packets", 211 qp->name); 212 return NULL; 213 } 214 215 return rte_ring_create(qp->name, ring_size, socket_id, 216 RING_F_SP_ENQ | RING_F_SC_DEQ); 217 } 218 219 /** set a unique name for the queue pair based on its name, dev_id and qp_id */ 220 static int 221 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, 222 struct isal_comp_qp *qp) 223 { 224 unsigned int n = snprintf(qp->name, sizeof(qp->name), 225 "isal_compression_pmd_%u_qp_%u", 226 dev->data->dev_id, qp->id); 227 228 if (n >= sizeof(qp->name)) 229 return -1; 230 231 return 0; 232 } 233 234 /* Setup a queue pair */ 235 static int 236 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 237 uint32_t max_inflight_ops, int socket_id) 238 { 239 struct isal_comp_qp *qp = NULL; 240 int retval; 241 242 /* Free memory prior to re-allocation if needed. */ 243 if (dev->data->queue_pairs[qp_id] != NULL) 244 isal_comp_pmd_qp_release(dev, qp_id); 245 246 /* Allocate the queue pair data structure. */ 247 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), 248 RTE_CACHE_LINE_SIZE, socket_id); 249 if (qp == NULL) { 250 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); 251 return (-ENOMEM); 252 } 253 254 /* Initialize memory for compression stream structure */ 255 qp->stream = rte_zmalloc_socket("Isa-l compression stream ", 256 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, 257 socket_id); 258 259 /* Initialize memory for compression level buffer */ 260 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", 261 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, 262 socket_id); 263 264 /* Initialize memory for decompression state structure */ 265 qp->state = rte_zmalloc_socket("Isa-l decompression state", 266 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, 267 socket_id); 268 269 qp->id = qp_id; 270 dev->data->queue_pairs[qp_id] = qp; 271 272 retval = isal_comp_pmd_qp_set_unique_name(dev, qp); 273 if (retval) { 274 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 275 "compression device"); 276 goto qp_setup_cleanup; 277 } 278 279 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, 280 max_inflight_ops, socket_id); 281 if (qp->processed_pkts == NULL) { 282 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 283 "compression device"); 284 goto qp_setup_cleanup; 285 } 286 287 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); 288 289 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 290 return 0; 291 292 qp_setup_cleanup: 293 if (qp) 294 rte_free(qp); 295 296 return -1; 297 } 298 299 /** Set private xform data*/ 300 static int 301 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, 302 const struct rte_comp_xform *xform, void **priv_xform) 303 { 304 int ret; 305 struct isal_comp_private *internals = dev->data->dev_private; 306 307 if (xform == NULL) { 308 ISAL_PMD_LOG(ERR, "Invalid Xform struct"); 309 return -EINVAL; 310 } 311 312 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { 313 ISAL_PMD_LOG(ERR, 314 "Couldn't get object from private xform mempool"); 315 return -ENOMEM; 316 } 317 318 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); 319 if (ret != 0) { 320 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); 321 322 /* Return private xform to mempool */ 323 rte_mempool_put(internals->priv_xform_mp, priv_xform); 324 return ret; 325 } 326 return 0; 327 } 328 329 /** Clear memory of the private xform so it doesn't leave key material behind */ 330 static int 331 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) 332 { 333 struct isal_comp_private *internals = dev->data->dev_private; 334 335 /* Zero out the whole structure */ 336 if (priv_xform) { 337 memset(priv_xform, 0, sizeof(struct isal_priv_xform)); 338 rte_mempool_put(internals->priv_xform_mp, priv_xform); 339 } 340 return 0; 341 } 342 343 struct rte_compressdev_ops isal_pmd_ops = { 344 .dev_configure = isal_comp_pmd_config, 345 .dev_start = isal_comp_pmd_start, 346 .dev_stop = isal_comp_pmd_stop, 347 .dev_close = isal_comp_pmd_close, 348 349 .stats_get = isal_comp_pmd_stats_get, 350 .stats_reset = isal_comp_pmd_stats_reset, 351 352 .dev_infos_get = isal_comp_pmd_info_get, 353 354 .queue_pair_setup = isal_comp_pmd_qp_setup, 355 .queue_pair_release = isal_comp_pmd_qp_release, 356 357 .private_xform_create = isal_comp_pmd_priv_xform_create, 358 .private_xform_free = isal_comp_pmd_priv_xform_free, 359 }; 360 361 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; 362