1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_common.h> 7 #include <rte_compressdev_pmd.h> 8 #include <rte_malloc.h> 9 10 #include "isal_compress_pmd_private.h" 11 12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { 13 { 14 .algo = RTE_COMP_ALGO_DEFLATE, 15 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | 16 RTE_COMP_FF_OOP_SGL_IN_LB_OUT | 17 RTE_COMP_FF_OOP_LB_IN_SGL_OUT | 18 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 19 RTE_COMP_FF_HUFFMAN_FIXED | 20 RTE_COMP_FF_HUFFMAN_DYNAMIC | 21 RTE_COMP_FF_CRC32_CHECKSUM | 22 RTE_COMP_FF_ADLER32_CHECKSUM, 23 .window_size = { 24 .min = 15, 25 .max = 15, 26 .increment = 0 27 }, 28 }, 29 RTE_COMP_END_OF_CAPABILITIES_LIST() 30 }; 31 32 /** Configure device */ 33 static int 34 isal_comp_pmd_config(struct rte_compressdev *dev, 35 struct rte_compressdev_config *config) 36 { 37 int ret = 0; 38 unsigned int n; 39 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 40 unsigned int elt_size = sizeof(struct isal_priv_xform); 41 struct isal_comp_private *internals = dev->data->dev_private; 42 43 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", 44 dev->data->dev_id); 45 if (n > sizeof(mp_name)) { 46 ISAL_PMD_LOG(ERR, 47 "Unable to create unique name for xform mempool"); 48 return -ENOMEM; 49 } 50 51 internals->priv_xform_mp = rte_mempool_lookup(mp_name); 52 53 if (internals->priv_xform_mp != NULL) { 54 if (((internals->priv_xform_mp)->elt_size != elt_size) || 55 ((internals->priv_xform_mp)->size < 56 config->max_nb_priv_xforms)) { 57 58 ISAL_PMD_LOG(ERR, "%s mempool already exists with different" 59 " initialization parameters", mp_name); 60 internals->priv_xform_mp = NULL; 61 return -ENOMEM; 62 } 63 } else { /* First time configuration */ 64 internals->priv_xform_mp = rte_mempool_create( 65 mp_name, /* mempool name */ 66 /* number of elements*/ 67 config->max_nb_priv_xforms, 68 elt_size, /* element size*/ 69 0, /* Cache size*/ 70 0, /* private data size */ 71 NULL, /* obj initialization constructor */ 72 NULL, /* obj initialization constructor arg */ 73 NULL, /**< obj constructor*/ 74 NULL, /* obj constructor arg */ 75 config->socket_id, /* socket id */ 76 0); /* flags */ 77 } 78 79 if (internals->priv_xform_mp == NULL) { 80 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); 81 return -ENOMEM; 82 } 83 84 dev->data->dev_private = internals; 85 86 return ret; 87 } 88 89 /** Start device */ 90 static int 91 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) 92 { 93 return 0; 94 } 95 96 /** Stop device */ 97 static void 98 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) 99 { 100 } 101 102 /** Close device */ 103 static int 104 isal_comp_pmd_close(struct rte_compressdev *dev) 105 { 106 /* Free private data */ 107 struct isal_comp_private *internals = dev->data->dev_private; 108 109 rte_mempool_free(internals->priv_xform_mp); 110 return 0; 111 } 112 113 /** Get device statistics */ 114 static void 115 isal_comp_pmd_stats_get(struct rte_compressdev *dev, 116 struct rte_compressdev_stats *stats) 117 { 118 uint16_t qp_id; 119 120 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 121 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 122 123 stats->enqueued_count += qp->qp_stats.enqueued_count; 124 stats->dequeued_count += qp->qp_stats.dequeued_count; 125 126 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 127 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 128 } 129 } 130 131 /** Get device info */ 132 static void 133 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, 134 struct rte_compressdev_info *dev_info) 135 { 136 if (dev_info != NULL) { 137 dev_info->capabilities = isal_pmd_capabilities; 138 139 /* Check CPU for supported vector instruction and set 140 * feature_flags 141 */ 142 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) 143 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512; 144 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) 145 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2; 146 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) 147 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX; 148 else 149 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE; 150 } 151 } 152 153 /** Reset device statistics */ 154 static void 155 isal_comp_pmd_stats_reset(struct rte_compressdev *dev) 156 { 157 uint16_t qp_id; 158 159 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 160 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 161 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 162 } 163 } 164 165 /** Release queue pair */ 166 static int 167 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 168 { 169 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 170 171 if (qp == NULL) 172 return -EINVAL; 173 174 if (qp->stream) 175 rte_free(qp->stream->level_buf); 176 177 rte_free(qp->state); 178 rte_ring_free(qp->processed_pkts); 179 rte_free(qp->stream); 180 rte_free(qp); 181 dev->data->queue_pairs[qp_id] = NULL; 182 183 return 0; 184 } 185 186 /** Create a ring to place process packets on */ 187 static struct rte_ring * 188 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, 189 unsigned int ring_size, int socket_id) 190 { 191 struct rte_ring *r; 192 193 r = rte_ring_lookup(qp->name); 194 if (r) { 195 if (rte_ring_get_size(r) >= ring_size) { 196 ISAL_PMD_LOG(DEBUG, 197 "Reusing existing ring %s for processed packets", 198 qp->name); 199 return r; 200 } 201 202 ISAL_PMD_LOG(ERR, 203 "Unable to reuse existing ring %s" 204 " for processed packets", 205 qp->name); 206 return NULL; 207 } 208 209 return rte_ring_create(qp->name, ring_size, socket_id, 210 RING_F_SP_ENQ | RING_F_SC_DEQ); 211 } 212 213 /** set a unique name for the queue pair based on its name, dev_id and qp_id */ 214 static int 215 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, 216 struct isal_comp_qp *qp) 217 { 218 unsigned int n = snprintf(qp->name, sizeof(qp->name), 219 "isal_comp_pmd_%u_qp_%u", 220 dev->data->dev_id, qp->id); 221 222 if (n >= sizeof(qp->name)) 223 return -1; 224 225 return 0; 226 } 227 228 /* Setup a queue pair */ 229 static int 230 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 231 uint32_t max_inflight_ops, int socket_id) 232 { 233 struct isal_comp_qp *qp = NULL; 234 int retval; 235 236 /* Free memory prior to re-allocation if needed. */ 237 if (dev->data->queue_pairs[qp_id] != NULL) 238 isal_comp_pmd_qp_release(dev, qp_id); 239 240 /* Allocate the queue pair data structure. */ 241 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), 242 RTE_CACHE_LINE_SIZE, socket_id); 243 if (qp == NULL) { 244 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); 245 return (-ENOMEM); 246 } 247 248 /* Initialize memory for compression stream structure */ 249 qp->stream = rte_zmalloc_socket("Isa-l compression stream ", 250 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, 251 socket_id); 252 253 /* Initialize memory for compression level buffer */ 254 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", 255 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, 256 socket_id); 257 258 /* Initialize memory for decompression state structure */ 259 qp->state = rte_zmalloc_socket("Isa-l decompression state", 260 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, 261 socket_id); 262 263 qp->id = qp_id; 264 dev->data->queue_pairs[qp_id] = qp; 265 266 retval = isal_comp_pmd_qp_set_unique_name(dev, qp); 267 if (retval) { 268 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 269 "compression device"); 270 goto qp_setup_cleanup; 271 } 272 273 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, 274 max_inflight_ops, socket_id); 275 if (qp->processed_pkts == NULL) { 276 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 277 "compression device"); 278 goto qp_setup_cleanup; 279 } 280 281 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); 282 283 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 284 return 0; 285 286 qp_setup_cleanup: 287 if (qp) 288 rte_free(qp); 289 290 return -1; 291 } 292 293 /** Set private xform data*/ 294 static int 295 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, 296 const struct rte_comp_xform *xform, void **priv_xform) 297 { 298 int ret; 299 struct isal_comp_private *internals = dev->data->dev_private; 300 301 if (xform == NULL) { 302 ISAL_PMD_LOG(ERR, "Invalid Xform struct"); 303 return -EINVAL; 304 } 305 306 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { 307 ISAL_PMD_LOG(ERR, 308 "Couldn't get object from private xform mempool"); 309 return -ENOMEM; 310 } 311 312 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); 313 if (ret != 0) { 314 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); 315 316 /* Return private xform to mempool */ 317 rte_mempool_put(internals->priv_xform_mp, priv_xform); 318 return ret; 319 } 320 return 0; 321 } 322 323 /** Clear memory of the private xform so it doesn't leave key material behind */ 324 static int 325 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) 326 { 327 struct isal_comp_private *internals = dev->data->dev_private; 328 329 /* Zero out the whole structure */ 330 if (priv_xform) { 331 memset(priv_xform, 0, sizeof(struct isal_priv_xform)); 332 rte_mempool_put(internals->priv_xform_mp, priv_xform); 333 } 334 return 0; 335 } 336 337 struct rte_compressdev_ops isal_pmd_ops = { 338 .dev_configure = isal_comp_pmd_config, 339 .dev_start = isal_comp_pmd_start, 340 .dev_stop = isal_comp_pmd_stop, 341 .dev_close = isal_comp_pmd_close, 342 343 .stats_get = isal_comp_pmd_stats_get, 344 .stats_reset = isal_comp_pmd_stats_reset, 345 346 .dev_infos_get = isal_comp_pmd_info_get, 347 348 .queue_pair_setup = isal_comp_pmd_qp_setup, 349 .queue_pair_release = isal_comp_pmd_qp_release, 350 351 .private_xform_create = isal_comp_pmd_priv_xform_create, 352 .private_xform_free = isal_comp_pmd_priv_xform_free, 353 }; 354 355 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; 356