1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <isa-l.h> 5 6 #include <rte_common.h> 7 #include <rte_cpuflags.h> 8 #include <rte_compressdev_pmd.h> 9 #include <rte_malloc.h> 10 11 #include "isal_compress_pmd_private.h" 12 13 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = { 14 { 15 .algo = RTE_COMP_ALGO_DEFLATE, 16 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | 17 RTE_COMP_FF_OOP_SGL_IN_LB_OUT | 18 RTE_COMP_FF_OOP_LB_IN_SGL_OUT | 19 RTE_COMP_FF_SHAREABLE_PRIV_XFORM | 20 RTE_COMP_FF_HUFFMAN_FIXED | 21 RTE_COMP_FF_HUFFMAN_DYNAMIC | 22 RTE_COMP_FF_CRC32_CHECKSUM | 23 RTE_COMP_FF_ADLER32_CHECKSUM, 24 .window_size = { 25 .min = 15, 26 .max = 15, 27 .increment = 0 28 }, 29 }, 30 RTE_COMP_END_OF_CAPABILITIES_LIST() 31 }; 32 33 /** Configure device */ 34 static int 35 isal_comp_pmd_config(struct rte_compressdev *dev, 36 struct rte_compressdev_config *config) 37 { 38 int ret = 0; 39 unsigned int n; 40 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; 41 unsigned int elt_size = sizeof(struct isal_priv_xform); 42 struct isal_comp_private *internals = dev->data->dev_private; 43 44 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp", 45 dev->data->dev_id); 46 if (n > sizeof(mp_name)) { 47 ISAL_PMD_LOG(ERR, 48 "Unable to create unique name for xform mempool"); 49 return -ENOMEM; 50 } 51 52 internals->priv_xform_mp = rte_mempool_lookup(mp_name); 53 54 if (internals->priv_xform_mp != NULL) { 55 if (((internals->priv_xform_mp)->elt_size != elt_size) || 56 ((internals->priv_xform_mp)->size < 57 config->max_nb_priv_xforms)) { 58 59 ISAL_PMD_LOG(ERR, "%s mempool already exists with different" 60 " initialization parameters", mp_name); 61 internals->priv_xform_mp = NULL; 62 return -ENOMEM; 63 } 64 } else { /* First time configuration */ 65 internals->priv_xform_mp = rte_mempool_create( 66 mp_name, /* mempool name */ 67 /* number of elements*/ 68 config->max_nb_priv_xforms, 69 elt_size, /* element size*/ 70 0, /* Cache size*/ 71 0, /* private data size */ 72 NULL, /* obj initialization constructor */ 73 NULL, /* obj initialization constructor arg */ 74 NULL, /**< obj constructor*/ 75 NULL, /* obj constructor arg */ 76 config->socket_id, /* socket id */ 77 0); /* flags */ 78 } 79 80 if (internals->priv_xform_mp == NULL) { 81 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name); 82 return -ENOMEM; 83 } 84 85 dev->data->dev_private = internals; 86 87 return ret; 88 } 89 90 /** Start device */ 91 static int 92 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev) 93 { 94 return 0; 95 } 96 97 /** Stop device */ 98 static void 99 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev) 100 { 101 } 102 103 /** Close device */ 104 static int 105 isal_comp_pmd_close(struct rte_compressdev *dev) 106 { 107 /* Free private data */ 108 struct isal_comp_private *internals = dev->data->dev_private; 109 110 rte_mempool_free(internals->priv_xform_mp); 111 return 0; 112 } 113 114 /** Get device statistics */ 115 static void 116 isal_comp_pmd_stats_get(struct rte_compressdev *dev, 117 struct rte_compressdev_stats *stats) 118 { 119 uint16_t qp_id; 120 121 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 122 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 123 124 stats->enqueued_count += qp->qp_stats.enqueued_count; 125 stats->dequeued_count += qp->qp_stats.dequeued_count; 126 127 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; 128 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; 129 } 130 } 131 132 /** Get device info */ 133 static void 134 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused, 135 struct rte_compressdev_info *dev_info) 136 { 137 if (dev_info != NULL) { 138 dev_info->capabilities = isal_pmd_capabilities; 139 140 /* Check CPU for supported vector instruction and set 141 * feature_flags 142 */ 143 #if defined(RTE_ARCH_X86) 144 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) 145 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512; 146 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) 147 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2; 148 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) 149 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX; 150 else 151 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE; 152 #elif defined(RTE_ARCH_ARM) 153 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) 154 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_NEON; 155 #endif 156 } 157 } 158 159 /** Reset device statistics */ 160 static void 161 isal_comp_pmd_stats_reset(struct rte_compressdev *dev) 162 { 163 uint16_t qp_id; 164 165 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 166 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 167 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 168 } 169 } 170 171 /** Release queue pair */ 172 static int 173 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 174 { 175 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id]; 176 177 if (qp == NULL) 178 return -EINVAL; 179 180 if (qp->stream) 181 rte_free(qp->stream->level_buf); 182 183 rte_free(qp->state); 184 rte_ring_free(qp->processed_pkts); 185 rte_free(qp->stream); 186 rte_free(qp); 187 dev->data->queue_pairs[qp_id] = NULL; 188 189 return 0; 190 } 191 192 /** Create a ring to place process packets on */ 193 static struct rte_ring * 194 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp, 195 unsigned int ring_size, int socket_id) 196 { 197 struct rte_ring *r; 198 199 r = rte_ring_lookup(qp->name); 200 if (r) { 201 if (rte_ring_get_size(r) >= ring_size) { 202 ISAL_PMD_LOG(DEBUG, 203 "Reusing existing ring %s for processed packets", 204 qp->name); 205 return r; 206 } 207 208 ISAL_PMD_LOG(ERR, 209 "Unable to reuse existing ring %s" 210 " for processed packets", 211 qp->name); 212 return NULL; 213 } 214 215 return rte_ring_create(qp->name, ring_size, socket_id, 216 RING_F_SP_ENQ | RING_F_SC_DEQ); 217 } 218 219 /** set a unique name for the queue pair based on its name, dev_id and qp_id */ 220 static int 221 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev, 222 struct isal_comp_qp *qp) 223 { 224 unsigned int n = snprintf(qp->name, sizeof(qp->name), 225 "isal_comp_pmd_%u_qp_%u", 226 dev->data->dev_id, qp->id); 227 228 if (n >= sizeof(qp->name)) 229 return -1; 230 231 return 0; 232 } 233 234 /* Setup a queue pair */ 235 static int 236 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 237 uint32_t max_inflight_ops, int socket_id) 238 { 239 struct isal_comp_qp *qp = NULL; 240 int retval; 241 242 /* Free memory prior to re-allocation if needed. */ 243 if (dev->data->queue_pairs[qp_id] != NULL) 244 isal_comp_pmd_qp_release(dev, qp_id); 245 246 /* Allocate the queue pair data structure. */ 247 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp), 248 RTE_CACHE_LINE_SIZE, socket_id); 249 if (qp == NULL) { 250 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory"); 251 return (-ENOMEM); 252 } 253 254 /* Initialize memory for compression stream structure */ 255 qp->stream = rte_zmalloc_socket("Isa-l compression stream ", 256 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE, 257 socket_id); 258 if (qp->stream == NULL) { 259 ISAL_PMD_LOG(ERR, "Failed to allocate compression stream memory"); 260 goto qp_setup_cleanup; 261 } 262 /* Initialize memory for compression level buffer */ 263 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf", 264 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE, 265 socket_id); 266 if (qp->stream->level_buf == NULL) { 267 ISAL_PMD_LOG(ERR, "Failed to allocate compression level_buf memory"); 268 goto qp_setup_cleanup; 269 } 270 271 /* Initialize memory for decompression state structure */ 272 qp->state = rte_zmalloc_socket("Isa-l decompression state", 273 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE, 274 socket_id); 275 if (qp->state == NULL) { 276 ISAL_PMD_LOG(ERR, "Failed to allocate decompression state memory"); 277 goto qp_setup_cleanup; 278 } 279 280 qp->id = qp_id; 281 dev->data->queue_pairs[qp_id] = qp; 282 283 retval = isal_comp_pmd_qp_set_unique_name(dev, qp); 284 if (retval) { 285 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 286 "compression device"); 287 goto qp_setup_cleanup; 288 } 289 290 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp, 291 max_inflight_ops, socket_id); 292 if (qp->processed_pkts == NULL) { 293 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal " 294 "compression device"); 295 goto qp_setup_cleanup; 296 } 297 298 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts); 299 300 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); 301 return 0; 302 303 qp_setup_cleanup: 304 if (qp->stream) 305 rte_free(qp->stream->level_buf); 306 rte_free(qp->stream); 307 rte_free(qp->state); 308 rte_free(qp); 309 310 return -1; 311 } 312 313 /** Set private xform data*/ 314 static int 315 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev, 316 const struct rte_comp_xform *xform, void **priv_xform) 317 { 318 int ret; 319 struct isal_comp_private *internals = dev->data->dev_private; 320 321 if (xform == NULL) { 322 ISAL_PMD_LOG(ERR, "Invalid Xform struct"); 323 return -EINVAL; 324 } 325 326 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) { 327 ISAL_PMD_LOG(ERR, 328 "Couldn't get object from private xform mempool"); 329 return -ENOMEM; 330 } 331 332 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform); 333 if (ret != 0) { 334 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters"); 335 336 /* Return private xform to mempool */ 337 rte_mempool_put(internals->priv_xform_mp, priv_xform); 338 return ret; 339 } 340 return 0; 341 } 342 343 /** Clear memory of the private xform so it doesn't leave key material behind */ 344 static int 345 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform) 346 { 347 struct isal_comp_private *internals = dev->data->dev_private; 348 349 /* Zero out the whole structure */ 350 if (priv_xform) { 351 memset(priv_xform, 0, sizeof(struct isal_priv_xform)); 352 rte_mempool_put(internals->priv_xform_mp, priv_xform); 353 } 354 return 0; 355 } 356 357 struct rte_compressdev_ops isal_pmd_ops = { 358 .dev_configure = isal_comp_pmd_config, 359 .dev_start = isal_comp_pmd_start, 360 .dev_stop = isal_comp_pmd_stop, 361 .dev_close = isal_comp_pmd_close, 362 363 .stats_get = isal_comp_pmd_stats_get, 364 .stats_reset = isal_comp_pmd_stats_reset, 365 366 .dev_infos_get = isal_comp_pmd_info_get, 367 368 .queue_pair_setup = isal_comp_pmd_qp_setup, 369 .queue_pair_release = isal_comp_pmd_qp_release, 370 371 .private_xform_create = isal_comp_pmd_priv_xform_create, 372 .private_xform_free = isal_comp_pmd_priv_xform_free, 373 }; 374 375 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops; 376