1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4 #include <isa-l.h>
5
6 #include <rte_common.h>
7 #include <rte_cpuflags.h>
8 #include <rte_compressdev_pmd.h>
9 #include <rte_malloc.h>
10
11 #include "isal_compress_pmd_private.h"
12
13 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
14 {
15 .algo = RTE_COMP_ALGO_DEFLATE,
16 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
17 RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
18 RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
19 RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
20 RTE_COMP_FF_HUFFMAN_FIXED |
21 RTE_COMP_FF_HUFFMAN_DYNAMIC |
22 RTE_COMP_FF_CRC32_CHECKSUM |
23 RTE_COMP_FF_ADLER32_CHECKSUM,
24 .window_size = {
25 .min = 15,
26 .max = 15,
27 .increment = 0
28 },
29 },
30 RTE_COMP_END_OF_CAPABILITIES_LIST()
31 };
32
33 /** Configure device */
34 static int
isal_comp_pmd_config(struct rte_compressdev * dev,struct rte_compressdev_config * config)35 isal_comp_pmd_config(struct rte_compressdev *dev,
36 struct rte_compressdev_config *config)
37 {
38 int ret = 0;
39 unsigned int n;
40 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
41 unsigned int elt_size = sizeof(struct isal_priv_xform);
42 struct isal_comp_private *internals = dev->data->dev_private;
43
44 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp",
45 dev->data->dev_id);
46 if (n > sizeof(mp_name)) {
47 ISAL_PMD_LOG(ERR,
48 "Unable to create unique name for xform mempool");
49 return -ENOMEM;
50 }
51
52 internals->priv_xform_mp = rte_mempool_lookup(mp_name);
53
54 if (internals->priv_xform_mp != NULL) {
55 if (((internals->priv_xform_mp)->elt_size != elt_size) ||
56 ((internals->priv_xform_mp)->size <
57 config->max_nb_priv_xforms)) {
58
59 ISAL_PMD_LOG(ERR, "%s mempool already exists with different"
60 " initialization parameters", mp_name);
61 internals->priv_xform_mp = NULL;
62 return -ENOMEM;
63 }
64 } else { /* First time configuration */
65 internals->priv_xform_mp = rte_mempool_create(
66 mp_name, /* mempool name */
67 /* number of elements*/
68 config->max_nb_priv_xforms,
69 elt_size, /* element size*/
70 0, /* Cache size*/
71 0, /* private data size */
72 NULL, /* obj initialization constructor */
73 NULL, /* obj initialization constructor arg */
74 NULL, /**< obj constructor*/
75 NULL, /* obj constructor arg */
76 config->socket_id, /* socket id */
77 0); /* flags */
78 }
79
80 if (internals->priv_xform_mp == NULL) {
81 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name);
82 return -ENOMEM;
83 }
84
85 dev->data->dev_private = internals;
86
87 return ret;
88 }
89
90 /** Start device */
91 static int
isal_comp_pmd_start(__rte_unused struct rte_compressdev * dev)92 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev)
93 {
94 return 0;
95 }
96
97 /** Stop device */
98 static void
isal_comp_pmd_stop(__rte_unused struct rte_compressdev * dev)99 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev)
100 {
101 }
102
103 /** Close device */
104 static int
isal_comp_pmd_close(struct rte_compressdev * dev)105 isal_comp_pmd_close(struct rte_compressdev *dev)
106 {
107 /* Free private data */
108 struct isal_comp_private *internals = dev->data->dev_private;
109
110 rte_mempool_free(internals->priv_xform_mp);
111 return 0;
112 }
113
114 /** Get device statistics */
115 static void
isal_comp_pmd_stats_get(struct rte_compressdev * dev,struct rte_compressdev_stats * stats)116 isal_comp_pmd_stats_get(struct rte_compressdev *dev,
117 struct rte_compressdev_stats *stats)
118 {
119 uint16_t qp_id;
120
121 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
122 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
123
124 stats->enqueued_count += qp->qp_stats.enqueued_count;
125 stats->dequeued_count += qp->qp_stats.dequeued_count;
126
127 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
128 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
129 }
130 }
131
132 /** Get device info */
133 static void
isal_comp_pmd_info_get(struct rte_compressdev * dev __rte_unused,struct rte_compressdev_info * dev_info)134 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused,
135 struct rte_compressdev_info *dev_info)
136 {
137 if (dev_info != NULL) {
138 dev_info->capabilities = isal_pmd_capabilities;
139
140 /* Check CPU for supported vector instruction and set
141 * feature_flags
142 */
143 #if defined(RTE_ARCH_X86)
144 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
145 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512;
146 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
147 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2;
148 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
149 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX;
150 else
151 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE;
152 #elif defined(RTE_ARCH_ARM)
153 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))
154 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_NEON;
155 #endif
156 }
157 }
158
159 /** Reset device statistics */
160 static void
isal_comp_pmd_stats_reset(struct rte_compressdev * dev)161 isal_comp_pmd_stats_reset(struct rte_compressdev *dev)
162 {
163 uint16_t qp_id;
164
165 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
166 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
167 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
168 }
169 }
170
171 /** Release queue pair */
172 static int
isal_comp_pmd_qp_release(struct rte_compressdev * dev,uint16_t qp_id)173 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
174 {
175 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
176
177 if (qp == NULL)
178 return -EINVAL;
179
180 if (qp->stream)
181 rte_free(qp->stream->level_buf);
182
183 rte_free(qp->state);
184 rte_ring_free(qp->processed_pkts);
185 rte_free(qp->stream);
186 rte_free(qp);
187 dev->data->queue_pairs[qp_id] = NULL;
188
189 return 0;
190 }
191
192 /** Create a ring to place process packets on */
193 static struct rte_ring *
isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp * qp,unsigned int ring_size,int socket_id)194 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp,
195 unsigned int ring_size, int socket_id)
196 {
197 struct rte_ring *r;
198
199 r = rte_ring_lookup(qp->name);
200 if (r) {
201 if (rte_ring_get_size(r) >= ring_size) {
202 ISAL_PMD_LOG(DEBUG,
203 "Reusing existing ring %s for processed packets",
204 qp->name);
205 return r;
206 }
207
208 ISAL_PMD_LOG(ERR,
209 "Unable to reuse existing ring %s"
210 " for processed packets",
211 qp->name);
212 return NULL;
213 }
214
215 return rte_ring_create(qp->name, ring_size, socket_id,
216 RING_F_SP_ENQ | RING_F_SC_DEQ);
217 }
218
219 /** set a unique name for the queue pair based on its name, dev_id and qp_id */
220 static int
isal_comp_pmd_qp_set_unique_name(struct rte_compressdev * dev,struct isal_comp_qp * qp)221 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev,
222 struct isal_comp_qp *qp)
223 {
224 unsigned int n = snprintf(qp->name, sizeof(qp->name),
225 "isal_comp_pmd_%u_qp_%u",
226 dev->data->dev_id, qp->id);
227
228 if (n >= sizeof(qp->name))
229 return -1;
230
231 return 0;
232 }
233
234 /* Setup a queue pair */
235 static int
isal_comp_pmd_qp_setup(struct rte_compressdev * dev,uint16_t qp_id,uint32_t max_inflight_ops,int socket_id)236 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
237 uint32_t max_inflight_ops, int socket_id)
238 {
239 struct isal_comp_qp *qp = NULL;
240 int retval;
241
242 /* Free memory prior to re-allocation if needed. */
243 if (dev->data->queue_pairs[qp_id] != NULL)
244 isal_comp_pmd_qp_release(dev, qp_id);
245
246 /* Allocate the queue pair data structure. */
247 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp),
248 RTE_CACHE_LINE_SIZE, socket_id);
249 if (qp == NULL) {
250 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory");
251 return (-ENOMEM);
252 }
253
254 /* Initialize memory for compression stream structure */
255 qp->stream = rte_zmalloc_socket("Isa-l compression stream ",
256 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE,
257 socket_id);
258 if (qp->stream == NULL) {
259 ISAL_PMD_LOG(ERR, "Failed to allocate compression stream memory");
260 goto qp_setup_cleanup;
261 }
262 /* Initialize memory for compression level buffer */
263 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf",
264 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE,
265 socket_id);
266 if (qp->stream->level_buf == NULL) {
267 ISAL_PMD_LOG(ERR, "Failed to allocate compression level_buf memory");
268 goto qp_setup_cleanup;
269 }
270
271 /* Initialize memory for decompression state structure */
272 qp->state = rte_zmalloc_socket("Isa-l decompression state",
273 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE,
274 socket_id);
275 if (qp->state == NULL) {
276 ISAL_PMD_LOG(ERR, "Failed to allocate decompression state memory");
277 goto qp_setup_cleanup;
278 }
279
280 qp->id = qp_id;
281 dev->data->queue_pairs[qp_id] = qp;
282
283 retval = isal_comp_pmd_qp_set_unique_name(dev, qp);
284 if (retval) {
285 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
286 "compression device");
287 goto qp_setup_cleanup;
288 }
289
290 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp,
291 max_inflight_ops, socket_id);
292 if (qp->processed_pkts == NULL) {
293 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
294 "compression device");
295 goto qp_setup_cleanup;
296 }
297
298 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts);
299
300 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
301 return 0;
302
303 qp_setup_cleanup:
304 if (qp->stream)
305 rte_free(qp->stream->level_buf);
306 rte_free(qp->stream);
307 rte_free(qp->state);
308 rte_free(qp);
309
310 return -1;
311 }
312
313 /** Set private xform data*/
314 static int
isal_comp_pmd_priv_xform_create(struct rte_compressdev * dev,const struct rte_comp_xform * xform,void ** priv_xform)315 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev,
316 const struct rte_comp_xform *xform, void **priv_xform)
317 {
318 int ret;
319 struct isal_comp_private *internals = dev->data->dev_private;
320
321 if (xform == NULL) {
322 ISAL_PMD_LOG(ERR, "Invalid Xform struct");
323 return -EINVAL;
324 }
325
326 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) {
327 ISAL_PMD_LOG(ERR,
328 "Couldn't get object from private xform mempool");
329 return -ENOMEM;
330 }
331
332 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform);
333 if (ret != 0) {
334 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters");
335
336 /* Return private xform to mempool */
337 rte_mempool_put(internals->priv_xform_mp, priv_xform);
338 return ret;
339 }
340 return 0;
341 }
342
343 /** Clear memory of the private xform so it doesn't leave key material behind */
344 static int
isal_comp_pmd_priv_xform_free(struct rte_compressdev * dev,void * priv_xform)345 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform)
346 {
347 struct isal_comp_private *internals = dev->data->dev_private;
348
349 /* Zero out the whole structure */
350 if (priv_xform) {
351 memset(priv_xform, 0, sizeof(struct isal_priv_xform));
352 rte_mempool_put(internals->priv_xform_mp, priv_xform);
353 }
354 return 0;
355 }
356
357 struct rte_compressdev_ops isal_pmd_ops = {
358 .dev_configure = isal_comp_pmd_config,
359 .dev_start = isal_comp_pmd_start,
360 .dev_stop = isal_comp_pmd_stop,
361 .dev_close = isal_comp_pmd_close,
362
363 .stats_get = isal_comp_pmd_stats_get,
364 .stats_reset = isal_comp_pmd_stats_reset,
365
366 .dev_infos_get = isal_comp_pmd_info_get,
367
368 .queue_pair_setup = isal_comp_pmd_qp_setup,
369 .queue_pair_release = isal_comp_pmd_qp_release,
370
371 .private_xform_create = isal_comp_pmd_priv_xform_create,
372 .private_xform_free = isal_comp_pmd_priv_xform_free,
373 };
374
375 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops;
376