1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2022 Intel Corporation
3 */
4
5 #include <errno.h>
6 #include <stdio.h>
7 #include <stdint.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <poll.h>
13 #include <sys/eventfd.h>
14
15 #include <rte_eal.h>
16 #include <rte_malloc.h>
17 #include <rte_memzone.h>
18 #include <rte_rawdev_pmd.h>
19
20 #include "afu_pmd_core.h"
21
22 static struct rte_afu_uuid afu_pmd_uuid_map[AFU_RAWDEV_MAX_DRVS+1];
23 TAILQ_HEAD(afu_drv_list, afu_rawdev_drv);
24 static struct afu_drv_list afu_pmd_list = TAILQ_HEAD_INITIALIZER(afu_pmd_list);
25
26 #define afu_rawdev_trylock(dev) rte_spinlock_trylock(&dev->sd->lock)
27 #define afu_rawdev_unlock(dev) rte_spinlock_unlock(&dev->sd->lock)
28
afu_rawdev_configure(const struct rte_rawdev * rawdev,rte_rawdev_obj_t config,size_t config_size)29 static int afu_rawdev_configure(const struct rte_rawdev *rawdev,
30 rte_rawdev_obj_t config, size_t config_size)
31 {
32 struct afu_rawdev *dev = NULL;
33 int ret = 0;
34
35 IFPGA_RAWDEV_PMD_FUNC_TRACE();
36
37 dev = afu_rawdev_get_priv(rawdev);
38 if (!dev)
39 return -ENODEV;
40
41 if (dev->ops && dev->ops->config)
42 ret = (*dev->ops->config)(dev, config, config_size);
43
44 return ret;
45 }
46
afu_rawdev_start(struct rte_rawdev * rawdev)47 static int afu_rawdev_start(struct rte_rawdev *rawdev)
48 {
49 struct afu_rawdev *dev = NULL;
50 int ret = 0;
51
52 IFPGA_RAWDEV_PMD_FUNC_TRACE();
53
54 dev = afu_rawdev_get_priv(rawdev);
55 if (!dev)
56 return -ENODEV;
57
58 ret = afu_rawdev_trylock(dev);
59 if (!ret) {
60 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please start it later");
61 return ret;
62 }
63
64 if (dev->ops && dev->ops->start)
65 ret = (*dev->ops->start)(dev);
66
67 afu_rawdev_unlock(dev);
68
69 return ret;
70 }
71
afu_rawdev_stop(struct rte_rawdev * rawdev)72 static void afu_rawdev_stop(struct rte_rawdev *rawdev)
73 {
74 struct afu_rawdev *dev = NULL;
75 int ret = 0;
76
77 IFPGA_RAWDEV_PMD_FUNC_TRACE();
78
79 dev = afu_rawdev_get_priv(rawdev);
80 if (!dev)
81 return;
82
83 ret = afu_rawdev_trylock(dev);
84 if (!ret) {
85 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please stop it later");
86 return;
87 }
88
89 if (dev->ops && dev->ops->stop)
90 ret = (*dev->ops->stop)(dev);
91
92 afu_rawdev_unlock(dev);
93 }
94
afu_rawdev_close(struct rte_rawdev * rawdev)95 static int afu_rawdev_close(struct rte_rawdev *rawdev)
96 {
97 struct afu_rawdev *dev = NULL;
98 int ret = 0;
99
100 IFPGA_RAWDEV_PMD_FUNC_TRACE();
101
102 dev = afu_rawdev_get_priv(rawdev);
103 if (!dev)
104 return -ENODEV;
105
106 if (dev->ops && dev->ops->close)
107 ret = (*dev->ops->close)(dev);
108
109 return ret;
110 }
111
afu_rawdev_reset(struct rte_rawdev * rawdev)112 static int afu_rawdev_reset(struct rte_rawdev *rawdev)
113 {
114 struct afu_rawdev *dev = NULL;
115 int ret = 0;
116
117 IFPGA_RAWDEV_PMD_FUNC_TRACE();
118
119 dev = afu_rawdev_get_priv(rawdev);
120 if (!dev)
121 return -ENODEV;
122
123 ret = afu_rawdev_trylock(dev);
124 if (!ret) {
125 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please reset it later");
126 return ret;
127 }
128
129 if (dev->ops && dev->ops->reset)
130 ret = (*dev->ops->reset)(dev);
131
132 afu_rawdev_unlock(dev);
133
134 return ret;
135 }
136
afu_rawdev_selftest(uint16_t dev_id)137 static int afu_rawdev_selftest(uint16_t dev_id)
138 {
139 struct afu_rawdev *dev = NULL;
140 int ret = 0;
141
142 IFPGA_RAWDEV_PMD_FUNC_TRACE();
143
144 if (!rte_rawdev_pmd_is_valid_dev(dev_id))
145 return -ENODEV;
146
147 dev = afu_rawdev_get_priv(&rte_rawdevs[dev_id]);
148 if (!dev)
149 return -ENOENT;
150
151 ret = afu_rawdev_trylock(dev);
152 if (!ret) {
153 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please test it later");
154 return ret;
155 }
156
157 if (dev->ops && dev->ops->test)
158 ret = (*dev->ops->test)(dev);
159
160 afu_rawdev_unlock(dev);
161
162 return ret;
163 }
164
afu_rawdev_dump(struct rte_rawdev * rawdev,FILE * f)165 static int afu_rawdev_dump(struct rte_rawdev *rawdev, FILE *f)
166 {
167 struct afu_rawdev *dev = NULL;
168 int ret = 0;
169
170 IFPGA_RAWDEV_PMD_FUNC_TRACE();
171
172 dev = afu_rawdev_get_priv(rawdev);
173 if (!dev)
174 return -ENODEV;
175
176 if (dev->ops && dev->ops->dump)
177 ret = (*dev->ops->dump)(dev, f);
178
179 return ret;
180 }
181
182 static const struct rte_rawdev_ops afu_rawdev_ops = {
183 .dev_info_get = NULL,
184 .dev_configure = afu_rawdev_configure,
185 .dev_start = afu_rawdev_start,
186 .dev_stop = afu_rawdev_stop,
187 .dev_close = afu_rawdev_close,
188 .dev_reset = afu_rawdev_reset,
189
190 .queue_def_conf = NULL,
191 .queue_setup = NULL,
192 .queue_release = NULL,
193 .queue_count = NULL,
194
195 .attr_get = NULL,
196 .attr_set = NULL,
197
198 .enqueue_bufs = NULL,
199 .dequeue_bufs = NULL,
200
201 .dump = afu_rawdev_dump,
202
203 .xstats_get = NULL,
204 .xstats_get_names = NULL,
205 .xstats_get_by_name = NULL,
206 .xstats_reset = NULL,
207
208 .firmware_status_get = NULL,
209 .firmware_version_get = NULL,
210 .firmware_load = NULL,
211 .firmware_unload = NULL,
212
213 .dev_selftest = afu_rawdev_selftest,
214 };
215
afu_shared_data_alloc(const char * name,struct afu_shared_data ** data,int socket_id)216 static int afu_shared_data_alloc(const char *name,
217 struct afu_shared_data **data, int socket_id)
218 {
219 const struct rte_memzone *mz;
220 char mz_name[RTE_MEMZONE_NAMESIZE];
221 struct afu_shared_data *sd = NULL;
222 int init_mz = 0;
223
224 if (!name || !data)
225 return -EINVAL;
226
227 /* name format is afu_?|??:??.? which is unique */
228 snprintf(mz_name, sizeof(mz_name), "%s", name);
229
230 mz = rte_memzone_lookup(mz_name);
231 if (!mz) {
232 mz = rte_memzone_reserve(mz_name, sizeof(struct afu_shared_data),
233 socket_id, 0);
234 init_mz = 1;
235 }
236
237 if (!mz) {
238 IFPGA_RAWDEV_PMD_ERR("Allocate memory zone %s failed!",
239 mz_name);
240 return -ENOMEM;
241 }
242
243 sd = (struct afu_shared_data *)mz->addr;
244
245 if (init_mz) /* initialize memory zone on the first time */
246 rte_spinlock_init(&sd->lock);
247
248 *data = sd;
249
250 return 0;
251 }
252
afu_rawdev_name_get(struct rte_afu_device * afu_dev,char * name,size_t size)253 static int afu_rawdev_name_get(struct rte_afu_device *afu_dev, char *name,
254 size_t size)
255 {
256 int n = 0;
257
258 if (!afu_dev || !name || !size)
259 return -EINVAL;
260
261 n = snprintf(name, size, "afu_%s", afu_dev->device.name);
262 if (n >= (int)size) {
263 IFPGA_RAWDEV_PMD_ERR("Name of AFU device is too long!");
264 return -ENAMETOOLONG;
265 }
266
267 return 0;
268 }
269
afu_ops_get(struct rte_afu_uuid * afu_id)270 static struct afu_ops *afu_ops_get(struct rte_afu_uuid *afu_id)
271 {
272 struct afu_rawdev_drv *drv = NULL;
273
274 if (!afu_id)
275 return NULL;
276
277 TAILQ_FOREACH(drv, &afu_pmd_list, next) {
278 if ((drv->uuid.uuid_low == afu_id->uuid_low) &&
279 (drv->uuid.uuid_high == afu_id->uuid_high))
280 break;
281 }
282
283 return drv ? drv->ops : NULL;
284 }
285
afu_rawdev_create(struct rte_afu_device * afu_dev,int socket_id)286 static int afu_rawdev_create(struct rte_afu_device *afu_dev, int socket_id)
287 {
288 struct rte_rawdev *rawdev = NULL;
289 struct afu_rawdev *dev = NULL;
290 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
291 int ret = 0;
292
293 if (!afu_dev)
294 return -EINVAL;
295
296 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name));
297 if (ret)
298 return ret;
299
300 IFPGA_RAWDEV_PMD_INFO("Create raw device %s on NUMA node %d",
301 name, socket_id);
302
303 /* Allocate device structure */
304 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct afu_rawdev),
305 socket_id);
306 if (!rawdev) {
307 IFPGA_RAWDEV_PMD_ERR("Unable to allocate raw device");
308 return -ENOMEM;
309 }
310
311 rawdev->dev_ops = &afu_rawdev_ops;
312 rawdev->device = &afu_dev->device;
313 rawdev->driver_name = afu_dev->driver->driver.name;
314
315 dev = afu_rawdev_get_priv(rawdev);
316 if (!dev)
317 goto cleanup;
318
319 dev->rawdev = rawdev;
320 dev->port = afu_dev->id.port;
321 dev->addr = afu_dev->mem_resource[0].addr;
322 dev->ops = afu_ops_get(&afu_dev->id.uuid);
323 if (dev->ops == NULL) {
324 IFPGA_RAWDEV_PMD_ERR("Unsupported AFU device");
325 goto cleanup;
326 }
327
328 if (dev->ops->init) {
329 ret = (*dev->ops->init)(dev);
330 if (ret) {
331 IFPGA_RAWDEV_PMD_ERR("Failed to init %s", name);
332 goto cleanup;
333 }
334 }
335
336 ret = afu_shared_data_alloc(name, &dev->sd, socket_id);
337 if (ret)
338 goto cleanup;
339
340 return ret;
341
342 cleanup:
343 rte_rawdev_pmd_release(rawdev);
344 return ret;
345 }
346
afu_rawdev_destroy(struct rte_afu_device * afu_dev)347 static int afu_rawdev_destroy(struct rte_afu_device *afu_dev)
348 {
349 struct rte_rawdev *rawdev = NULL;
350 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
351 int ret = 0;
352
353 if (!afu_dev)
354 return -EINVAL;
355
356 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name));
357 if (ret)
358 return ret;
359
360 IFPGA_RAWDEV_PMD_INFO("Destroy raw device %s", name);
361
362 rawdev = rte_rawdev_pmd_get_named_dev(name);
363 if (!rawdev) {
364 IFPGA_RAWDEV_PMD_ERR("Raw device %s not found", name);
365 return -EINVAL;
366 }
367
368 /* rte_rawdev_close is called by pmd_release */
369 ret = rte_rawdev_pmd_release(rawdev);
370 if (ret)
371 IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
372
373 return 0;
374 }
375
afu_rawdev_probe(struct rte_afu_device * afu_dev)376 static int afu_rawdev_probe(struct rte_afu_device *afu_dev)
377 {
378 IFPGA_RAWDEV_PMD_FUNC_TRACE();
379 return afu_rawdev_create(afu_dev, rte_socket_id());
380 }
381
afu_rawdev_remove(struct rte_afu_device * afu_dev)382 static int afu_rawdev_remove(struct rte_afu_device *afu_dev)
383 {
384 IFPGA_RAWDEV_PMD_FUNC_TRACE();
385 return afu_rawdev_destroy(afu_dev);
386 }
387
388 static struct rte_afu_driver afu_pmd = {
389 .id_table = afu_pmd_uuid_map,
390 .probe = afu_rawdev_probe,
391 .remove = afu_rawdev_remove
392 };
393
394 RTE_PMD_REGISTER_AFU(afu_rawdev_driver, afu_pmd);
395
update_uuid_map(void)396 static void update_uuid_map(void)
397 {
398 int i = 0;
399 struct rte_afu_uuid *afu_id = afu_pmd_uuid_map;
400 struct afu_rawdev_drv *drv;
401
402 TAILQ_FOREACH(drv, &afu_pmd_list, next) {
403 if (i++ < AFU_RAWDEV_MAX_DRVS) {
404 afu_id->uuid_low = drv->uuid.uuid_low;
405 afu_id->uuid_high = drv->uuid.uuid_high;
406 afu_id++;
407 }
408 }
409 if (i <= AFU_RAWDEV_MAX_DRVS) {
410 afu_id->uuid_low = 0;
411 afu_id->uuid_high = 0;
412 }
413 }
414
afu_pmd_register(struct afu_rawdev_drv * driver)415 void afu_pmd_register(struct afu_rawdev_drv *driver)
416 {
417 TAILQ_INSERT_TAIL(&afu_pmd_list, driver, next);
418 update_uuid_map();
419 }
420
afu_pmd_unregister(struct afu_rawdev_drv * driver)421 void afu_pmd_unregister(struct afu_rawdev_drv *driver)
422 {
423 TAILQ_REMOVE(&afu_pmd_list, driver, next);
424 update_uuid_map();
425 }
426