1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4
5 #include <dirent.h>
6 #include <fcntl.h>
7 #include <unistd.h>
8 #include <sys/mman.h>
9 #include <libgen.h>
10
11 #include <bus_driver.h>
12 #include <dev_driver.h>
13 #include <rte_devargs.h>
14 #include <rte_eal.h>
15 #include <rte_memory.h>
16 #include <rte_log.h>
17 #include <rte_dmadev_pmd.h>
18 #include <rte_string_fns.h>
19
20 #include "idxd_internal.h"
21
22 /* default value for DSA paths, but allow override in environment for testing */
23 #define DSA_DEV_PATH "/dev/dsa"
24 #define DSA_SYSFS_PATH "/sys/bus/dsa/devices"
25
26 static unsigned int devcount;
27
28 /** unique identifier for a DSA device/WQ instance */
29 struct dsa_wq_addr {
30 uint16_t device_id;
31 uint16_t wq_id;
32 };
33
34 /** a DSA device instance */
35 struct rte_dsa_device {
36 struct rte_device device; /**< Inherit core device */
37 TAILQ_ENTRY(rte_dsa_device) next; /**< next dev in list */
38
39 char wq_name[32]; /**< the workqueue name/number e.g. wq0.1 */
40 struct dsa_wq_addr addr; /**< Identifies the specific WQ */
41 };
42
43 /* forward prototypes */
44 struct dsa_bus;
45 static int dsa_scan(void);
46 static int dsa_probe(void);
47 static struct rte_device *dsa_find_device(const struct rte_device *start,
48 rte_dev_cmp_t cmp, const void *data);
49 static enum rte_iova_mode dsa_get_iommu_class(void);
50 static int dsa_addr_parse(const char *name, void *addr);
51
52 /** List of devices */
53 TAILQ_HEAD(dsa_device_list, rte_dsa_device);
54
55 /**
56 * Structure describing the DSA bus
57 */
58 struct dsa_bus {
59 struct rte_bus bus; /**< Inherit the generic class */
60 struct rte_driver driver; /**< Driver struct for devices to point to */
61 struct dsa_device_list device_list; /**< List of PCI devices */
62 };
63
64 struct dsa_bus dsa_bus = {
65 .bus = {
66 .scan = dsa_scan,
67 .probe = dsa_probe,
68 .find_device = dsa_find_device,
69 .get_iommu_class = dsa_get_iommu_class,
70 .parse = dsa_addr_parse,
71 },
72 .driver = {
73 .name = "dmadev_idxd"
74 },
75 .device_list = TAILQ_HEAD_INITIALIZER(dsa_bus.device_list),
76 };
77
78 static inline const char *
dsa_get_dev_path(void)79 dsa_get_dev_path(void)
80 {
81 const char *path = getenv("DSA_DEV_PATH");
82 return path ? path : DSA_DEV_PATH;
83 }
84
85 static inline const char *
dsa_get_sysfs_path(void)86 dsa_get_sysfs_path(void)
87 {
88 const char *path = getenv("DSA_SYSFS_PATH");
89 return path ? path : DSA_SYSFS_PATH;
90 }
91
92 static int
idxd_dev_close(struct rte_dma_dev * dev)93 idxd_dev_close(struct rte_dma_dev *dev)
94 {
95 struct idxd_dmadev *idxd = dev->data->dev_private;
96 munmap(idxd->portal, 0x1000);
97 return 0;
98 }
99
100 static const struct rte_dma_dev_ops idxd_bus_ops = {
101 .dev_close = idxd_dev_close,
102 .dev_dump = idxd_dump,
103 .dev_configure = idxd_configure,
104 .vchan_setup = idxd_vchan_setup,
105 .dev_info_get = idxd_info_get,
106 .stats_get = idxd_stats_get,
107 .stats_reset = idxd_stats_reset,
108 .vchan_status = idxd_vchan_status,
109 };
110
111 static void *
idxd_bus_mmap_wq(struct rte_dsa_device * dev)112 idxd_bus_mmap_wq(struct rte_dsa_device *dev)
113 {
114 void *addr;
115 char path[PATH_MAX];
116 int fd;
117
118 snprintf(path, sizeof(path), "%s/%s", dsa_get_dev_path(), dev->wq_name);
119 fd = open(path, O_RDWR);
120 if (fd < 0) {
121 IDXD_PMD_ERR("Failed to open device path: %s", path);
122 return NULL;
123 }
124
125 addr = mmap(NULL, 0x1000, PROT_WRITE, MAP_SHARED, fd, 0);
126 close(fd);
127 if (addr == MAP_FAILED) {
128 IDXD_PMD_ERR("Failed to mmap device %s", path);
129 return NULL;
130 }
131
132 return addr;
133 }
134
135 static int
read_wq_string(struct rte_dsa_device * dev,const char * filename,char * value,size_t valuelen)136 read_wq_string(struct rte_dsa_device *dev, const char *filename,
137 char *value, size_t valuelen)
138 {
139 char sysfs_node[PATH_MAX];
140 int len;
141 int fd;
142
143 snprintf(sysfs_node, sizeof(sysfs_node), "%s/%s/%s",
144 dsa_get_sysfs_path(), dev->wq_name, filename);
145 fd = open(sysfs_node, O_RDONLY);
146 if (fd < 0) {
147 IDXD_PMD_ERR("%s(): opening file '%s' failed: %s",
148 __func__, sysfs_node, strerror(errno));
149 return -1;
150 }
151
152 len = read(fd, value, valuelen - 1);
153 close(fd);
154 if (len < 0) {
155 IDXD_PMD_ERR("%s(): error reading file '%s': %s",
156 __func__, sysfs_node, strerror(errno));
157 return -1;
158 }
159 value[len] = '\0';
160 return 0;
161 }
162
163 static int
read_wq_int(struct rte_dsa_device * dev,const char * filename,int * value)164 read_wq_int(struct rte_dsa_device *dev, const char *filename,
165 int *value)
166 {
167 char sysfs_node[PATH_MAX];
168 FILE *f;
169 int ret = 0;
170
171 snprintf(sysfs_node, sizeof(sysfs_node), "%s/%s/%s",
172 dsa_get_sysfs_path(), dev->wq_name, filename);
173 f = fopen(sysfs_node, "r");
174 if (f == NULL) {
175 IDXD_PMD_ERR("%s(): opening file '%s' failed: %s",
176 __func__, sysfs_node, strerror(errno));
177 return -1;
178 }
179
180 if (fscanf(f, "%d", value) != 1) {
181 IDXD_PMD_ERR("%s(): error reading file '%s': %s",
182 __func__, sysfs_node, strerror(errno));
183 ret = -1;
184 }
185
186 fclose(f);
187 return ret;
188 }
189
190 static int
read_device_int(struct rte_dsa_device * dev,const char * filename,int * value)191 read_device_int(struct rte_dsa_device *dev, const char *filename,
192 int *value)
193 {
194 char sysfs_node[PATH_MAX];
195 FILE *f;
196 int ret = 0;
197
198 snprintf(sysfs_node, sizeof(sysfs_node), "%s/dsa%d/%s",
199 dsa_get_sysfs_path(), dev->addr.device_id, filename);
200 f = fopen(sysfs_node, "r");
201 if (f == NULL) {
202 IDXD_PMD_ERR("%s(): opening file '%s' failed: %s",
203 __func__, sysfs_node, strerror(errno));
204 return -1;
205 }
206
207 if (fscanf(f, "%d", value) != 1) {
208 IDXD_PMD_ERR("%s(): error reading file '%s': %s",
209 __func__, sysfs_node, strerror(errno));
210 ret = -1;
211 }
212
213 fclose(f);
214 return ret;
215 }
216
217 static int
idxd_probe_dsa(struct rte_dsa_device * dev)218 idxd_probe_dsa(struct rte_dsa_device *dev)
219 {
220 struct idxd_dmadev idxd = {0};
221 int ret = 0;
222
223 IDXD_PMD_INFO("Probing device %s on numa node %d",
224 dev->wq_name, dev->device.numa_node);
225 if (read_wq_int(dev, "size", &ret) < 0)
226 return -1;
227 idxd.max_batches = ret;
228 if (read_wq_int(dev, "max_batch_size", &ret) < 0)
229 return -1;
230 idxd.max_batch_size = ret;
231 idxd.qid = dev->addr.wq_id;
232 idxd.u.bus.dsa_id = dev->addr.device_id;
233 idxd.sva_support = 1;
234
235 idxd.portal = idxd_bus_mmap_wq(dev);
236 if (idxd.portal == NULL) {
237 IDXD_PMD_ERR("WQ mmap failed");
238 return -ENOENT;
239 }
240
241 ret = idxd_dmadev_create(dev->wq_name, &dev->device, &idxd, &idxd_bus_ops);
242 if (ret) {
243 IDXD_PMD_ERR("Failed to create dmadev %s", dev->wq_name);
244 return ret;
245 }
246
247 return 0;
248 }
249
search_devargs(const char * name)250 static int search_devargs(const char *name)
251 {
252 struct rte_devargs *devargs;
253 RTE_EAL_DEVARGS_FOREACH(dsa_bus.bus.name, devargs) {
254 if (strcmp(devargs->name, name) == 0)
255 return 1;
256 }
257 return 0;
258 }
259
260 static int
is_for_this_process_use(struct rte_dsa_device * dev,const char * name)261 is_for_this_process_use(struct rte_dsa_device *dev, const char *name)
262 {
263 char *runtime_dir = strdup(rte_eal_get_runtime_dir());
264 int retval = 0;
265 int prefixlen;
266 char *prefix;
267
268 if (runtime_dir == NULL)
269 return retval;
270
271 prefix = basename(runtime_dir);
272 prefixlen = strlen(prefix);
273
274 if (strncmp(name, "dpdk_", 5) == 0)
275 retval = 1;
276 if (strncmp(name, prefix, prefixlen) == 0 && name[prefixlen] == '_')
277 retval = 1;
278
279 if (retval && dsa_bus.bus.conf.scan_mode != RTE_BUS_SCAN_UNDEFINED) {
280 if (dsa_bus.bus.conf.scan_mode == RTE_BUS_SCAN_ALLOWLIST)
281 retval = search_devargs(dev->device.name);
282 else
283 retval = !search_devargs(dev->device.name);
284 }
285
286 free(runtime_dir);
287 return retval;
288 }
289
290 static int
dsa_probe(void)291 dsa_probe(void)
292 {
293 struct rte_dsa_device *dev;
294
295 TAILQ_FOREACH(dev, &dsa_bus.device_list, next) {
296 char type[64], name[64];
297
298 if (read_wq_string(dev, "type", type, sizeof(type)) < 0 ||
299 read_wq_string(dev, "name", name, sizeof(name)) < 0)
300 continue;
301
302 if (strncmp(type, "user", 4) == 0 &&
303 is_for_this_process_use(dev, name)) {
304 dev->device.driver = &dsa_bus.driver;
305 idxd_probe_dsa(dev);
306 continue;
307 }
308 IDXD_PMD_DEBUG("WQ '%s', not allocated to DPDK", dev->wq_name);
309 }
310
311 return 0;
312 }
313
314 static int
dsa_scan(void)315 dsa_scan(void)
316 {
317 const char *path = dsa_get_dev_path();
318 struct dirent *wq;
319 DIR *dev_dir;
320
321 dev_dir = opendir(path);
322 if (dev_dir == NULL) {
323 if (errno == ENOENT)
324 return 0; /* no bus, return without error */
325 IDXD_PMD_ERR("%s(): opendir '%s' failed: %s",
326 __func__, path, strerror(errno));
327 return -1;
328 }
329
330 while ((wq = readdir(dev_dir)) != NULL) {
331 struct rte_dsa_device *dev;
332 int numa_node = SOCKET_ID_ANY;
333
334 if (strncmp(wq->d_name, "wq", 2) != 0)
335 continue;
336 if (strnlen(wq->d_name, sizeof(dev->wq_name)) == sizeof(dev->wq_name)) {
337 IDXD_PMD_ERR("%s(): wq name too long: '%s', skipping",
338 __func__, wq->d_name);
339 continue;
340 }
341 IDXD_PMD_DEBUG("%s(): found %s/%s", __func__, path, wq->d_name);
342
343 dev = malloc(sizeof(*dev));
344 if (dev == NULL) {
345 closedir(dev_dir);
346 return -ENOMEM;
347 }
348 if (dsa_addr_parse(wq->d_name, &dev->addr) < 0) {
349 IDXD_PMD_ERR("Error parsing WQ name: %s", wq->d_name);
350 free(dev);
351 continue;
352 }
353 dev->device.bus = &dsa_bus.bus;
354 strlcpy(dev->wq_name, wq->d_name, sizeof(dev->wq_name));
355 TAILQ_INSERT_TAIL(&dsa_bus.device_list, dev, next);
356 devcount++;
357
358 read_device_int(dev, "numa_node", &numa_node);
359 dev->device.numa_node = numa_node;
360 dev->device.name = dev->wq_name;
361 }
362
363 closedir(dev_dir);
364 return 0;
365 }
366
367 static struct rte_device *
dsa_find_device(const struct rte_device * start,rte_dev_cmp_t cmp,const void * data)368 dsa_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
369 const void *data)
370 {
371 struct rte_dsa_device *dev = TAILQ_FIRST(&dsa_bus.device_list);
372
373 /* the rte_device struct must be at start of dsa structure */
374 RTE_BUILD_BUG_ON(offsetof(struct rte_dsa_device, device) != 0);
375
376 if (start != NULL) /* jump to start point if given */
377 dev = TAILQ_NEXT((const struct rte_dsa_device *)start, next);
378 while (dev != NULL) {
379 if (cmp(&dev->device, data) == 0)
380 return &dev->device;
381 dev = TAILQ_NEXT(dev, next);
382 }
383 return NULL;
384 }
385
386 static enum rte_iova_mode
dsa_get_iommu_class(void)387 dsa_get_iommu_class(void)
388 {
389 /* if there are no devices, report don't care, otherwise VA mode */
390 return devcount > 0 ? RTE_IOVA_VA : RTE_IOVA_DC;
391 }
392
393 static int
dsa_addr_parse(const char * name,void * addr)394 dsa_addr_parse(const char *name, void *addr)
395 {
396 struct dsa_wq_addr *wq = addr;
397 unsigned int device_id, wq_id;
398
399 if (sscanf(name, "wq%u.%u", &device_id, &wq_id) != 2) {
400 IDXD_PMD_DEBUG("Parsing WQ name failed: %s", name);
401 return -1;
402 }
403
404 if (wq != NULL) {
405 wq->device_id = device_id;
406 wq->wq_id = wq_id;
407 }
408
409 return 0;
410 }
411
412 RTE_REGISTER_BUS(dsa, dsa_bus.bus);
413