xref: /spdk/lib/env_dpdk/pci.c (revision b58a5d73ef70b9458ef589f5858c2a72c1f8e47c) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "env_internal.h"
35 
36 #include "spdk/env.h"
37 
38 #define SYSFS_PCI_DRIVERS	"/sys/bus/pci/drivers"
39 
40 #define PCI_CFG_SIZE		256
41 #define PCI_EXT_CAP_ID_SN	0x03
42 
43 int
44 spdk_pci_device_init(struct rte_pci_driver *driver,
45 		     struct rte_pci_device *device)
46 {
47 	struct spdk_pci_enum_ctx *ctx = (struct spdk_pci_enum_ctx *)driver;
48 
49 	if (!ctx->cb_fn) {
50 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
51 		rte_eal_pci_unmap_device(device);
52 #endif
53 
54 		/* Return a positive value to indicate that this device does not belong to this driver, but
55 		 * this isn't an error. */
56 		return 1;
57 	}
58 
59 	if (device->kdrv == RTE_KDRV_VFIO) {
60 		/*
61 		 * TODO: This is a workaround for an issue where the device is not ready after VFIO reset.
62 		 * Figure out what is actually going wrong and remove this sleep.
63 		 */
64 		usleep(500 * 1000);
65 	}
66 
67 	return ctx->cb_fn(ctx->cb_arg, (struct spdk_pci_device *)device);
68 }
69 
70 int
71 spdk_pci_device_fini(struct rte_pci_device *device)
72 {
73 	return 0;
74 }
75 
76 void
77 spdk_pci_device_detach(struct spdk_pci_device *device)
78 {
79 	struct rte_pci_addr	addr;
80 
81 	addr.domain = device->addr.domain;
82 	addr.bus = device->addr.bus;
83 	addr.devid = device->addr.devid;
84 	addr.function = device->addr.function;
85 
86 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
87 	rte_eal_device_remove(&device->device);
88 #endif
89 	rte_eal_pci_detach(&addr);
90 	/* This will not actually load any drivers because our
91 	 * callback isn't set, but it will re-add the device
92 	 * to DPDK's internal list.
93 	 */
94 	rte_eal_pci_probe_one(&addr);
95 }
96 
97 int
98 spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx,
99 		       spdk_pci_enum_cb enum_cb,
100 		       void *enum_ctx, struct spdk_pci_addr *pci_address)
101 {
102 	struct rte_pci_addr		addr;
103 
104 	addr.domain = pci_address->domain;
105 	addr.bus = pci_address->bus;
106 	addr.devid = pci_address->dev;
107 	addr.function = pci_address->func;
108 
109 	pthread_mutex_lock(&ctx->mtx);
110 
111 	if (!ctx->is_registered) {
112 		ctx->is_registered = true;
113 		rte_eal_pci_register(&ctx->driver);
114 	}
115 
116 	ctx->cb_fn = enum_cb;
117 	ctx->cb_arg = enum_ctx;
118 
119 	if (rte_eal_pci_probe_one(&addr) != 0) {
120 		ctx->cb_arg = NULL;
121 		ctx->cb_fn = NULL;
122 		pthread_mutex_unlock(&ctx->mtx);
123 		return -1;
124 	}
125 
126 	ctx->cb_arg = NULL;
127 	ctx->cb_fn = NULL;
128 	pthread_mutex_unlock(&ctx->mtx);
129 
130 	return 0;
131 }
132 
133 /* Note: You can call spdk_pci_enumerate from more than one thread
134  *       simultaneously safely, but you cannot call spdk_pci_enumerate
135  *       and rte_eal_pci_probe simultaneously.
136 */
137 int
138 spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx,
139 		   spdk_pci_enum_cb enum_cb,
140 		   void *enum_ctx)
141 {
142 	pthread_mutex_lock(&ctx->mtx);
143 
144 	if (!ctx->is_registered) {
145 		ctx->is_registered = true;
146 		rte_eal_pci_register(&ctx->driver);
147 	}
148 
149 	ctx->cb_fn = enum_cb;
150 	ctx->cb_arg = enum_ctx;
151 
152 	if (rte_eal_pci_probe() != 0) {
153 		ctx->cb_arg = NULL;
154 		ctx->cb_fn = NULL;
155 		pthread_mutex_unlock(&ctx->mtx);
156 		return -1;
157 	}
158 
159 	ctx->cb_arg = NULL;
160 	ctx->cb_fn = NULL;
161 	pthread_mutex_unlock(&ctx->mtx);
162 
163 	return 0;
164 }
165 
166 struct spdk_pci_device *
167 spdk_pci_get_device(struct spdk_pci_addr *pci_addr)
168 {
169 	struct rte_pci_device	*dev;
170 	struct rte_pci_addr	addr;
171 	int			rc;
172 
173 	addr.domain = pci_addr->domain;
174 	addr.bus = pci_addr->bus;
175 	addr.devid = pci_addr->dev;
176 	addr.function = pci_addr->func;
177 
178 	TAILQ_FOREACH(dev, &pci_device_list, next) {
179 		rc = rte_eal_compare_pci_addr(&dev->addr, &addr);
180 		if (rc < 0) {
181 			continue;
182 		}
183 
184 		if (rc == 0) {
185 			return (struct spdk_pci_device *)dev;
186 		} else {
187 			break;
188 		}
189 	}
190 
191 	return NULL;
192 }
193 
194 int
195 spdk_pci_device_map_bar(struct spdk_pci_device *device, uint32_t bar,
196 			void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
197 {
198 	struct rte_pci_device *dev = device;
199 
200 	*mapped_addr = dev->mem_resource[bar].addr;
201 	*phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr;
202 	*size = (uint64_t)dev->mem_resource[bar].len;
203 
204 	return 0;
205 }
206 
207 int
208 spdk_pci_device_unmap_bar(struct spdk_pci_device *device, uint32_t bar, void *addr)
209 {
210 	return 0;
211 }
212 
213 uint16_t
214 spdk_pci_device_get_domain(struct spdk_pci_device *dev)
215 {
216 	return dev->addr.domain;
217 }
218 
219 uint8_t
220 spdk_pci_device_get_bus(struct spdk_pci_device *dev)
221 {
222 	return dev->addr.bus;
223 }
224 
225 uint8_t
226 spdk_pci_device_get_dev(struct spdk_pci_device *dev)
227 {
228 	return dev->addr.devid;
229 }
230 
231 uint8_t
232 spdk_pci_device_get_func(struct spdk_pci_device *dev)
233 {
234 	return dev->addr.function;
235 }
236 
237 uint16_t
238 spdk_pci_device_get_vendor_id(struct spdk_pci_device *dev)
239 {
240 	return dev->id.vendor_id;
241 }
242 
243 uint16_t
244 spdk_pci_device_get_device_id(struct spdk_pci_device *dev)
245 {
246 	return dev->id.device_id;
247 }
248 
249 uint16_t
250 spdk_pci_device_get_subvendor_id(struct spdk_pci_device *dev)
251 {
252 	return dev->id.subsystem_vendor_id;
253 }
254 
255 uint16_t
256 spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev)
257 {
258 	return dev->id.subsystem_device_id;
259 }
260 
261 struct spdk_pci_id
262 spdk_pci_device_get_id(struct spdk_pci_device *pci_dev)
263 {
264 	struct spdk_pci_id pci_id;
265 
266 	pci_id.vendor_id = spdk_pci_device_get_vendor_id(pci_dev);
267 	pci_id.device_id = spdk_pci_device_get_device_id(pci_dev);
268 	pci_id.subvendor_id = spdk_pci_device_get_subvendor_id(pci_dev);
269 	pci_id.subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev);
270 
271 	return pci_id;
272 }
273 
274 int
275 spdk_pci_device_get_socket_id(struct spdk_pci_device *pci_dev)
276 {
277 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
278 	return pci_dev->device.numa_node;
279 #else
280 	return pci_dev->numa_node;
281 #endif
282 }
283 
284 int
285 spdk_pci_device_cfg_read8(struct spdk_pci_device *dev, uint8_t *value, uint32_t offset)
286 {
287 	return rte_eal_pci_read_config(dev, value, 1, offset) == 1 ? 0 : -1;
288 }
289 
290 int
291 spdk_pci_device_cfg_write8(struct spdk_pci_device *dev, uint8_t value, uint32_t offset)
292 {
293 	return rte_eal_pci_write_config(dev, &value, 1, offset) == 1 ? 0 : -1;
294 }
295 
296 int
297 spdk_pci_device_cfg_read16(struct spdk_pci_device *dev, uint16_t *value, uint32_t offset)
298 {
299 	return rte_eal_pci_read_config(dev, value, 2, offset) == 2 ? 0 : -1;
300 }
301 
302 int
303 spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uint32_t offset)
304 {
305 	return rte_eal_pci_write_config(dev, &value, 2, offset) == 2 ? 0 : -1;
306 }
307 
308 int
309 spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset)
310 {
311 	return rte_eal_pci_read_config(dev, value, 4, offset) == 4 ? 0 : -1;
312 }
313 
314 int
315 spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset)
316 {
317 	return rte_eal_pci_write_config(dev, &value, 4, offset) == 4 ? 0 : -1;
318 }
319 
320 int
321 spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t len)
322 {
323 	int err;
324 	uint32_t pos, header = 0;
325 	uint32_t i, buf[2];
326 
327 	if (len < 17)
328 		return -1;
329 
330 	err = spdk_pci_device_cfg_read32(dev, &header, PCI_CFG_SIZE);
331 	if (err || !header)
332 		return -1;
333 
334 	pos = PCI_CFG_SIZE;
335 	while (1) {
336 		if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) {
337 			if (pos) {
338 				/*skip the header*/
339 				pos += 4;
340 				for (i = 0; i < 2; i++) {
341 					err = spdk_pci_device_cfg_read32(dev, &buf[i], pos + 4 * i);
342 					if (err)
343 						return -1;
344 				}
345 				sprintf(sn, "%08x%08x", buf[1], buf[0]);
346 				return 0;
347 			}
348 		}
349 		pos = (header >> 20) & 0xffc;
350 		/*0 if no other items exist*/
351 		if (pos < PCI_CFG_SIZE)
352 			return -1;
353 		err = spdk_pci_device_cfg_read32(dev, &header, pos);
354 		if (err)
355 			return -1;
356 	}
357 	return -1;
358 }
359 
360 struct spdk_pci_addr
361 spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev)
362 {
363 	struct spdk_pci_addr pci_addr;
364 
365 	pci_addr.domain = spdk_pci_device_get_domain(pci_dev);
366 	pci_addr.bus = spdk_pci_device_get_bus(pci_dev);
367 	pci_addr.dev = spdk_pci_device_get_dev(pci_dev);
368 	pci_addr.func = spdk_pci_device_get_func(pci_dev);
369 
370 	return pci_addr;
371 }
372 
373 int
374 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2)
375 {
376 	if (a1->domain > a2->domain) {
377 		return 1;
378 	} else if (a1->domain < a2->domain) {
379 		return -1;
380 	} else if (a1->bus > a2->bus) {
381 		return 1;
382 	} else if (a1->bus < a2->bus) {
383 		return -1;
384 	} else if (a1->dev > a2->dev) {
385 		return 1;
386 	} else if (a1->dev < a2->dev) {
387 		return -1;
388 	} else if (a1->func > a2->func) {
389 		return 1;
390 	} else if (a1->func < a2->func) {
391 		return -1;
392 	}
393 
394 	return 0;
395 }
396 
397 #ifdef __linux__
398 int
399 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr)
400 {
401 	int dev_fd;
402 	char shm_name[64];
403 	int pid;
404 	void *dev_map;
405 	struct flock pcidev_lock = {
406 		.l_type = F_WRLCK,
407 		.l_whence = SEEK_SET,
408 		.l_start = 0,
409 		.l_len = 0,
410 	};
411 
412 	sprintf(shm_name, PCI_PRI_FMT, pci_addr->domain, pci_addr->bus, pci_addr->dev, pci_addr->func);
413 
414 	dev_fd = shm_open(shm_name, O_RDWR | O_CREAT, 0600);
415 	if (dev_fd == -1) {
416 		fprintf(stderr, "could not shm_open %s\n", shm_name);
417 		return -1;
418 	}
419 
420 	if (ftruncate(dev_fd, sizeof(int)) != 0) {
421 		fprintf(stderr, "could not truncate shm %s\n", shm_name);
422 		close(dev_fd);
423 		return -1;
424 	}
425 
426 	dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,
427 		       MAP_SHARED, dev_fd, 0);
428 	if (dev_map == NULL) {
429 		fprintf(stderr, "could not mmap shm %s\n", shm_name);
430 		close(dev_fd);
431 		return -1;
432 	}
433 
434 	if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) {
435 		pid = *(int *)dev_map;
436 		fprintf(stderr, "Cannot create lock on device %s, probably"
437 			" process %d has claimed it\n", shm_name, pid);
438 		munmap(dev_map, sizeof(int));
439 		close(dev_fd);
440 		return -1;
441 	}
442 
443 	*(int *)dev_map = (int)getpid();
444 	munmap(dev_map, sizeof(int));
445 	/* Keep dev_fd open to maintain the lock. */
446 	return 0;
447 }
448 #endif /* __linux__ */
449 
450 #ifdef __FreeBSD__
451 int
452 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr)
453 {
454 	/* TODO */
455 	return 0;
456 }
457 #endif /* __FreeBSD__ */
458 
459 int
460 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf)
461 {
462 	unsigned domain, bus, dev, func;
463 
464 	if (addr == NULL || bdf == NULL) {
465 		return -EINVAL;
466 	}
467 
468 	if (sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) {
469 		/* Matched a full address - all variables are initialized */
470 	} else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) {
471 		func = 0;
472 	} else if (sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) {
473 		domain = 0;
474 	} else if (sscanf(bdf, "%x:%x", &bus, &dev) == 2) {
475 		domain = 0;
476 		func = 0;
477 	} else {
478 		return -EINVAL;
479 	}
480 
481 	if (domain > 0xFFFF || bus > 0xFF || dev > 0x1F || func > 7) {
482 		return -EINVAL;
483 	}
484 
485 	addr->domain = domain;
486 	addr->bus = bus;
487 	addr->dev = dev;
488 	addr->func = func;
489 
490 	return 0;
491 }
492 
493 int
494 spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr)
495 {
496 	int rc;
497 
498 	rc = snprintf(bdf, sz, PCI_PRI_FMT,
499 		      addr->domain, addr->bus,
500 		      addr->dev, addr->func);
501 
502 	if (rc > 0 && (size_t)rc < sz) {
503 		return 0;
504 	}
505 
506 	return -1;
507 }
508