xref: /dpdk/drivers/net/virtio/virtio_pci.c (revision 7bb1168d984aaa7e204c52d13c4701eac0f82989)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation
3  */
4 #include <stdint.h>
5 
6 #ifdef RTE_EXEC_ENV_LINUX
7  #include <dirent.h>
8  #include <fcntl.h>
9 #endif
10 
11 #include <rte_io.h>
12 #include <bus_driver.h>
13 
14 #include "virtio_pci.h"
15 #include "virtio_logs.h"
16 #include "virtqueue.h"
17 
18 /*
19  * The remaining space is defined by each driver as the per-driver
20  * configuration space.
21  */
22 #define VIRTIO_PCI_CONFIG(dev) \
23 		(((dev)->msix_status == VIRTIO_MSIX_ENABLED) ? 24 : 20)
24 
25 struct virtio_pci_internal virtio_pci_internal[RTE_MAX_ETHPORTS];
26 
27 static enum virtio_msix_status
vtpci_msix_detect(struct rte_pci_device * dev)28 vtpci_msix_detect(struct rte_pci_device *dev)
29 {
30 	uint16_t flags;
31 	off_t pos;
32 
33 	pos = rte_pci_find_capability(dev, RTE_PCI_CAP_ID_MSIX);
34 	if (pos > 0 && rte_pci_read_config(dev, &flags, sizeof(flags),
35 			pos + RTE_PCI_MSIX_FLAGS) == sizeof(flags)) {
36 		if (flags & RTE_PCI_MSIX_FLAGS_ENABLE)
37 			return VIRTIO_MSIX_ENABLED;
38 		else
39 			return VIRTIO_MSIX_DISABLED;
40 	}
41 
42 	return VIRTIO_MSIX_NONE;
43 }
44 
45 /*
46  * Since we are in legacy mode:
47  * http://ozlabs.org/~rusty/virtio-spec/virtio-0.9.5.pdf
48  *
49  * "Note that this is possible because while the virtio header is PCI (i.e.
50  * little) endian, the device-specific region is encoded in the native endian of
51  * the guest (where such distinction is applicable)."
52  *
53  * For powerpc which supports both, qemu supposes that cpu is big endian and
54  * enforces this for the virtio-net stuff.
55  */
56 static void
legacy_read_dev_config(struct virtio_hw * hw,size_t offset,void * dst,int length)57 legacy_read_dev_config(struct virtio_hw *hw, size_t offset,
58 		       void *dst, int length)
59 {
60 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
61 #ifdef RTE_ARCH_PPC_64
62 	int size;
63 
64 	while (length > 0) {
65 		if (length >= 4) {
66 			size = 4;
67 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
68 				VIRTIO_PCI_CONFIG(dev) + offset);
69 			*(uint32_t *)dst = rte_be_to_cpu_32(*(uint32_t *)dst);
70 		} else if (length >= 2) {
71 			size = 2;
72 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
73 				VIRTIO_PCI_CONFIG(dev) + offset);
74 			*(uint16_t *)dst = rte_be_to_cpu_16(*(uint16_t *)dst);
75 		} else {
76 			size = 1;
77 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
78 				VIRTIO_PCI_CONFIG(dev) + offset);
79 		}
80 
81 		dst = (char *)dst + size;
82 		offset += size;
83 		length -= size;
84 	}
85 #else
86 	rte_pci_ioport_read(VTPCI_IO(hw), dst, length,
87 		VIRTIO_PCI_CONFIG(dev) + offset);
88 #endif
89 }
90 
91 static void
legacy_write_dev_config(struct virtio_hw * hw,size_t offset,const void * src,int length)92 legacy_write_dev_config(struct virtio_hw *hw, size_t offset,
93 			const void *src, int length)
94 {
95 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
96 #ifdef RTE_ARCH_PPC_64
97 	union {
98 		uint32_t u32;
99 		uint16_t u16;
100 	} tmp;
101 	int size;
102 
103 	while (length > 0) {
104 		if (length >= 4) {
105 			size = 4;
106 			tmp.u32 = rte_cpu_to_be_32(*(const uint32_t *)src);
107 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u32, size,
108 				VIRTIO_PCI_CONFIG(dev) + offset);
109 		} else if (length >= 2) {
110 			size = 2;
111 			tmp.u16 = rte_cpu_to_be_16(*(const uint16_t *)src);
112 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u16, size,
113 				VIRTIO_PCI_CONFIG(dev) + offset);
114 		} else {
115 			size = 1;
116 			rte_pci_ioport_write(VTPCI_IO(hw), src, size,
117 				VIRTIO_PCI_CONFIG(dev) + offset);
118 		}
119 
120 		src = (const char *)src + size;
121 		offset += size;
122 		length -= size;
123 	}
124 #else
125 	rte_pci_ioport_write(VTPCI_IO(hw), src, length,
126 		VIRTIO_PCI_CONFIG(dev) + offset);
127 #endif
128 }
129 
130 static uint64_t
legacy_get_features(struct virtio_hw * hw)131 legacy_get_features(struct virtio_hw *hw)
132 {
133 	uint32_t dst;
134 
135 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 4, VIRTIO_PCI_HOST_FEATURES);
136 	return dst;
137 }
138 
139 static void
legacy_set_features(struct virtio_hw * hw,uint64_t features)140 legacy_set_features(struct virtio_hw *hw, uint64_t features)
141 {
142 	if ((features >> 32) != 0) {
143 		PMD_DRV_LOG(ERR,
144 			"only 32 bit features are allowed for legacy virtio!");
145 		return;
146 	}
147 	rte_pci_ioport_write(VTPCI_IO(hw), &features, 4,
148 		VIRTIO_PCI_GUEST_FEATURES);
149 }
150 
151 static int
legacy_features_ok(struct virtio_hw * hw __rte_unused)152 legacy_features_ok(struct virtio_hw *hw __rte_unused)
153 {
154 	return 0;
155 }
156 
157 static uint8_t
legacy_get_status(struct virtio_hw * hw)158 legacy_get_status(struct virtio_hw *hw)
159 {
160 	uint8_t dst;
161 
162 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_STATUS);
163 	return dst;
164 }
165 
166 static void
legacy_set_status(struct virtio_hw * hw,uint8_t status)167 legacy_set_status(struct virtio_hw *hw, uint8_t status)
168 {
169 	rte_pci_ioport_write(VTPCI_IO(hw), &status, 1, VIRTIO_PCI_STATUS);
170 }
171 
172 static uint8_t
legacy_get_isr(struct virtio_hw * hw)173 legacy_get_isr(struct virtio_hw *hw)
174 {
175 	uint8_t dst;
176 
177 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_ISR);
178 	return dst;
179 }
180 
181 /* Enable one vector (0) for Link State Interrupt */
182 static uint16_t
legacy_set_config_irq(struct virtio_hw * hw,uint16_t vec)183 legacy_set_config_irq(struct virtio_hw *hw, uint16_t vec)
184 {
185 	uint16_t dst;
186 
187 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_CONFIG_VECTOR);
188 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_CONFIG_VECTOR);
189 	return dst;
190 }
191 
192 static uint16_t
legacy_set_queue_irq(struct virtio_hw * hw,struct virtqueue * vq,uint16_t vec)193 legacy_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
194 {
195 	uint16_t dst;
196 
197 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
198 		VIRTIO_PCI_QUEUE_SEL);
199 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_QUEUE_VECTOR);
200 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_QUEUE_VECTOR);
201 	return dst;
202 }
203 
204 static uint16_t
legacy_get_queue_num(struct virtio_hw * hw,uint16_t queue_id)205 legacy_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
206 {
207 	uint16_t dst;
208 
209 	rte_pci_ioport_write(VTPCI_IO(hw), &queue_id, 2, VIRTIO_PCI_QUEUE_SEL);
210 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_PCI_QUEUE_NUM);
211 	return dst;
212 }
213 
214 static int
legacy_setup_queue(struct virtio_hw * hw,struct virtqueue * vq)215 legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
216 {
217 	uint32_t src;
218 
219 	/* Virtio PCI device VIRTIO_PCI_QUEUE_PFN register is 32bit,
220 	 * and only accepts 32 bit page frame number.
221 	 * Check if the allocated physical memory exceeds 16TB.
222 	 */
223 	if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >>
224 			(VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
225 		PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
226 		return -1;
227 	}
228 
229 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
230 		VIRTIO_PCI_QUEUE_SEL);
231 	src = vq->vq_ring_mem >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
232 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
233 
234 	return 0;
235 }
236 
237 static void
legacy_del_queue(struct virtio_hw * hw,struct virtqueue * vq)238 legacy_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
239 {
240 	uint32_t src = 0;
241 
242 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
243 		VIRTIO_PCI_QUEUE_SEL);
244 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
245 }
246 
247 static void
legacy_notify_queue(struct virtio_hw * hw,struct virtqueue * vq)248 legacy_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
249 {
250 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
251 		VIRTIO_PCI_QUEUE_NOTIFY);
252 }
253 
254 static void
legacy_intr_detect(struct virtio_hw * hw)255 legacy_intr_detect(struct virtio_hw *hw)
256 {
257 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
258 
259 	dev->msix_status = vtpci_msix_detect(VTPCI_DEV(hw));
260 	hw->intr_lsc = !!dev->msix_status;
261 }
262 
263 static int
legacy_dev_close(struct virtio_hw * hw)264 legacy_dev_close(struct virtio_hw *hw)
265 {
266 	rte_pci_unmap_device(VTPCI_DEV(hw));
267 	rte_pci_ioport_unmap(VTPCI_IO(hw));
268 
269 	return 0;
270 }
271 
272 const struct virtio_ops legacy_ops = {
273 	.read_dev_cfg	= legacy_read_dev_config,
274 	.write_dev_cfg	= legacy_write_dev_config,
275 	.get_status	= legacy_get_status,
276 	.set_status	= legacy_set_status,
277 	.get_features	= legacy_get_features,
278 	.set_features	= legacy_set_features,
279 	.features_ok	= legacy_features_ok,
280 	.get_isr	= legacy_get_isr,
281 	.set_config_irq	= legacy_set_config_irq,
282 	.set_queue_irq  = legacy_set_queue_irq,
283 	.get_queue_num	= legacy_get_queue_num,
284 	.setup_queue	= legacy_setup_queue,
285 	.del_queue	= legacy_del_queue,
286 	.notify_queue	= legacy_notify_queue,
287 	.intr_detect	= legacy_intr_detect,
288 	.dev_close	= legacy_dev_close,
289 };
290 
291 static inline void
io_write64_twopart(uint64_t val,uint32_t * lo,uint32_t * hi)292 io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi)
293 {
294 	rte_write32(val & ((1ULL << 32) - 1), lo);
295 	rte_write32(val >> 32,		     hi);
296 }
297 
298 static void
modern_read_dev_config(struct virtio_hw * hw,size_t offset,void * dst,int length)299 modern_read_dev_config(struct virtio_hw *hw, size_t offset,
300 		       void *dst, int length)
301 {
302 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
303 	int i;
304 	uint8_t *p;
305 	uint8_t old_gen, new_gen;
306 
307 	do {
308 		old_gen = rte_read8(&dev->common_cfg->config_generation);
309 
310 		p = dst;
311 		for (i = 0;  i < length; i++)
312 			*p++ = rte_read8((uint8_t *)dev->dev_cfg + offset + i);
313 
314 		new_gen = rte_read8(&dev->common_cfg->config_generation);
315 	} while (old_gen != new_gen);
316 }
317 
318 static void
modern_write_dev_config(struct virtio_hw * hw,size_t offset,const void * src,int length)319 modern_write_dev_config(struct virtio_hw *hw, size_t offset,
320 			const void *src, int length)
321 {
322 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
323 	int i;
324 	const uint8_t *p = src;
325 
326 	for (i = 0;  i < length; i++)
327 		rte_write8((*p++), (((uint8_t *)dev->dev_cfg) + offset + i));
328 }
329 
330 static uint64_t
modern_get_features(struct virtio_hw * hw)331 modern_get_features(struct virtio_hw *hw)
332 {
333 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
334 	uint32_t features_lo, features_hi;
335 
336 	rte_write32(0, &dev->common_cfg->device_feature_select);
337 	features_lo = rte_read32(&dev->common_cfg->device_feature);
338 
339 	rte_write32(1, &dev->common_cfg->device_feature_select);
340 	features_hi = rte_read32(&dev->common_cfg->device_feature);
341 
342 	return ((uint64_t)features_hi << 32) | features_lo;
343 }
344 
345 static void
modern_set_features(struct virtio_hw * hw,uint64_t features)346 modern_set_features(struct virtio_hw *hw, uint64_t features)
347 {
348 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
349 
350 	rte_write32(0, &dev->common_cfg->guest_feature_select);
351 	rte_write32(features & ((1ULL << 32) - 1),
352 		    &dev->common_cfg->guest_feature);
353 
354 	rte_write32(1, &dev->common_cfg->guest_feature_select);
355 	rte_write32(features >> 32,
356 		    &dev->common_cfg->guest_feature);
357 }
358 
359 static int
modern_features_ok(struct virtio_hw * hw)360 modern_features_ok(struct virtio_hw *hw)
361 {
362 	if (!virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
363 		PMD_INIT_LOG(ERR, "Version 1+ required with modern devices");
364 		return -1;
365 	}
366 
367 	return 0;
368 }
369 
370 static uint8_t
modern_get_status(struct virtio_hw * hw)371 modern_get_status(struct virtio_hw *hw)
372 {
373 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
374 
375 	return rte_read8(&dev->common_cfg->device_status);
376 }
377 
378 static void
modern_set_status(struct virtio_hw * hw,uint8_t status)379 modern_set_status(struct virtio_hw *hw, uint8_t status)
380 {
381 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
382 
383 	rte_write8(status, &dev->common_cfg->device_status);
384 }
385 
386 static uint8_t
modern_get_isr(struct virtio_hw * hw)387 modern_get_isr(struct virtio_hw *hw)
388 {
389 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
390 
391 	return rte_read8(dev->isr);
392 }
393 
394 static uint16_t
modern_set_config_irq(struct virtio_hw * hw,uint16_t vec)395 modern_set_config_irq(struct virtio_hw *hw, uint16_t vec)
396 {
397 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
398 
399 	rte_write16(vec, &dev->common_cfg->msix_config);
400 	return rte_read16(&dev->common_cfg->msix_config);
401 }
402 
403 static uint16_t
modern_set_queue_irq(struct virtio_hw * hw,struct virtqueue * vq,uint16_t vec)404 modern_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
405 {
406 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
407 
408 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
409 	rte_write16(vec, &dev->common_cfg->queue_msix_vector);
410 	return rte_read16(&dev->common_cfg->queue_msix_vector);
411 }
412 
413 static uint16_t
modern_get_queue_num(struct virtio_hw * hw,uint16_t queue_id)414 modern_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
415 {
416 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
417 
418 	rte_write16(queue_id, &dev->common_cfg->queue_select);
419 	return rte_read16(&dev->common_cfg->queue_size);
420 }
421 
422 static int
modern_setup_queue(struct virtio_hw * hw,struct virtqueue * vq)423 modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
424 {
425 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
426 	uint64_t desc_addr, avail_addr, used_addr;
427 	uint16_t notify_off;
428 
429 	desc_addr = vq->vq_ring_mem;
430 	avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
431 	used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
432 							 ring[vq->vq_nentries]),
433 				   VIRTIO_VRING_ALIGN);
434 
435 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
436 
437 	io_write64_twopart(desc_addr, &dev->common_cfg->queue_desc_lo,
438 				      &dev->common_cfg->queue_desc_hi);
439 	io_write64_twopart(avail_addr, &dev->common_cfg->queue_avail_lo,
440 				       &dev->common_cfg->queue_avail_hi);
441 	io_write64_twopart(used_addr, &dev->common_cfg->queue_used_lo,
442 				      &dev->common_cfg->queue_used_hi);
443 
444 	notify_off = rte_read16(&dev->common_cfg->queue_notify_off);
445 	vq->notify_addr = (void *)((uint8_t *)dev->notify_base +
446 				notify_off * dev->notify_off_multiplier);
447 
448 	rte_write16(1, &dev->common_cfg->queue_enable);
449 
450 	PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index);
451 	PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr);
452 	PMD_INIT_LOG(DEBUG, "\t aval_addr: %" PRIx64, avail_addr);
453 	PMD_INIT_LOG(DEBUG, "\t used_addr: %" PRIx64, used_addr);
454 	PMD_INIT_LOG(DEBUG, "\t notify addr: %p (notify offset: %u)",
455 		vq->notify_addr, notify_off);
456 
457 	return 0;
458 }
459 
460 static void
modern_del_queue(struct virtio_hw * hw,struct virtqueue * vq)461 modern_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
462 {
463 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
464 
465 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
466 
467 	io_write64_twopart(0, &dev->common_cfg->queue_desc_lo,
468 				  &dev->common_cfg->queue_desc_hi);
469 	io_write64_twopart(0, &dev->common_cfg->queue_avail_lo,
470 				  &dev->common_cfg->queue_avail_hi);
471 	io_write64_twopart(0, &dev->common_cfg->queue_used_lo,
472 				  &dev->common_cfg->queue_used_hi);
473 
474 	rte_write16(0, &dev->common_cfg->queue_enable);
475 }
476 
477 static void
modern_notify_queue(struct virtio_hw * hw,struct virtqueue * vq)478 modern_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
479 {
480 	uint32_t notify_data;
481 
482 	if (!virtio_with_feature(hw, VIRTIO_F_NOTIFICATION_DATA)) {
483 		rte_write16(vq->vq_queue_index, vq->notify_addr);
484 		return;
485 	}
486 
487 	if (virtio_with_packed_queue(hw)) {
488 		/*
489 		 * Bit[0:15]: vq queue index
490 		 * Bit[16:30]: avail index
491 		 * Bit[31]: avail wrap counter
492 		 */
493 		notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags &
494 				VRING_PACKED_DESC_F_AVAIL)) << 31) |
495 				((uint32_t)vq->vq_avail_idx << 16) |
496 				vq->vq_queue_index;
497 	} else {
498 		/*
499 		 * Bit[0:15]: vq queue index
500 		 * Bit[16:31]: avail index
501 		 */
502 		notify_data = ((uint32_t)vq->vq_avail_idx << 16) |
503 				vq->vq_queue_index;
504 	}
505 	rte_write32(notify_data, vq->notify_addr);
506 }
507 
508 
509 
510 static void
modern_intr_detect(struct virtio_hw * hw)511 modern_intr_detect(struct virtio_hw *hw)
512 {
513 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
514 
515 	dev->msix_status = vtpci_msix_detect(VTPCI_DEV(hw));
516 	hw->intr_lsc = !!dev->msix_status;
517 }
518 
519 static int
modern_dev_close(struct virtio_hw * hw)520 modern_dev_close(struct virtio_hw *hw)
521 {
522 	rte_pci_unmap_device(VTPCI_DEV(hw));
523 
524 	return 0;
525 }
526 
527 const struct virtio_ops modern_ops = {
528 	.read_dev_cfg	= modern_read_dev_config,
529 	.write_dev_cfg	= modern_write_dev_config,
530 	.get_status	= modern_get_status,
531 	.set_status	= modern_set_status,
532 	.get_features	= modern_get_features,
533 	.set_features	= modern_set_features,
534 	.features_ok	= modern_features_ok,
535 	.get_isr	= modern_get_isr,
536 	.set_config_irq	= modern_set_config_irq,
537 	.set_queue_irq  = modern_set_queue_irq,
538 	.get_queue_num	= modern_get_queue_num,
539 	.setup_queue	= modern_setup_queue,
540 	.del_queue	= modern_del_queue,
541 	.notify_queue	= modern_notify_queue,
542 	.intr_detect	= modern_intr_detect,
543 	.dev_close	= modern_dev_close,
544 };
545 
546 static void *
get_cfg_addr(struct rte_pci_device * dev,struct virtio_pci_cap * cap)547 get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap)
548 {
549 	uint8_t  bar    = cap->bar;
550 	uint32_t length = cap->length;
551 	uint32_t offset = cap->offset;
552 	uint8_t *base;
553 
554 	if (bar >= PCI_MAX_RESOURCE) {
555 		PMD_INIT_LOG(ERR, "invalid bar: %u", bar);
556 		return NULL;
557 	}
558 
559 	if (offset + length < offset) {
560 		PMD_INIT_LOG(ERR, "offset(%u) + length(%u) overflows",
561 			offset, length);
562 		return NULL;
563 	}
564 
565 	if (offset + length > dev->mem_resource[bar].len) {
566 		PMD_INIT_LOG(ERR,
567 			"invalid cap: overflows bar space: %u > %" PRIu64,
568 			offset + length, dev->mem_resource[bar].len);
569 		return NULL;
570 	}
571 
572 	base = dev->mem_resource[bar].addr;
573 	if (base == NULL) {
574 		PMD_INIT_LOG(ERR, "bar %u base addr is NULL", bar);
575 		return NULL;
576 	}
577 
578 	return base + offset;
579 }
580 
581 static int
virtio_read_caps(struct rte_pci_device * pci_dev,struct virtio_hw * hw)582 virtio_read_caps(struct rte_pci_device *pci_dev, struct virtio_hw *hw)
583 {
584 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
585 	struct virtio_pci_cap cap;
586 	off_t pos;
587 	int ret;
588 
589 	if (rte_pci_map_device(pci_dev)) {
590 		PMD_INIT_LOG(DEBUG, "failed to map pci device!");
591 		return -1;
592 	}
593 
594 	/*
595 	 * Transitional devices would also have this capability,
596 	 * that's why we also check if msix is enabled.
597 	 */
598 	dev->msix_status = vtpci_msix_detect(pci_dev);
599 
600 	pos = rte_pci_find_capability(pci_dev, RTE_PCI_CAP_ID_VNDR);
601 	while (pos > 0) {
602 		if (rte_pci_read_config(pci_dev, &cap, sizeof(cap), pos) != sizeof(cap))
603 			break;
604 		PMD_INIT_LOG(DEBUG,
605 			"[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u",
606 			(unsigned int)pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
607 
608 		switch (cap.cfg_type) {
609 		case VIRTIO_PCI_CAP_COMMON_CFG:
610 			dev->common_cfg = get_cfg_addr(pci_dev, &cap);
611 			break;
612 		case VIRTIO_PCI_CAP_NOTIFY_CFG:
613 			ret = rte_pci_read_config(pci_dev, &dev->notify_off_multiplier,
614 				4, pos + sizeof(cap));
615 			if (ret != 4)
616 				PMD_INIT_LOG(DEBUG,
617 					"failed to read notify_off_multiplier, ret %d",
618 					ret);
619 			else
620 				dev->notify_base = get_cfg_addr(pci_dev, &cap);
621 			break;
622 		case VIRTIO_PCI_CAP_DEVICE_CFG:
623 			dev->dev_cfg = get_cfg_addr(pci_dev, &cap);
624 			break;
625 		case VIRTIO_PCI_CAP_ISR_CFG:
626 			dev->isr = get_cfg_addr(pci_dev, &cap);
627 			break;
628 		}
629 
630 		pos = rte_pci_find_next_capability(pci_dev, RTE_PCI_CAP_ID_VNDR, pos);
631 	}
632 
633 	if (dev->common_cfg == NULL || dev->notify_base == NULL ||
634 	    dev->dev_cfg == NULL    || dev->isr == NULL) {
635 		PMD_INIT_LOG(INFO, "no modern virtio pci device found.");
636 		return -1;
637 	}
638 
639 	PMD_INIT_LOG(INFO, "found modern virtio pci device.");
640 
641 	PMD_INIT_LOG(DEBUG, "common cfg mapped at: %p", dev->common_cfg);
642 	PMD_INIT_LOG(DEBUG, "device cfg mapped at: %p", dev->dev_cfg);
643 	PMD_INIT_LOG(DEBUG, "isr cfg mapped at: %p", dev->isr);
644 	PMD_INIT_LOG(DEBUG, "notify base: %p, notify off multiplier: %u",
645 		dev->notify_base, dev->notify_off_multiplier);
646 
647 	return 0;
648 }
649 
650 /*
651  * Return -1:
652  *   if there is error mapping with VFIO/UIO.
653  *   if port map error when driver type is KDRV_NONE.
654  *   if marked as allowed but driver type is KDRV_UNKNOWN.
655  * Return 1 if kernel driver is managing the device.
656  * Return 0 on success.
657  */
658 int
vtpci_init(struct rte_pci_device * pci_dev,struct virtio_pci_dev * dev)659 vtpci_init(struct rte_pci_device *pci_dev, struct virtio_pci_dev *dev)
660 {
661 	struct virtio_hw *hw = &dev->hw;
662 
663 	RTE_BUILD_BUG_ON(offsetof(struct virtio_pci_dev, hw) != 0);
664 
665 	/*
666 	 * Try if we can succeed reading virtio pci caps, which exists
667 	 * only on modern pci device. If failed, we fallback to legacy
668 	 * virtio handling.
669 	 */
670 	if (virtio_read_caps(pci_dev, hw) == 0) {
671 		PMD_INIT_LOG(INFO, "modern virtio pci detected.");
672 		VIRTIO_OPS(hw) = &modern_ops;
673 		dev->modern = true;
674 		goto msix_detect;
675 	}
676 
677 	PMD_INIT_LOG(INFO, "trying with legacy virtio pci.");
678 	if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0) {
679 		rte_pci_unmap_device(pci_dev);
680 		if (pci_dev->kdrv == RTE_PCI_KDRV_UNKNOWN &&
681 		    (!pci_dev->device.devargs ||
682 		     pci_dev->device.devargs->bus !=
683 		     rte_bus_find_by_name("pci"))) {
684 			PMD_INIT_LOG(INFO,
685 				"skip kernel managed virtio device.");
686 			return 1;
687 		}
688 		return -1;
689 	}
690 
691 	VIRTIO_OPS(hw) = &legacy_ops;
692 	dev->modern = false;
693 
694 msix_detect:
695 	VIRTIO_OPS(hw)->intr_detect(hw);
696 
697 	return 0;
698 }
699 
vtpci_legacy_ioport_unmap(struct virtio_hw * hw)700 void vtpci_legacy_ioport_unmap(struct virtio_hw *hw)
701 {
702 	rte_pci_ioport_unmap(VTPCI_IO(hw));
703 }
704 
vtpci_legacy_ioport_map(struct virtio_hw * hw)705 int vtpci_legacy_ioport_map(struct virtio_hw *hw)
706 {
707 	return rte_pci_ioport_map(VTPCI_DEV(hw), 0, VTPCI_IO(hw));
708 }
709