xref: /netbsd-src/sys/arch/x86/pci/pci_msi_machdep.c (revision d909946ca08dceb44d7d0f22ec9488679695d976) 
1 /*	$NetBSD: pci_msi_machdep.c,v 1.9 2015/08/17 06:16:03 knakahara Exp $	*/
2 
3 /*
4  * Copyright (c) 2015 Internet Initiative Japan Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26  * POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /*
30  * TODO
31  *
32  *     - PBA (Pending Bit Array) support
33  *     - HyperTransport mapping support
34  */
35 
36 #include <sys/cdefs.h>
37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.9 2015/08/17 06:16:03 knakahara Exp $");
38 
39 #include "opt_intrdebug.h"
40 
41 #include <sys/types.h>
42 #include <sys/param.h>
43 #include <sys/time.h>
44 #include <sys/systm.h>
45 #include <sys/cpu.h>
46 #include <sys/errno.h>
47 #include <sys/device.h>
48 #include <sys/intr.h>
49 #include <sys/kmem.h>
50 #include <sys/malloc.h>
51 
52 #include <dev/pci/pcivar.h>
53 
54 #include <machine/i82093var.h>
55 #include <machine/pic.h>
56 
57 #include <x86/pci/msipic.h>
58 #include <x86/pci/pci_msi_machdep.h>
59 
60 #ifdef INTRDEBUG
61 #define MSIDEBUG
62 #endif
63 
64 #ifdef MSIDEBUG
65 #define DPRINTF(msg) printf msg
66 #else
67 #define DPRINTF(msg)
68 #endif
69 
70 static pci_intr_handle_t
71 pci_msi_calculate_handle(struct pic *msi_pic, int vector)
72 {
73 	pci_intr_handle_t pih;
74 
75 	KASSERT(msipic_is_msi_pic(msi_pic));
76 
77 	pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK)
78 	    | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK)
79 	    | APIC_INT_VIA_MSI;
80 	if (msi_pic->pic_type == PIC_MSI)
81 		MSI_INT_MAKE_MSI(pih);
82 	else if (msi_pic->pic_type == PIC_MSIX)
83 		MSI_INT_MAKE_MSIX(pih);
84 	else
85 		panic("%s: Unexpected pic_type: %d\n", __func__,
86 		    msi_pic->pic_type);
87 
88 	return pih;
89 }
90 
91 static pci_intr_handle_t *
92 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count)
93 {
94 	struct intrsource *isp;
95 	pci_intr_handle_t *vectors, pih;
96 	int i;
97 	const char *intrstr;
98 	char intrstr_buf[INTRIDBUF];
99 
100 	vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP);
101 	if (vectors == NULL) {
102 		DPRINTF(("cannot allocate vectors\n"));
103 		return NULL;
104 	}
105 
106 	mutex_enter(&cpu_lock);
107 	for (i = 0; i < *count; i++) {
108 		u_int table_index;
109 
110 		if (table_indexes == NULL)
111 			table_index = i;
112 		else
113 			table_index = table_indexes[i];
114 
115 		pih = pci_msi_calculate_handle(msi_pic, table_index);
116 
117 		intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
118 		    sizeof(intrstr_buf));
119 		isp = intr_allocate_io_intrsource(intrstr);
120 		if (isp == NULL) {
121 			mutex_exit(&cpu_lock);
122 			DPRINTF(("can't allocate io_intersource\n"));
123 			kmem_free(vectors, sizeof(vectors[0]) * (*count));
124 			return NULL;
125 		}
126 
127 		vectors[i] = pih;
128 	}
129 	mutex_exit(&cpu_lock);
130 
131 	return vectors;
132 }
133 
134 static void
135 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count)
136 {
137 	pci_intr_handle_t pih;
138 	int i;
139 	const char *intrstr;
140 	char intrstr_buf[INTRIDBUF];
141 
142 	mutex_enter(&cpu_lock);
143 	for (i = 0; i < count; i++) {
144 		pih = pci_msi_calculate_handle(msi_pic, i);
145 		intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
146 		    sizeof(intrstr_buf));
147 		intr_free_io_intrsource(intrstr);
148 	}
149 	mutex_exit(&cpu_lock);
150 
151 	kmem_free(pihs, sizeof(pihs[0]) * count);
152 }
153 
154 static int
155 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count,
156     const struct pci_attach_args *pa, bool exact)
157 {
158 	struct pic *msi_pic;
159 	pci_intr_handle_t *vectors;
160 	int error, i;
161 
162 	if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) {
163 		DPRINTF(("PCI host bridge does not support MSI.\n"));
164 		return ENODEV;
165 	}
166 
167 	msi_pic = msipic_construct_msi_pic(pa);
168 	if (msi_pic == NULL) {
169 		DPRINTF(("cannot allocate MSI pic.\n"));
170 		return EINVAL;
171 	}
172 
173 	vectors = NULL;
174 	while (*count > 0) {
175 		vectors = pci_msi_alloc_vectors(msi_pic, NULL, count);
176 		if (vectors != NULL)
177 			break;
178 
179 		if (exact) {
180 			DPRINTF(("cannot allocate MSI vectors.\n"));
181 			msipic_destruct_msi_pic(msi_pic);
182 			return ENOMEM;
183 		} else {
184 			(*count) >>= 1; /* must be power of 2. */
185 			continue;
186 		}
187 	}
188 	if (vectors == NULL) {
189 		DPRINTF(("cannot allocate MSI vectors.\n"));
190 		msipic_destruct_msi_pic(msi_pic);
191 		return ENOMEM;
192 	}
193 
194 	for (i = 0; i < *count; i++) {
195 		MSI_INT_MAKE_MSI(vectors[i]);
196 	}
197 
198 	error = msipic_set_msi_vectors(msi_pic, NULL, *count);
199 	if (error) {
200 		pci_msi_free_vectors(msi_pic, vectors, *count);
201 		msipic_destruct_msi_pic(msi_pic);
202 		return error;
203 	}
204 
205 	*ihps = vectors;
206 	return 0;
207 }
208 
209 static void *
210 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
211     int level, int (*func)(void *), void *arg, struct pic *pic,
212     const char *xname)
213 {
214 	int irq, pin;
215 	bool mpsafe;
216 
217 	KASSERT(INT_VIA_MSI(ih));
218 
219 	irq = -1;
220 	pin = MSI_INT_VEC(ih);
221 	mpsafe = ((ih & MPSAFE_MASK) != 0);
222 
223 	return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg,
224 	    mpsafe, xname);
225 }
226 
227 static void
228 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie)
229 {
230 
231 	intr_disestablish(cookie);
232 }
233 
234 static int
235 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes,
236     int *count, const struct pci_attach_args *pa, bool exact)
237 {
238 	struct pic *msix_pic;
239 	pci_intr_handle_t *vectors;
240 	int error, i;
241 
242 	if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) {
243 		DPRINTF(("PCI host bridge does not support MSI-X.\n"));
244 		return ENODEV;
245 	}
246 
247 	msix_pic = msipic_construct_msix_pic(pa);
248 	if (msix_pic == NULL)
249 		return EINVAL;
250 
251 	vectors = NULL;
252 	while (*count > 0) {
253 		vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count);
254 		if (vectors != NULL)
255 			break;
256 
257 		if (exact) {
258 			DPRINTF(("cannot allocate MSI-X vectors.\n"));
259 			msipic_destruct_msix_pic(msix_pic);
260 			return ENOMEM;
261 		} else {
262 			(*count)--;
263 			continue;
264 		}
265 	}
266 	if (vectors == NULL) {
267 		DPRINTF(("cannot allocate MSI-X vectors.\n"));
268 		msipic_destruct_msix_pic(msix_pic);
269 		return ENOMEM;
270 	}
271 
272 	for (i = 0; i < *count; i++) {
273 		MSI_INT_MAKE_MSIX(vectors[i]);
274 	}
275 
276 	error = msipic_set_msi_vectors(msix_pic, vectors, *count);
277 	if (error) {
278 		pci_msi_free_vectors(msix_pic, vectors, *count);
279 		msipic_destruct_msix_pic(msix_pic);
280 		return error;
281 	}
282 
283 	*ihps = vectors;
284 	return 0;
285 }
286 
287 static int
288 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count,
289     const struct pci_attach_args *pa)
290 {
291 
292 	return pci_msi_alloc_common(ihps, count, pa, false);
293 }
294 
295 static int
296 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count,
297     const struct pci_attach_args *pa)
298 {
299 
300 	return pci_msi_alloc_common(ihps, &count, pa, true);
301 }
302 
303 static void
304 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count)
305 {
306 	struct pic *pic;
307 
308 	pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
309 	if (pic == NULL)
310 		return;
311 
312 	pci_msi_free_vectors(pic, pihs, count);
313 	msipic_destruct_msi_pic(pic);
314 }
315 
316 static int
317 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count,
318     const struct pci_attach_args *pa)
319 {
320 
321 	return pci_msix_alloc_common(ihps, NULL, count, pa, false);
322 }
323 
324 static int
325 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count,
326     const struct pci_attach_args *pa)
327 {
328 
329 	return pci_msix_alloc_common(ihps, NULL, &count, pa, true);
330 }
331 
332 static int
333 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes,
334     int count, const struct pci_attach_args *pa)
335 {
336 
337 	return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true);
338 }
339 
340 static void
341 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count)
342 {
343 	struct pic *pic;
344 
345 	pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
346 	if (pic == NULL)
347 		return;
348 
349 	pci_msi_free_vectors(pic, pihs, count);
350 	msipic_destruct_msix_pic(pic);
351 }
352 
353 /*****************************************************************************/
354 /*
355  * extern for MD code.
356  */
357 
358 /*
359  * Return intrid for a MSI/MSI-X device.
360  * "buf" must be allocated by caller.
361  */
362 const char *
363 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf,
364     size_t len)
365 {
366 	int dev, vec;
367 
368 	KASSERT(INT_VIA_MSI(ih));
369 
370 	dev = MSI_INT_DEV(ih);
371 	vec = MSI_INT_VEC(ih);
372 	if (MSI_INT_IS_MSIX(ih))
373 		snprintf(buf, len, "msix%d vec %d", dev, vec);
374 	else
375 		snprintf(buf, len, "msi%d vec %d", dev, vec);
376 
377 	return buf;
378 }
379 
380 /*
381  * Release MSI handles.
382  */
383 void
384 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
385 {
386 
387 	if (count < 1)
388 		return;
389 
390 	return x86_pci_msi_release_internal(pihs, count);
391 }
392 
393 /*
394  * Establish a MSI handle.
395  * If multiple MSI handle is requied to establish, device driver must call
396  * this function for each handle.
397  */
398 void *
399 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
400     int level, int (*func)(void *), void *arg, const char *xname)
401 {
402 	struct pic *pic;
403 
404 	pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
405 	if (pic == NULL) {
406 		DPRINTF(("pci_intr_handler has no msi_pic\n"));
407 		return NULL;
408 	}
409 
410 	return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
411 }
412 
413 /*
414  * Disestablish a MSI handle.
415  * If multiple MSI handle is requied to disestablish, device driver must call
416  * this function for each handle.
417  */
418 void
419 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie)
420 {
421 
422 	pci_msi_common_disestablish(pc, cookie);
423 }
424 
425 /*
426  * Release MSI-X handles.
427  */
428 void
429 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
430 {
431 
432 	if (count < 1)
433 		return;
434 
435 	return x86_pci_msix_release_internal(pihs, count);
436 }
437 
438 /*
439  * Establish a MSI-X handle.
440  * If multiple MSI-X handle is requied to establish, device driver must call
441  * this function for each handle.
442  */
443 void *
444 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
445     int level, int (*func)(void *), void *arg, const char *xname)
446 {
447 	struct pic *pic;
448 
449 	pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
450 	if (pic == NULL) {
451 		DPRINTF(("pci_intr_handler has no msi_pic\n"));
452 		return NULL;
453 	}
454 
455 	return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
456 }
457 
458 /*
459  * Disestablish a MSI-X handle.
460  * If multiple MSI-X handle is requied to disestablish, device driver must call
461  * this function for each handle.
462  */
463 void
464 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie)
465 {
466 
467 	pci_msi_common_disestablish(pc, cookie);
468 }
469 
470 /*****************************************************************************/
471 /*
472  * extern for MI code.
473  */
474 
475 /*
476  * This function is used by device drivers like pci_intr_map().
477  *
478  * "ihps" is the array  of vector numbers which MSI used instead of IRQ number.
479  * "count" must be power of 2.
480  * "count" can decrease if struct intrsource cannot be allocated.
481  * if count == 0, return non-zero value.
482  */
483 int
484 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
485     int *count)
486 {
487 	int hw_max;
488 
489 	/* MSI vector count must be power of 2. */
490 	KASSERT(*count > 0);
491 	KASSERT(((*count - 1) & *count) == 0);
492 
493 	hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
494 	if (hw_max == 0)
495 		return ENODEV;
496 
497 	if (*count > hw_max) {
498 		DPRINTF(("cut off MSI count to %d\n", hw_max));
499 		*count = hw_max; /* cut off hw_max */
500 	}
501 
502 	return x86_pci_msi_alloc(ihps, count, pa);
503 }
504 
505 /*
506  * This function is used by device drivers like pci_intr_map().
507  *
508  * "ihps" is the array  of vector numbers which MSI used instead of IRQ number.
509  * "count" must be power of 2.
510  * "count" can not decrease.
511  * If "count" struct intrsources cannot be allocated, return non-zero value.
512  */
513 int
514 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
515     int count)
516 {
517 	int hw_max;
518 
519 	/* MSI vector count must be power of 2. */
520 	KASSERT(count > 0);
521 	KASSERT(((count - 1) & count) == 0);
522 
523 	hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
524 	if (hw_max == 0)
525 		return ENODEV;
526 
527 	if (count > hw_max) {
528 		DPRINTF(("over hardware max MSI count %d\n", hw_max));
529 		return EINVAL;
530 	}
531 
532 	return x86_pci_msi_alloc_exact(ihps, count, pa);
533 }
534 
535 /*
536  * This function is used by device drivers like pci_intr_map().
537  *
538  * "ihps" is the array  of vector numbers which MSI-X used instead of IRQ number.
539  * "count" can decrease if enough struct intrsources cannot be allocated.
540  * if count == 0, return non-zero value.
541  */
542 int
543 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
544     int *count)
545 {
546 	int hw_max;
547 
548 	KASSERT(*count > 0);
549 
550 	hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
551 	if (hw_max == 0)
552 		return ENODEV;
553 
554 	if (*count > hw_max) {
555 		DPRINTF(("cut off MSI-X count to %d\n", hw_max));
556 		*count = hw_max; /* cut off hw_max */
557 	}
558 
559 	return x86_pci_msix_alloc(ihps, count, pa);
560 }
561 
562 /*
563  * This function is used by device drivers like pci_intr_map().
564  *
565  * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
566  * "count" can not decrease.
567  * If "count" struct intrsource cannot be allocated, return non-zero value.
568  */
569 int
570 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
571     int count)
572 {
573 	int hw_max;
574 
575 	KASSERT(count > 0);
576 
577 	hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
578 	if (hw_max == 0)
579 		return ENODEV;
580 
581 	if (count > hw_max) {
582 		DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
583 		return EINVAL;
584 	}
585 
586 	return x86_pci_msix_alloc_exact(ihps, count, pa);
587 }
588 
589 /*
590  * This function is used by device drivers like pci_intr_map().
591  * Futhermore, this function can map each handle to a MSI-X table index.
592  *
593  * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
594  * "count" can not decrease.
595  * "map" size must be equal to "count".
596  * If "count" struct intrsource cannot be allocated, return non-zero value.
597  * e.g.
598  *     If "map" = { 1, 4, 0 },
599  *     1st handle is bound to MSI-X index 1
600  *     2nd handle is bound to MSI-X index 4
601  *     3rd handle is bound to MSI-X index 0
602  */
603 int
604 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
605     u_int *table_indexes, int count)
606 {
607 	int hw_max, i, j;
608 
609 	KASSERT(count > 0);
610 
611 	hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
612 	if (hw_max == 0)
613 		return ENODEV;
614 
615 	if (count > hw_max) {
616 		DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
617 		return EINVAL;
618 	}
619 
620 	/* check not to duplicate table_index */
621 	for (i = 0; i < count; i++) {
622 		u_int basing = table_indexes[i];
623 
624 		KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS);
625 		if (basing >= hw_max) {
626 			DPRINTF(("table index is over hardware max MSI-X index %d\n",
627 				hw_max - 1));
628 			return EINVAL;
629 		}
630 
631 		for (j = i + 1; j < count; j++) {
632 			if (basing == table_indexes[j]) {
633 				DPRINTF(("MSI-X table index duplicated\n"));
634 				return EINVAL;
635 			}
636 		}
637 	}
638 
639 	return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa);
640 }
641