12b3ad188SAdrian Chadd /*- 2bff6be3eSSvatopluk Kraus * Copyright (c) 2015-2016 Svatopluk Kraus 3bff6be3eSSvatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun 42b3ad188SAdrian Chadd * All rights reserved. 5fae8755fSJessica Clarke * Copyright (c) 2015-2016 The FreeBSD Foundation 6fae8755fSJessica Clarke * Copyright (c) 2021 Jessica Clarke <jrtc27@FreeBSD.org> 7fae8755fSJessica Clarke * 8fae8755fSJessica Clarke * Portions of this software were developed by Andrew Turner under 9fae8755fSJessica Clarke * sponsorship from the FreeBSD Foundation. 102b3ad188SAdrian Chadd * 112b3ad188SAdrian Chadd * Redistribution and use in source and binary forms, with or without 122b3ad188SAdrian Chadd * modification, are permitted provided that the following conditions 132b3ad188SAdrian Chadd * are met: 142b3ad188SAdrian Chadd * 1. Redistributions of source code must retain the above copyright 152b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer. 162b3ad188SAdrian Chadd * 2. Redistributions in binary form must reproduce the above copyright 172b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer in the 182b3ad188SAdrian Chadd * documentation and/or other materials provided with the distribution. 192b3ad188SAdrian Chadd * 202b3ad188SAdrian Chadd * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 212b3ad188SAdrian Chadd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 222b3ad188SAdrian Chadd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 232b3ad188SAdrian Chadd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 242b3ad188SAdrian Chadd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 252b3ad188SAdrian Chadd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 262b3ad188SAdrian Chadd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 272b3ad188SAdrian Chadd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 282b3ad188SAdrian Chadd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 292b3ad188SAdrian Chadd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 302b3ad188SAdrian Chadd * SUCH DAMAGE. 312b3ad188SAdrian Chadd */ 322b3ad188SAdrian Chadd 332b3ad188SAdrian Chadd #include <sys/cdefs.h> 342b3ad188SAdrian Chadd /* 352b3ad188SAdrian Chadd * New-style Interrupt Framework 362b3ad188SAdrian Chadd * 37895c8b1cSMichal Meloun * TODO: - add support for disconnected PICs. 38895c8b1cSMichal Meloun * - to support IPI (PPI) enabling on other CPUs if already started. 39895c8b1cSMichal Meloun * - to complete things for removable PICs. 402b3ad188SAdrian Chadd */ 412b3ad188SAdrian Chadd 422b3ad188SAdrian Chadd #include "opt_ddb.h" 43df7a2251SAndrew Turner #include "opt_hwpmc_hooks.h" 44e707c8beSRuslan Bukin #include "opt_iommu.h" 452b3ad188SAdrian Chadd 462b3ad188SAdrian Chadd #include <sys/param.h> 472b3ad188SAdrian Chadd #include <sys/systm.h> 4889c52f9dSKyle Evans #include <sys/asan.h> 4928137bdbSMitchell Horne #include <sys/bitstring.h> 502b3ad188SAdrian Chadd #include <sys/bus.h> 512b3ad188SAdrian Chadd #include <sys/conf.h> 522b3ad188SAdrian Chadd #include <sys/cpuset.h> 5382e846dfSMitchell Horne #include <sys/interrupt.h> 544f12b529SKyle Evans #include <sys/intr.h> 5582e846dfSMitchell Horne #include <sys/kernel.h> 5682e846dfSMitchell Horne #include <sys/lock.h> 5782e846dfSMitchell Horne #include <sys/malloc.h> 58c05d7bdaSMark Johnston #include <sys/msan.h> 5982e846dfSMitchell Horne #include <sys/mutex.h> 6082e846dfSMitchell Horne #include <sys/proc.h> 6182e846dfSMitchell Horne #include <sys/queue.h> 626b42a1f4SAndrew Turner #include <sys/rman.h> 632b3ad188SAdrian Chadd #include <sys/sched.h> 642b3ad188SAdrian Chadd #include <sys/smp.h> 65248f0cabSOleksandr Tymoshenko #include <sys/sysctl.h> 6682e846dfSMitchell Horne #include <sys/syslog.h> 6782e846dfSMitchell Horne #include <sys/taskqueue.h> 6882e846dfSMitchell Horne #include <sys/tree.h> 699ed01c32SGleb Smirnoff #include <sys/vmmeter.h> 70df7a2251SAndrew Turner #ifdef HWPMC_HOOKS 71df7a2251SAndrew Turner #include <sys/pmckern.h> 72df7a2251SAndrew Turner #endif 73df7a2251SAndrew Turner 742b3ad188SAdrian Chadd #include <machine/atomic.h> 752b3ad188SAdrian Chadd #include <machine/cpu.h> 762b3ad188SAdrian Chadd #include <machine/smp.h> 772b3ad188SAdrian Chadd #include <machine/stdarg.h> 782b3ad188SAdrian Chadd 792b3ad188SAdrian Chadd #ifdef DDB 802b3ad188SAdrian Chadd #include <ddb/ddb.h> 812b3ad188SAdrian Chadd #endif 822b3ad188SAdrian Chadd 83e707c8beSRuslan Bukin #ifdef IOMMU 84e707c8beSRuslan Bukin #include <dev/iommu/iommu_msi.h> 85e707c8beSRuslan Bukin #endif 86e707c8beSRuslan Bukin 872b3ad188SAdrian Chadd #include "pic_if.h" 883fc155dcSAndrew Turner #include "msi_if.h" 892b3ad188SAdrian Chadd 902b3ad188SAdrian Chadd #define INTRNAME_LEN (2*MAXCOMLEN + 1) 912b3ad188SAdrian Chadd 922b3ad188SAdrian Chadd #ifdef DEBUG 932b3ad188SAdrian Chadd #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 942b3ad188SAdrian Chadd printf(fmt,##args); } while (0) 952b3ad188SAdrian Chadd #else 962b3ad188SAdrian Chadd #define debugf(fmt, args...) 972b3ad188SAdrian Chadd #endif 982b3ad188SAdrian Chadd 992b3ad188SAdrian Chadd MALLOC_DECLARE(M_INTRNG); 1002b3ad188SAdrian Chadd MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 1012b3ad188SAdrian Chadd 1022b3ad188SAdrian Chadd /* Root interrupt controller stuff. */ 10385918bebSAyrton Munoz struct intr_irq_root { 10485918bebSAyrton Munoz device_t dev; 10585918bebSAyrton Munoz intr_irq_filter_t *filter; 10685918bebSAyrton Munoz void *arg; 10785918bebSAyrton Munoz }; 10885918bebSAyrton Munoz 109*487788a6SElliott Mitchell static struct intr_irq_root intr_irq_roots[INTR_ROOT_COUNT]; 1102b3ad188SAdrian Chadd 111d1605cdaSAndrew Turner struct intr_pic_child { 112d1605cdaSAndrew Turner SLIST_ENTRY(intr_pic_child) pc_next; 113d1605cdaSAndrew Turner struct intr_pic *pc_pic; 114d1605cdaSAndrew Turner intr_child_irq_filter_t *pc_filter; 115d1605cdaSAndrew Turner void *pc_filter_arg; 116d1605cdaSAndrew Turner uintptr_t pc_start; 117d1605cdaSAndrew Turner uintptr_t pc_length; 118d1605cdaSAndrew Turner }; 119d1605cdaSAndrew Turner 1202b3ad188SAdrian Chadd /* Interrupt controller definition. */ 1212b3ad188SAdrian Chadd struct intr_pic { 1222b3ad188SAdrian Chadd SLIST_ENTRY(intr_pic) pic_next; 1232b3ad188SAdrian Chadd intptr_t pic_xref; /* hardware identification */ 1242b3ad188SAdrian Chadd device_t pic_dev; 125c0d52370SAndrew Turner /* Only one of FLAG_PIC or FLAG_MSI may be set */ 1263fc155dcSAndrew Turner #define FLAG_PIC (1 << 0) 1273fc155dcSAndrew Turner #define FLAG_MSI (1 << 1) 128c0d52370SAndrew Turner #define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) 1293fc155dcSAndrew Turner u_int pic_flags; 130d1605cdaSAndrew Turner struct mtx pic_child_lock; 131d1605cdaSAndrew Turner SLIST_HEAD(, intr_pic_child) pic_children; 1322b3ad188SAdrian Chadd }; 1332b3ad188SAdrian Chadd 134fae8755fSJessica Clarke #ifdef SMP 135fae8755fSJessica Clarke #define INTR_IPI_NAMELEN (MAXCOMLEN + 1) 136fae8755fSJessica Clarke 137fae8755fSJessica Clarke struct intr_ipi { 138fae8755fSJessica Clarke intr_ipi_handler_t *ii_handler; 139fae8755fSJessica Clarke void *ii_handler_arg; 140fae8755fSJessica Clarke struct intr_irqsrc *ii_isrc; 141fae8755fSJessica Clarke char ii_name[INTR_IPI_NAMELEN]; 142fae8755fSJessica Clarke u_long *ii_count; 143fae8755fSJessica Clarke }; 144103d39efSJessica Clarke 145103d39efSJessica Clarke static device_t intr_ipi_dev; 146103d39efSJessica Clarke static u_int intr_ipi_dev_priority; 147103d39efSJessica Clarke static bool intr_ipi_dev_frozen; 148fae8755fSJessica Clarke #endif 149fae8755fSJessica Clarke 1502b3ad188SAdrian Chadd static struct mtx pic_list_lock; 1512b3ad188SAdrian Chadd static SLIST_HEAD(, intr_pic) pic_list; 1522b3ad188SAdrian Chadd 1539f3a552fSElliott Mitchell static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, u_int flags); 1542b3ad188SAdrian Chadd 1552b3ad188SAdrian Chadd /* Interrupt source definition. */ 1562b3ad188SAdrian Chadd static struct mtx isrc_table_lock; 157248f0cabSOleksandr Tymoshenko static struct intr_irqsrc **irq_sources; 1581e0ba9d4SAndrew Turner static u_int irq_next_free; 1592b3ad188SAdrian Chadd 1602b3ad188SAdrian Chadd #ifdef SMP 161dc425090SMitchell Horne #ifdef EARLY_AP_STARTUP 162dc425090SMitchell Horne static bool irq_assign_cpu = true; 163dc425090SMitchell Horne #else 164dc425090SMitchell Horne static bool irq_assign_cpu = false; 165dc425090SMitchell Horne #endif 166fae8755fSJessica Clarke 167fae8755fSJessica Clarke static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; 1682b3ad188SAdrian Chadd #endif 1692b3ad188SAdrian Chadd 170a3c7da3dSElliott Mitchell u_int intr_nirq = NIRQ; 171248f0cabSOleksandr Tymoshenko SYSCTL_UINT(_machdep, OID_AUTO, nirq, CTLFLAG_RDTUN, &intr_nirq, 0, 172248f0cabSOleksandr Tymoshenko "Number of IRQs"); 1732b3ad188SAdrian Chadd 1742b3ad188SAdrian Chadd /* Data for MI statistics reporting. */ 175248f0cabSOleksandr Tymoshenko u_long *intrcnt; 176248f0cabSOleksandr Tymoshenko char *intrnames; 177248f0cabSOleksandr Tymoshenko size_t sintrcnt; 178248f0cabSOleksandr Tymoshenko size_t sintrnames; 17928137bdbSMitchell Horne int nintrcnt; 18028137bdbSMitchell Horne static bitstr_t *intrcnt_bitmap; 1812b3ad188SAdrian Chadd 182895c8b1cSMichal Meloun static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); 183895c8b1cSMichal Meloun static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); 184609b0fe9SOleksandr Tymoshenko static struct intr_map_data * intr_map_get_map_data(u_int res_id); 185895c8b1cSMichal Meloun static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, 186895c8b1cSMichal Meloun struct intr_map_data **data); 187895c8b1cSMichal Meloun 1882b3ad188SAdrian Chadd /* 1892b3ad188SAdrian Chadd * Interrupt framework initialization routine. 1902b3ad188SAdrian Chadd */ 1912b3ad188SAdrian Chadd static void 1922b3ad188SAdrian Chadd intr_irq_init(void *dummy __unused) 1932b3ad188SAdrian Chadd { 1942b3ad188SAdrian Chadd 1952b3ad188SAdrian Chadd SLIST_INIT(&pic_list); 1962b3ad188SAdrian Chadd mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 1973fc155dcSAndrew Turner 1982b3ad188SAdrian Chadd mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 199248f0cabSOleksandr Tymoshenko 200248f0cabSOleksandr Tymoshenko /* 201248f0cabSOleksandr Tymoshenko * - 2 counters for each I/O interrupt. 2022f0b059eSElliott Mitchell * - mp_maxid + 1 counters for each IPI counters for SMP. 203248f0cabSOleksandr Tymoshenko */ 20428137bdbSMitchell Horne nintrcnt = intr_nirq * 2; 205248f0cabSOleksandr Tymoshenko #ifdef SMP 2062f0b059eSElliott Mitchell nintrcnt += INTR_IPI_COUNT * (mp_maxid + 1); 207248f0cabSOleksandr Tymoshenko #endif 208248f0cabSOleksandr Tymoshenko 20928137bdbSMitchell Horne intrcnt = mallocarray(nintrcnt, sizeof(u_long), M_INTRNG, 210248f0cabSOleksandr Tymoshenko M_WAITOK | M_ZERO); 21128137bdbSMitchell Horne intrnames = mallocarray(nintrcnt, INTRNAME_LEN, M_INTRNG, 212248f0cabSOleksandr Tymoshenko M_WAITOK | M_ZERO); 21328137bdbSMitchell Horne sintrcnt = nintrcnt * sizeof(u_long); 21428137bdbSMitchell Horne sintrnames = nintrcnt * INTRNAME_LEN; 21528137bdbSMitchell Horne 21628137bdbSMitchell Horne /* Allocate the bitmap tracking counter allocations. */ 21728137bdbSMitchell Horne intrcnt_bitmap = bit_alloc(nintrcnt, M_INTRNG, M_WAITOK | M_ZERO); 21828137bdbSMitchell Horne 219248f0cabSOleksandr Tymoshenko irq_sources = mallocarray(intr_nirq, sizeof(struct intr_irqsrc*), 220248f0cabSOleksandr Tymoshenko M_INTRNG, M_WAITOK | M_ZERO); 2212b3ad188SAdrian Chadd } 2222b3ad188SAdrian Chadd SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 2232b3ad188SAdrian Chadd 2242b3ad188SAdrian Chadd static void 2252b3ad188SAdrian Chadd intrcnt_setname(const char *name, int index) 2262b3ad188SAdrian Chadd { 2272b3ad188SAdrian Chadd 2282b3ad188SAdrian Chadd snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 2292b3ad188SAdrian Chadd INTRNAME_LEN - 1, name); 2302b3ad188SAdrian Chadd } 2312b3ad188SAdrian Chadd 2322b3ad188SAdrian Chadd /* 2332b3ad188SAdrian Chadd * Update name for interrupt source with interrupt event. 2342b3ad188SAdrian Chadd */ 2352b3ad188SAdrian Chadd static void 2362b3ad188SAdrian Chadd intrcnt_updatename(struct intr_irqsrc *isrc) 2372b3ad188SAdrian Chadd { 2382b3ad188SAdrian Chadd 2392b3ad188SAdrian Chadd /* QQQ: What about stray counter name? */ 2402b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 2412b3ad188SAdrian Chadd intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 2422b3ad188SAdrian Chadd } 2432b3ad188SAdrian Chadd 2442b3ad188SAdrian Chadd /* 2452b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counter increment. 2462b3ad188SAdrian Chadd */ 2472b3ad188SAdrian Chadd static inline void 2482b3ad188SAdrian Chadd isrc_increment_count(struct intr_irqsrc *isrc) 2492b3ad188SAdrian Chadd { 2502b3ad188SAdrian Chadd 251bff6be3eSSvatopluk Kraus if (isrc->isrc_flags & INTR_ISRCF_PPI) 252bff6be3eSSvatopluk Kraus atomic_add_long(&isrc->isrc_count[0], 1); 253bff6be3eSSvatopluk Kraus else 2542b3ad188SAdrian Chadd isrc->isrc_count[0]++; 2552b3ad188SAdrian Chadd } 2562b3ad188SAdrian Chadd 2572b3ad188SAdrian Chadd /* 2582b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt stray counter increment. 2592b3ad188SAdrian Chadd */ 2602b3ad188SAdrian Chadd static inline void 2612b3ad188SAdrian Chadd isrc_increment_straycount(struct intr_irqsrc *isrc) 2622b3ad188SAdrian Chadd { 2632b3ad188SAdrian Chadd 2642b3ad188SAdrian Chadd isrc->isrc_count[1]++; 2652b3ad188SAdrian Chadd } 2662b3ad188SAdrian Chadd 2672b3ad188SAdrian Chadd /* 2682b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt name update. 2692b3ad188SAdrian Chadd */ 2702b3ad188SAdrian Chadd static void 2712b3ad188SAdrian Chadd isrc_update_name(struct intr_irqsrc *isrc, const char *name) 2722b3ad188SAdrian Chadd { 2732b3ad188SAdrian Chadd char str[INTRNAME_LEN]; 2742b3ad188SAdrian Chadd 2752b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 2762b3ad188SAdrian Chadd 2772b3ad188SAdrian Chadd if (name != NULL) { 2782b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 2792b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 2802b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 2812b3ad188SAdrian Chadd name); 2822b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 2832b3ad188SAdrian Chadd } else { 2842b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 2852b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 2862b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 2872b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 2882b3ad188SAdrian Chadd } 2892b3ad188SAdrian Chadd } 2902b3ad188SAdrian Chadd 2912b3ad188SAdrian Chadd /* 2922b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counters setup. 2932b3ad188SAdrian Chadd */ 2942b3ad188SAdrian Chadd static void 2952b3ad188SAdrian Chadd isrc_setup_counters(struct intr_irqsrc *isrc) 2962b3ad188SAdrian Chadd { 29728137bdbSMitchell Horne int index; 29828137bdbSMitchell Horne 29928137bdbSMitchell Horne mtx_assert(&isrc_table_lock, MA_OWNED); 3002b3ad188SAdrian Chadd 3012b3ad188SAdrian Chadd /* 30228137bdbSMitchell Horne * Allocate two counter values, the second tracking "stray" interrupts. 3032b3ad188SAdrian Chadd */ 30428137bdbSMitchell Horne bit_ffc_area(intrcnt_bitmap, nintrcnt, 2, &index); 30528137bdbSMitchell Horne if (index == -1) 30628137bdbSMitchell Horne panic("Failed to allocate 2 counters. Array exhausted?"); 30728137bdbSMitchell Horne bit_nset(intrcnt_bitmap, index, index + 1); 3082b3ad188SAdrian Chadd isrc->isrc_index = index; 3092b3ad188SAdrian Chadd isrc->isrc_count = &intrcnt[index]; 3102b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 3112b3ad188SAdrian Chadd } 3122b3ad188SAdrian Chadd 313bff6be3eSSvatopluk Kraus /* 314bff6be3eSSvatopluk Kraus * Virtualization for interrupt source interrupt counters release. 315bff6be3eSSvatopluk Kraus */ 316bff6be3eSSvatopluk Kraus static void 317bff6be3eSSvatopluk Kraus isrc_release_counters(struct intr_irqsrc *isrc) 318bff6be3eSSvatopluk Kraus { 31928137bdbSMitchell Horne int idx = isrc->isrc_index; 320bff6be3eSSvatopluk Kraus 32128137bdbSMitchell Horne mtx_assert(&isrc_table_lock, MA_OWNED); 32228137bdbSMitchell Horne 32328137bdbSMitchell Horne bit_nclear(intrcnt_bitmap, idx, idx + 1); 324bff6be3eSSvatopluk Kraus } 325bff6be3eSSvatopluk Kraus 3262b3ad188SAdrian Chadd /* 3272b3ad188SAdrian Chadd * Main interrupt dispatch handler. It's called straight 3282b3ad188SAdrian Chadd * from the assembler, where CPU interrupt is served. 3292b3ad188SAdrian Chadd */ 3302b3ad188SAdrian Chadd void 3314b01a7faSKyle Evans intr_irq_handler(struct trapframe *tf, uint32_t rootnum) 3322b3ad188SAdrian Chadd { 3332b3ad188SAdrian Chadd struct trapframe * oldframe; 3342b3ad188SAdrian Chadd struct thread * td; 33585918bebSAyrton Munoz struct intr_irq_root *root; 3362b3ad188SAdrian Chadd 337*487788a6SElliott Mitchell KASSERT(rootnum < INTR_ROOT_COUNT, 3384b01a7faSKyle Evans ("%s: invalid interrupt root %d", __func__, rootnum)); 33985918bebSAyrton Munoz 3404b01a7faSKyle Evans root = &intr_irq_roots[rootnum]; 34185918bebSAyrton Munoz KASSERT(root->filter != NULL, ("%s: no filter", __func__)); 3422b3ad188SAdrian Chadd 34389c52f9dSKyle Evans kasan_mark(tf, sizeof(*tf), sizeof(*tf), 0); 344c05d7bdaSMark Johnston kmsan_mark(tf, sizeof(*tf), KMSAN_STATE_INITED); 34589c52f9dSKyle Evans 34683c9dea1SGleb Smirnoff VM_CNT_INC(v_intr); 3472b3ad188SAdrian Chadd critical_enter(); 3482b3ad188SAdrian Chadd td = curthread; 3492b3ad188SAdrian Chadd oldframe = td->td_intr_frame; 3502b3ad188SAdrian Chadd td->td_intr_frame = tf; 35185918bebSAyrton Munoz (root->filter)(root->arg); 3522b3ad188SAdrian Chadd td->td_intr_frame = oldframe; 3532b3ad188SAdrian Chadd critical_exit(); 354df7a2251SAndrew Turner #ifdef HWPMC_HOOKS 355974692e3SAndrew Turner if (pmc_hook && TRAPF_USERMODE(tf) && 356974692e3SAndrew Turner (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) 357df7a2251SAndrew Turner pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); 358df7a2251SAndrew Turner #endif 3592b3ad188SAdrian Chadd } 3602b3ad188SAdrian Chadd 361d1605cdaSAndrew Turner int 362d1605cdaSAndrew Turner intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) 363d1605cdaSAndrew Turner { 364d1605cdaSAndrew Turner struct intr_pic_child *child; 365d1605cdaSAndrew Turner bool found; 366d1605cdaSAndrew Turner 367d1605cdaSAndrew Turner found = false; 368d1605cdaSAndrew Turner mtx_lock_spin(&parent->pic_child_lock); 369d1605cdaSAndrew Turner SLIST_FOREACH(child, &parent->pic_children, pc_next) { 370d1605cdaSAndrew Turner if (child->pc_start <= irq && 371d1605cdaSAndrew Turner irq < (child->pc_start + child->pc_length)) { 372d1605cdaSAndrew Turner found = true; 373d1605cdaSAndrew Turner break; 374d1605cdaSAndrew Turner } 375d1605cdaSAndrew Turner } 376d1605cdaSAndrew Turner mtx_unlock_spin(&parent->pic_child_lock); 377d1605cdaSAndrew Turner 378d1605cdaSAndrew Turner if (found) 379d1605cdaSAndrew Turner return (child->pc_filter(child->pc_filter_arg, irq)); 380d1605cdaSAndrew Turner 381d1605cdaSAndrew Turner return (FILTER_STRAY); 382d1605cdaSAndrew Turner } 383d1605cdaSAndrew Turner 3842b3ad188SAdrian Chadd /* 3852b3ad188SAdrian Chadd * interrupt controller dispatch function for interrupts. It should 3862b3ad188SAdrian Chadd * be called straight from the interrupt controller, when associated interrupt 3872b3ad188SAdrian Chadd * source is learned. 3882b3ad188SAdrian Chadd */ 389bff6be3eSSvatopluk Kraus int 390bff6be3eSSvatopluk Kraus intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 3912b3ad188SAdrian Chadd { 3922b3ad188SAdrian Chadd 3932b3ad188SAdrian Chadd KASSERT(isrc != NULL, ("%s: no source", __func__)); 3942b3ad188SAdrian Chadd 395103d39efSJessica Clarke if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 3962b3ad188SAdrian Chadd isrc_increment_count(isrc); 3972b3ad188SAdrian Chadd 3982b3ad188SAdrian Chadd #ifdef INTR_SOLO 3992b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 4002b3ad188SAdrian Chadd int error; 4012b3ad188SAdrian Chadd error = isrc->isrc_filter(isrc->isrc_arg, tf); 4022b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 4032b3ad188SAdrian Chadd if (error == FILTER_HANDLED) 404bff6be3eSSvatopluk Kraus return (0); 4052b3ad188SAdrian Chadd } else 4062b3ad188SAdrian Chadd #endif 4072b3ad188SAdrian Chadd if (isrc->isrc_event != NULL) { 4082b3ad188SAdrian Chadd if (intr_event_handle(isrc->isrc_event, tf) == 0) 409bff6be3eSSvatopluk Kraus return (0); 4102b3ad188SAdrian Chadd } 4112b3ad188SAdrian Chadd 412103d39efSJessica Clarke if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 4132b3ad188SAdrian Chadd isrc_increment_straycount(isrc); 414bff6be3eSSvatopluk Kraus return (EINVAL); 4152b3ad188SAdrian Chadd } 4162b3ad188SAdrian Chadd 4172b3ad188SAdrian Chadd /* 4182b3ad188SAdrian Chadd * Alloc unique interrupt number (resource handle) for interrupt source. 4192b3ad188SAdrian Chadd * 4202b3ad188SAdrian Chadd * There could be various strategies how to allocate free interrupt number 4212b3ad188SAdrian Chadd * (resource handle) for new interrupt source. 4222b3ad188SAdrian Chadd * 4232b3ad188SAdrian Chadd * 1. Handles are always allocated forward, so handles are not recycled 4242b3ad188SAdrian Chadd * immediately. However, if only one free handle left which is reused 4252b3ad188SAdrian Chadd * constantly... 4262b3ad188SAdrian Chadd */ 427bff6be3eSSvatopluk Kraus static inline int 428bff6be3eSSvatopluk Kraus isrc_alloc_irq(struct intr_irqsrc *isrc) 4292b3ad188SAdrian Chadd { 430e88c3b1bSMichal Meloun u_int irq; 4312b3ad188SAdrian Chadd 4322b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 4332b3ad188SAdrian Chadd 434e88c3b1bSMichal Meloun if (irq_next_free >= intr_nirq) 4352b3ad188SAdrian Chadd return (ENOSPC); 4362b3ad188SAdrian Chadd 437e88c3b1bSMichal Meloun for (irq = irq_next_free; irq < intr_nirq; irq++) { 4382b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 4392b3ad188SAdrian Chadd goto found; 4402b3ad188SAdrian Chadd } 4412b3ad188SAdrian Chadd for (irq = 0; irq < irq_next_free; irq++) { 4422b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 4432b3ad188SAdrian Chadd goto found; 4442b3ad188SAdrian Chadd } 4452b3ad188SAdrian Chadd 446e88c3b1bSMichal Meloun irq_next_free = intr_nirq; 4472b3ad188SAdrian Chadd return (ENOSPC); 4482b3ad188SAdrian Chadd 4492b3ad188SAdrian Chadd found: 4502b3ad188SAdrian Chadd isrc->isrc_irq = irq; 4512b3ad188SAdrian Chadd irq_sources[irq] = isrc; 4522b3ad188SAdrian Chadd 4532b3ad188SAdrian Chadd irq_next_free = irq + 1; 454e88c3b1bSMichal Meloun if (irq_next_free >= intr_nirq) 4552b3ad188SAdrian Chadd irq_next_free = 0; 4562b3ad188SAdrian Chadd return (0); 4572b3ad188SAdrian Chadd } 458bff6be3eSSvatopluk Kraus 4592b3ad188SAdrian Chadd /* 4602b3ad188SAdrian Chadd * Free unique interrupt number (resource handle) from interrupt source. 4612b3ad188SAdrian Chadd */ 462bff6be3eSSvatopluk Kraus static inline int 4632b3ad188SAdrian Chadd isrc_free_irq(struct intr_irqsrc *isrc) 4642b3ad188SAdrian Chadd { 4652b3ad188SAdrian Chadd 466bff6be3eSSvatopluk Kraus mtx_assert(&isrc_table_lock, MA_OWNED); 4672b3ad188SAdrian Chadd 468248f0cabSOleksandr Tymoshenko if (isrc->isrc_irq >= intr_nirq) 4692b3ad188SAdrian Chadd return (EINVAL); 470bff6be3eSSvatopluk Kraus if (irq_sources[isrc->isrc_irq] != isrc) 4712b3ad188SAdrian Chadd return (EINVAL); 4722b3ad188SAdrian Chadd 4732b3ad188SAdrian Chadd irq_sources[isrc->isrc_irq] = NULL; 4748442087fSMichal Meloun isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 475a49f208dSMichal Meloun 476a49f208dSMichal Meloun /* 477a49f208dSMichal Meloun * If we are recovering from the state irq_sources table is full, 478a49f208dSMichal Meloun * then the following allocation should check the entire table. This 479a49f208dSMichal Meloun * will ensure maximum separation of allocation order from release 480a49f208dSMichal Meloun * order. 481a49f208dSMichal Meloun */ 482a49f208dSMichal Meloun if (irq_next_free >= intr_nirq) 483a49f208dSMichal Meloun irq_next_free = 0; 484a49f208dSMichal Meloun 4852b3ad188SAdrian Chadd return (0); 4862b3ad188SAdrian Chadd } 487bff6be3eSSvatopluk Kraus 48885918bebSAyrton Munoz device_t 4894b01a7faSKyle Evans intr_irq_root_device(uint32_t rootnum) 49085918bebSAyrton Munoz { 491*487788a6SElliott Mitchell KASSERT(rootnum < INTR_ROOT_COUNT, 4924b01a7faSKyle Evans ("%s: invalid interrupt root %d", __func__, rootnum)); 4934b01a7faSKyle Evans return (intr_irq_roots[rootnum].dev); 49485918bebSAyrton Munoz } 49585918bebSAyrton Munoz 4962b3ad188SAdrian Chadd /* 497bff6be3eSSvatopluk Kraus * Initialize interrupt source and register it into global interrupt table. 4982b3ad188SAdrian Chadd */ 499bff6be3eSSvatopluk Kraus int 500bff6be3eSSvatopluk Kraus intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, 501bff6be3eSSvatopluk Kraus const char *fmt, ...) 5022b3ad188SAdrian Chadd { 503bff6be3eSSvatopluk Kraus int error; 504bff6be3eSSvatopluk Kraus va_list ap; 5052b3ad188SAdrian Chadd 506bff6be3eSSvatopluk Kraus bzero(isrc, sizeof(struct intr_irqsrc)); 507bff6be3eSSvatopluk Kraus isrc->isrc_dev = dev; 5088442087fSMichal Meloun isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 509bff6be3eSSvatopluk Kraus isrc->isrc_flags = flags; 5102b3ad188SAdrian Chadd 511bff6be3eSSvatopluk Kraus va_start(ap, fmt); 512bff6be3eSSvatopluk Kraus vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 513bff6be3eSSvatopluk Kraus va_end(ap); 514bff6be3eSSvatopluk Kraus 515bff6be3eSSvatopluk Kraus mtx_lock(&isrc_table_lock); 516bff6be3eSSvatopluk Kraus error = isrc_alloc_irq(isrc); 517bff6be3eSSvatopluk Kraus if (error != 0) { 518bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 519bff6be3eSSvatopluk Kraus return (error); 5202b3ad188SAdrian Chadd } 521bff6be3eSSvatopluk Kraus /* 522bff6be3eSSvatopluk Kraus * Setup interrupt counters, but not for IPI sources. Those are setup 523bff6be3eSSvatopluk Kraus * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust 524bff6be3eSSvatopluk Kraus * our counter pool. 525bff6be3eSSvatopluk Kraus */ 526bff6be3eSSvatopluk Kraus if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 527bff6be3eSSvatopluk Kraus isrc_setup_counters(isrc); 528bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 529bff6be3eSSvatopluk Kraus return (0); 5302b3ad188SAdrian Chadd } 5312b3ad188SAdrian Chadd 5322b3ad188SAdrian Chadd /* 533bff6be3eSSvatopluk Kraus * Deregister interrupt source from global interrupt table. 534bff6be3eSSvatopluk Kraus */ 535bff6be3eSSvatopluk Kraus int 536bff6be3eSSvatopluk Kraus intr_isrc_deregister(struct intr_irqsrc *isrc) 537bff6be3eSSvatopluk Kraus { 538bff6be3eSSvatopluk Kraus int error; 539bff6be3eSSvatopluk Kraus 540bff6be3eSSvatopluk Kraus mtx_lock(&isrc_table_lock); 541bff6be3eSSvatopluk Kraus if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 542bff6be3eSSvatopluk Kraus isrc_release_counters(isrc); 543bff6be3eSSvatopluk Kraus error = isrc_free_irq(isrc); 544bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 545bff6be3eSSvatopluk Kraus return (error); 546bff6be3eSSvatopluk Kraus } 547bff6be3eSSvatopluk Kraus 5485b613c19SSvatopluk Kraus #ifdef SMP 5495b613c19SSvatopluk Kraus /* 5505b613c19SSvatopluk Kraus * A support function for a PIC to decide if provided ISRC should be inited 5515b613c19SSvatopluk Kraus * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of 5525b613c19SSvatopluk Kraus * struct intr_irqsrc is the following: 5535b613c19SSvatopluk Kraus * 5545b613c19SSvatopluk Kraus * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus 5555b613c19SSvatopluk Kraus * set in isrc_cpu. If not, the ISRC should be inited on every cpu and 5565b613c19SSvatopluk Kraus * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. 5575b613c19SSvatopluk Kraus */ 5585b613c19SSvatopluk Kraus bool 5595b613c19SSvatopluk Kraus intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) 5605b613c19SSvatopluk Kraus { 5615b613c19SSvatopluk Kraus 5625b613c19SSvatopluk Kraus if (isrc->isrc_handlers == 0) 5635b613c19SSvatopluk Kraus return (false); 5645b613c19SSvatopluk Kraus if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) 5655b613c19SSvatopluk Kraus return (false); 5665b613c19SSvatopluk Kraus if (isrc->isrc_flags & INTR_ISRCF_BOUND) 5675b613c19SSvatopluk Kraus return (CPU_ISSET(cpu, &isrc->isrc_cpu)); 5685b613c19SSvatopluk Kraus 5695b613c19SSvatopluk Kraus CPU_SET(cpu, &isrc->isrc_cpu); 5705b613c19SSvatopluk Kraus return (true); 5715b613c19SSvatopluk Kraus } 5725b613c19SSvatopluk Kraus #endif 5735b613c19SSvatopluk Kraus 5742b3ad188SAdrian Chadd #ifdef INTR_SOLO 5752b3ad188SAdrian Chadd /* 5762b3ad188SAdrian Chadd * Setup filter into interrupt source. 5772b3ad188SAdrian Chadd */ 5782b3ad188SAdrian Chadd static int 5792b3ad188SAdrian Chadd iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 5802b3ad188SAdrian Chadd intr_irq_filter_t *filter, void *arg, void **cookiep) 5812b3ad188SAdrian Chadd { 5822b3ad188SAdrian Chadd 5832b3ad188SAdrian Chadd if (filter == NULL) 5842b3ad188SAdrian Chadd return (EINVAL); 5852b3ad188SAdrian Chadd 5862b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 5872b3ad188SAdrian Chadd /* 5882b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 5892b3ad188SAdrian Chadd * how we handle interrupt sources. 5902b3ad188SAdrian Chadd */ 5912b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 5922b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 5932b3ad188SAdrian Chadd return (EBUSY); 5942b3ad188SAdrian Chadd } 5952b3ad188SAdrian Chadd isrc->isrc_filter = filter; 5962b3ad188SAdrian Chadd isrc->isrc_arg = arg; 5972b3ad188SAdrian Chadd isrc_update_name(isrc, name); 5982b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 5992b3ad188SAdrian Chadd 6002b3ad188SAdrian Chadd *cookiep = isrc; 6012b3ad188SAdrian Chadd return (0); 6022b3ad188SAdrian Chadd } 6032b3ad188SAdrian Chadd #endif 6042b3ad188SAdrian Chadd 6052b3ad188SAdrian Chadd /* 6062b3ad188SAdrian Chadd * Interrupt source pre_ithread method for MI interrupt framework. 6072b3ad188SAdrian Chadd */ 6082b3ad188SAdrian Chadd static void 6092b3ad188SAdrian Chadd intr_isrc_pre_ithread(void *arg) 6102b3ad188SAdrian Chadd { 6112b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 6122b3ad188SAdrian Chadd 6132b3ad188SAdrian Chadd PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 6142b3ad188SAdrian Chadd } 6152b3ad188SAdrian Chadd 6162b3ad188SAdrian Chadd /* 6172b3ad188SAdrian Chadd * Interrupt source post_ithread method for MI interrupt framework. 6182b3ad188SAdrian Chadd */ 6192b3ad188SAdrian Chadd static void 6202b3ad188SAdrian Chadd intr_isrc_post_ithread(void *arg) 6212b3ad188SAdrian Chadd { 6222b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 6232b3ad188SAdrian Chadd 6242b3ad188SAdrian Chadd PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 6252b3ad188SAdrian Chadd } 6262b3ad188SAdrian Chadd 6272b3ad188SAdrian Chadd /* 6282b3ad188SAdrian Chadd * Interrupt source post_filter method for MI interrupt framework. 6292b3ad188SAdrian Chadd */ 6302b3ad188SAdrian Chadd static void 6312b3ad188SAdrian Chadd intr_isrc_post_filter(void *arg) 6322b3ad188SAdrian Chadd { 6332b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 6342b3ad188SAdrian Chadd 6352b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 6362b3ad188SAdrian Chadd } 6372b3ad188SAdrian Chadd 6382b3ad188SAdrian Chadd /* 6392b3ad188SAdrian Chadd * Interrupt source assign_cpu method for MI interrupt framework. 6402b3ad188SAdrian Chadd */ 6412b3ad188SAdrian Chadd static int 6422b3ad188SAdrian Chadd intr_isrc_assign_cpu(void *arg, int cpu) 6432b3ad188SAdrian Chadd { 6442b3ad188SAdrian Chadd #ifdef SMP 6452b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 6462b3ad188SAdrian Chadd int error; 6472b3ad188SAdrian Chadd 6482b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 6492b3ad188SAdrian Chadd if (cpu == NOCPU) { 6502b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 6512b3ad188SAdrian Chadd isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 6522b3ad188SAdrian Chadd } else { 6532b3ad188SAdrian Chadd CPU_SETOF(cpu, &isrc->isrc_cpu); 6542b3ad188SAdrian Chadd isrc->isrc_flags |= INTR_ISRCF_BOUND; 6552b3ad188SAdrian Chadd } 6562b3ad188SAdrian Chadd 6572b3ad188SAdrian Chadd /* 6582b3ad188SAdrian Chadd * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 6592b3ad188SAdrian Chadd * re-balance it to another CPU or enable it on more CPUs. However, 6602b3ad188SAdrian Chadd * PIC is expected to change isrc_cpu appropriately to keep us well 661e3043798SPedro F. Giffuni * informed if the call is successful. 6622b3ad188SAdrian Chadd */ 6632b3ad188SAdrian Chadd if (irq_assign_cpu) { 664bff6be3eSSvatopluk Kraus error = PIC_BIND_INTR(isrc->isrc_dev, isrc); 6652b3ad188SAdrian Chadd if (error) { 6662b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 6672b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6682b3ad188SAdrian Chadd return (error); 6692b3ad188SAdrian Chadd } 6702b3ad188SAdrian Chadd } 6712b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6722b3ad188SAdrian Chadd return (0); 6732b3ad188SAdrian Chadd #else 6742b3ad188SAdrian Chadd return (EOPNOTSUPP); 6752b3ad188SAdrian Chadd #endif 6762b3ad188SAdrian Chadd } 6772b3ad188SAdrian Chadd 6782b3ad188SAdrian Chadd /* 6792b3ad188SAdrian Chadd * Create interrupt event for interrupt source. 6802b3ad188SAdrian Chadd */ 6812b3ad188SAdrian Chadd static int 6822b3ad188SAdrian Chadd isrc_event_create(struct intr_irqsrc *isrc) 6832b3ad188SAdrian Chadd { 6842b3ad188SAdrian Chadd struct intr_event *ie; 6852b3ad188SAdrian Chadd int error; 6862b3ad188SAdrian Chadd 6872b3ad188SAdrian Chadd error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 6882b3ad188SAdrian Chadd intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 6892b3ad188SAdrian Chadd intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 6902b3ad188SAdrian Chadd if (error) 6912b3ad188SAdrian Chadd return (error); 6922b3ad188SAdrian Chadd 6932b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 6942b3ad188SAdrian Chadd /* 6952b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 6962b3ad188SAdrian Chadd * how we handle interrupt sources. Let contested event wins. 6972b3ad188SAdrian Chadd */ 698169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 6992b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 700169e6abdSSvatopluk Kraus #else 701169e6abdSSvatopluk Kraus if (isrc->isrc_event != NULL) { 702169e6abdSSvatopluk Kraus #endif 7032b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 7042b3ad188SAdrian Chadd intr_event_destroy(ie); 7052b3ad188SAdrian Chadd return (isrc->isrc_event != NULL ? EBUSY : 0); 7062b3ad188SAdrian Chadd } 7072b3ad188SAdrian Chadd isrc->isrc_event = ie; 7082b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 7092b3ad188SAdrian Chadd 7102b3ad188SAdrian Chadd return (0); 7112b3ad188SAdrian Chadd } 7122b3ad188SAdrian Chadd #ifdef notyet 7132b3ad188SAdrian Chadd /* 7142b3ad188SAdrian Chadd * Destroy interrupt event for interrupt source. 7152b3ad188SAdrian Chadd */ 7162b3ad188SAdrian Chadd static void 7172b3ad188SAdrian Chadd isrc_event_destroy(struct intr_irqsrc *isrc) 7182b3ad188SAdrian Chadd { 7192b3ad188SAdrian Chadd struct intr_event *ie; 7202b3ad188SAdrian Chadd 7212b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 7222b3ad188SAdrian Chadd ie = isrc->isrc_event; 7232b3ad188SAdrian Chadd isrc->isrc_event = NULL; 7242b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 7252b3ad188SAdrian Chadd 7262b3ad188SAdrian Chadd if (ie != NULL) 7272b3ad188SAdrian Chadd intr_event_destroy(ie); 7282b3ad188SAdrian Chadd } 7292b3ad188SAdrian Chadd #endif 7302b3ad188SAdrian Chadd /* 7312b3ad188SAdrian Chadd * Add handler to interrupt source. 7322b3ad188SAdrian Chadd */ 7332b3ad188SAdrian Chadd static int 7342b3ad188SAdrian Chadd isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 7352b3ad188SAdrian Chadd driver_filter_t filter, driver_intr_t handler, void *arg, 7362b3ad188SAdrian Chadd enum intr_type flags, void **cookiep) 7372b3ad188SAdrian Chadd { 7382b3ad188SAdrian Chadd int error; 7392b3ad188SAdrian Chadd 7402b3ad188SAdrian Chadd if (isrc->isrc_event == NULL) { 7412b3ad188SAdrian Chadd error = isrc_event_create(isrc); 7422b3ad188SAdrian Chadd if (error) 7432b3ad188SAdrian Chadd return (error); 7442b3ad188SAdrian Chadd } 7452b3ad188SAdrian Chadd 7462b3ad188SAdrian Chadd error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 7472b3ad188SAdrian Chadd arg, intr_priority(flags), flags, cookiep); 7482b3ad188SAdrian Chadd if (error == 0) { 7492b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 7502b3ad188SAdrian Chadd intrcnt_updatename(isrc); 7512b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 7522b3ad188SAdrian Chadd } 7532b3ad188SAdrian Chadd 7542b3ad188SAdrian Chadd return (error); 7552b3ad188SAdrian Chadd } 7562b3ad188SAdrian Chadd 7572b3ad188SAdrian Chadd /* 7582b3ad188SAdrian Chadd * Lookup interrupt controller locked. 7592b3ad188SAdrian Chadd */ 760bff6be3eSSvatopluk Kraus static inline struct intr_pic * 7619f3a552fSElliott Mitchell pic_lookup_locked(device_t dev, intptr_t xref, u_int flags) 7622b3ad188SAdrian Chadd { 7632b3ad188SAdrian Chadd struct intr_pic *pic; 7642b3ad188SAdrian Chadd 7652b3ad188SAdrian Chadd mtx_assert(&pic_list_lock, MA_OWNED); 7662b3ad188SAdrian Chadd 7674be58cbaSSvatopluk Kraus if (dev == NULL && xref == 0) 7684be58cbaSSvatopluk Kraus return (NULL); 7694be58cbaSSvatopluk Kraus 7704be58cbaSSvatopluk Kraus /* Note that pic->pic_dev is never NULL on registered PIC. */ 7712b3ad188SAdrian Chadd SLIST_FOREACH(pic, &pic_list, pic_next) { 772c0d52370SAndrew Turner if ((pic->pic_flags & FLAG_TYPE_MASK) != 773c0d52370SAndrew Turner (flags & FLAG_TYPE_MASK)) 774c0d52370SAndrew Turner continue; 775c0d52370SAndrew Turner 7764be58cbaSSvatopluk Kraus if (dev == NULL) { 7774be58cbaSSvatopluk Kraus if (xref == pic->pic_xref) 7784be58cbaSSvatopluk Kraus return (pic); 7794be58cbaSSvatopluk Kraus } else if (xref == 0 || pic->pic_xref == 0) { 7804be58cbaSSvatopluk Kraus if (dev == pic->pic_dev) 7814be58cbaSSvatopluk Kraus return (pic); 7824be58cbaSSvatopluk Kraus } else if (xref == pic->pic_xref && dev == pic->pic_dev) 7832b3ad188SAdrian Chadd return (pic); 7842b3ad188SAdrian Chadd } 7852b3ad188SAdrian Chadd return (NULL); 7862b3ad188SAdrian Chadd } 7872b3ad188SAdrian Chadd 7882b3ad188SAdrian Chadd /* 7892b3ad188SAdrian Chadd * Lookup interrupt controller. 7902b3ad188SAdrian Chadd */ 7912b3ad188SAdrian Chadd static struct intr_pic * 7929f3a552fSElliott Mitchell pic_lookup(device_t dev, intptr_t xref, u_int flags) 7932b3ad188SAdrian Chadd { 7942b3ad188SAdrian Chadd struct intr_pic *pic; 7952b3ad188SAdrian Chadd 7962b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 797c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 7982b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7992b3ad188SAdrian Chadd return (pic); 8002b3ad188SAdrian Chadd } 8012b3ad188SAdrian Chadd 8022b3ad188SAdrian Chadd /* 8032b3ad188SAdrian Chadd * Create interrupt controller. 8042b3ad188SAdrian Chadd */ 8052b3ad188SAdrian Chadd static struct intr_pic * 8069f3a552fSElliott Mitchell pic_create(device_t dev, intptr_t xref, u_int flags) 8072b3ad188SAdrian Chadd { 8082b3ad188SAdrian Chadd struct intr_pic *pic; 8092b3ad188SAdrian Chadd 8102b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 811c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 8122b3ad188SAdrian Chadd if (pic != NULL) { 8132b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 8142b3ad188SAdrian Chadd return (pic); 8152b3ad188SAdrian Chadd } 8162b3ad188SAdrian Chadd pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 817b48c6083SAndrew Turner if (pic == NULL) { 818b48c6083SAndrew Turner mtx_unlock(&pic_list_lock); 819b48c6083SAndrew Turner return (NULL); 820b48c6083SAndrew Turner } 8212b3ad188SAdrian Chadd pic->pic_xref = xref; 8222b3ad188SAdrian Chadd pic->pic_dev = dev; 823c0d52370SAndrew Turner pic->pic_flags = flags; 824d1605cdaSAndrew Turner mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); 8252b3ad188SAdrian Chadd SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 8262b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 8272b3ad188SAdrian Chadd 8282b3ad188SAdrian Chadd return (pic); 8292b3ad188SAdrian Chadd } 8302b3ad188SAdrian Chadd #ifdef notyet 8312b3ad188SAdrian Chadd /* 8322b3ad188SAdrian Chadd * Destroy interrupt controller. 8332b3ad188SAdrian Chadd */ 8342b3ad188SAdrian Chadd static void 8359f3a552fSElliott Mitchell pic_destroy(device_t dev, intptr_t xref, u_int flags) 8362b3ad188SAdrian Chadd { 8372b3ad188SAdrian Chadd struct intr_pic *pic; 8382b3ad188SAdrian Chadd 8392b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 840c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 8412b3ad188SAdrian Chadd if (pic == NULL) { 8422b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 8432b3ad188SAdrian Chadd return; 8442b3ad188SAdrian Chadd } 8452b3ad188SAdrian Chadd SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 8462b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 8472b3ad188SAdrian Chadd 8482b3ad188SAdrian Chadd free(pic, M_INTRNG); 8492b3ad188SAdrian Chadd } 8502b3ad188SAdrian Chadd #endif 8512b3ad188SAdrian Chadd /* 8522b3ad188SAdrian Chadd * Register interrupt controller. 8532b3ad188SAdrian Chadd */ 8549346e913SAndrew Turner struct intr_pic * 8552b3ad188SAdrian Chadd intr_pic_register(device_t dev, intptr_t xref) 8562b3ad188SAdrian Chadd { 8572b3ad188SAdrian Chadd struct intr_pic *pic; 8582b3ad188SAdrian Chadd 8594be58cbaSSvatopluk Kraus if (dev == NULL) 8609346e913SAndrew Turner return (NULL); 861c0d52370SAndrew Turner pic = pic_create(dev, xref, FLAG_PIC); 8622b3ad188SAdrian Chadd if (pic == NULL) 8639346e913SAndrew Turner return (NULL); 8642b3ad188SAdrian Chadd 865cff33fa8SEd Maste debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 866cff33fa8SEd Maste device_get_nameunit(dev), dev, (uintmax_t)xref); 8679346e913SAndrew Turner return (pic); 8682b3ad188SAdrian Chadd } 8692b3ad188SAdrian Chadd 8702b3ad188SAdrian Chadd /* 8712b3ad188SAdrian Chadd * Unregister interrupt controller. 8722b3ad188SAdrian Chadd */ 8732b3ad188SAdrian Chadd int 874bff6be3eSSvatopluk Kraus intr_pic_deregister(device_t dev, intptr_t xref) 8752b3ad188SAdrian Chadd { 8762b3ad188SAdrian Chadd 8772b3ad188SAdrian Chadd panic("%s: not implemented", __func__); 8782b3ad188SAdrian Chadd } 8792b3ad188SAdrian Chadd 8802b3ad188SAdrian Chadd /* 8812b3ad188SAdrian Chadd * Mark interrupt controller (itself) as a root one. 8822b3ad188SAdrian Chadd * 8832b3ad188SAdrian Chadd * Note that only an interrupt controller can really know its position 8842b3ad188SAdrian Chadd * in interrupt controller's tree. So root PIC must claim itself as a root. 8852b3ad188SAdrian Chadd * 8862b3ad188SAdrian Chadd * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 8872b3ad188SAdrian Chadd * page 30: 8882b3ad188SAdrian Chadd * "The root of the interrupt tree is determined when traversal 8892b3ad188SAdrian Chadd * of the interrupt tree reaches an interrupt controller node without 8902b3ad188SAdrian Chadd * an interrupts property and thus no explicit interrupt parent." 8912b3ad188SAdrian Chadd */ 8922b3ad188SAdrian Chadd int 8932b3ad188SAdrian Chadd intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 8944b01a7faSKyle Evans void *arg, uint32_t rootnum) 8952b3ad188SAdrian Chadd { 8963fc155dcSAndrew Turner struct intr_pic *pic; 89785918bebSAyrton Munoz struct intr_irq_root *root; 8982b3ad188SAdrian Chadd 899c0d52370SAndrew Turner pic = pic_lookup(dev, xref, FLAG_PIC); 9003fc155dcSAndrew Turner if (pic == NULL) { 9012b3ad188SAdrian Chadd device_printf(dev, "not registered\n"); 9022b3ad188SAdrian Chadd return (EINVAL); 9032b3ad188SAdrian Chadd } 9043fc155dcSAndrew Turner 905c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 9063fc155dcSAndrew Turner ("%s: Found a non-PIC controller: %s", __func__, 9073fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 9083fc155dcSAndrew Turner 9092b3ad188SAdrian Chadd if (filter == NULL) { 9102b3ad188SAdrian Chadd device_printf(dev, "filter missing\n"); 9112b3ad188SAdrian Chadd return (EINVAL); 9122b3ad188SAdrian Chadd } 9132b3ad188SAdrian Chadd 9142b3ad188SAdrian Chadd /* 9152b3ad188SAdrian Chadd * Only one interrupt controllers could be on the root for now. 9162b3ad188SAdrian Chadd * Note that we further suppose that there is not threaded interrupt 9172b3ad188SAdrian Chadd * routine (handler) on the root. See intr_irq_handler(). 9182b3ad188SAdrian Chadd */ 919*487788a6SElliott Mitchell KASSERT(rootnum < INTR_ROOT_COUNT, 9204b01a7faSKyle Evans ("%s: invalid interrupt root %d", __func__, rootnum)); 9214b01a7faSKyle Evans root = &intr_irq_roots[rootnum]; 92285918bebSAyrton Munoz if (root->dev != NULL) { 9232b3ad188SAdrian Chadd device_printf(dev, "another root already set\n"); 9242b3ad188SAdrian Chadd return (EBUSY); 9252b3ad188SAdrian Chadd } 9262b3ad188SAdrian Chadd 92785918bebSAyrton Munoz root->dev = dev; 92885918bebSAyrton Munoz root->filter = filter; 92985918bebSAyrton Munoz root->arg = arg; 9302b3ad188SAdrian Chadd 9312b3ad188SAdrian Chadd debugf("irq root set to %s\n", device_get_nameunit(dev)); 9322b3ad188SAdrian Chadd return (0); 9332b3ad188SAdrian Chadd } 9342b3ad188SAdrian Chadd 935d1605cdaSAndrew Turner /* 936d1605cdaSAndrew Turner * Add a handler to manage a sub range of a parents interrupts. 937d1605cdaSAndrew Turner */ 938a3e828c9SJessica Clarke int 939d1605cdaSAndrew Turner intr_pic_add_handler(device_t parent, struct intr_pic *pic, 940d1605cdaSAndrew Turner intr_child_irq_filter_t *filter, void *arg, uintptr_t start, 941d1605cdaSAndrew Turner uintptr_t length) 942d1605cdaSAndrew Turner { 943d1605cdaSAndrew Turner struct intr_pic *parent_pic; 944d1605cdaSAndrew Turner struct intr_pic_child *newchild; 945d1605cdaSAndrew Turner #ifdef INVARIANTS 946d1605cdaSAndrew Turner struct intr_pic_child *child; 947d1605cdaSAndrew Turner #endif 948d1605cdaSAndrew Turner 949c0d52370SAndrew Turner /* Find the parent PIC */ 950c0d52370SAndrew Turner parent_pic = pic_lookup(parent, 0, FLAG_PIC); 951d1605cdaSAndrew Turner if (parent_pic == NULL) 952a3e828c9SJessica Clarke return (ENXIO); 953d1605cdaSAndrew Turner 954d1605cdaSAndrew Turner newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); 955d1605cdaSAndrew Turner newchild->pc_pic = pic; 956d1605cdaSAndrew Turner newchild->pc_filter = filter; 957d1605cdaSAndrew Turner newchild->pc_filter_arg = arg; 958d1605cdaSAndrew Turner newchild->pc_start = start; 959d1605cdaSAndrew Turner newchild->pc_length = length; 960d1605cdaSAndrew Turner 961d1605cdaSAndrew Turner mtx_lock_spin(&parent_pic->pic_child_lock); 962d1605cdaSAndrew Turner #ifdef INVARIANTS 963d1605cdaSAndrew Turner SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { 964d1605cdaSAndrew Turner KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", 965d1605cdaSAndrew Turner __func__)); 966d1605cdaSAndrew Turner } 967d1605cdaSAndrew Turner #endif 968d1605cdaSAndrew Turner SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); 969d1605cdaSAndrew Turner mtx_unlock_spin(&parent_pic->pic_child_lock); 970d1605cdaSAndrew Turner 971a3e828c9SJessica Clarke return (0); 972d1605cdaSAndrew Turner } 973d1605cdaSAndrew Turner 974895c8b1cSMichal Meloun static int 975895c8b1cSMichal Meloun intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, 976895c8b1cSMichal Meloun struct intr_irqsrc **isrc) 9772b3ad188SAdrian Chadd { 978bff6be3eSSvatopluk Kraus struct intr_pic *pic; 979895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 980bff6be3eSSvatopluk Kraus 981bff6be3eSSvatopluk Kraus if (data == NULL) 982bff6be3eSSvatopluk Kraus return (EINVAL); 983bff6be3eSSvatopluk Kraus 984c0d52370SAndrew Turner pic = pic_lookup(dev, xref, 985c0d52370SAndrew Turner (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); 98615adccc6SSvatopluk Kraus if (pic == NULL) 987bff6be3eSSvatopluk Kraus return (ESRCH); 988bff6be3eSSvatopluk Kraus 989895c8b1cSMichal Meloun switch (data->type) { 990895c8b1cSMichal Meloun case INTR_MAP_DATA_MSI: 991c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 992895c8b1cSMichal Meloun ("%s: Found a non-MSI controller: %s", __func__, 993895c8b1cSMichal Meloun device_get_name(pic->pic_dev))); 994895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)data; 995895c8b1cSMichal Meloun *isrc = msi->isrc; 996895c8b1cSMichal Meloun return (0); 997895c8b1cSMichal Meloun 998895c8b1cSMichal Meloun default: 999c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 10003fc155dcSAndrew Turner ("%s: Found a non-PIC controller: %s", __func__, 10013fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 1002895c8b1cSMichal Meloun return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); 1003895c8b1cSMichal Meloun } 1004895c8b1cSMichal Meloun } 1005895c8b1cSMichal Meloun 1006eb20867fSMichal Meloun bool 1007eb20867fSMichal Meloun intr_is_per_cpu(struct resource *res) 1008eb20867fSMichal Meloun { 1009eb20867fSMichal Meloun u_int res_id; 1010eb20867fSMichal Meloun struct intr_irqsrc *isrc; 1011eb20867fSMichal Meloun 1012eb20867fSMichal Meloun res_id = (u_int)rman_get_start(res); 1013eb20867fSMichal Meloun isrc = intr_map_get_isrc(res_id); 1014eb20867fSMichal Meloun 1015eb20867fSMichal Meloun if (isrc == NULL) 1016eb20867fSMichal Meloun panic("Attempt to get isrc for non-active resource id: %u\n", 1017eb20867fSMichal Meloun res_id); 1018eb20867fSMichal Meloun return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); 1019eb20867fSMichal Meloun } 1020eb20867fSMichal Meloun 1021895c8b1cSMichal Meloun int 1022895c8b1cSMichal Meloun intr_activate_irq(device_t dev, struct resource *res) 1023895c8b1cSMichal Meloun { 1024895c8b1cSMichal Meloun device_t map_dev; 1025895c8b1cSMichal Meloun intptr_t map_xref; 1026895c8b1cSMichal Meloun struct intr_map_data *data; 1027895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 1028895c8b1cSMichal Meloun u_int res_id; 1029895c8b1cSMichal Meloun int error; 1030895c8b1cSMichal Meloun 1031895c8b1cSMichal Meloun KASSERT(rman_get_start(res) == rman_get_end(res), 1032895c8b1cSMichal Meloun ("%s: more interrupts in resource", __func__)); 1033895c8b1cSMichal Meloun 1034895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1035895c8b1cSMichal Meloun if (intr_map_get_isrc(res_id) != NULL) 1036895c8b1cSMichal Meloun panic("Attempt to double activation of resource id: %u\n", 1037895c8b1cSMichal Meloun res_id); 1038895c8b1cSMichal Meloun intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); 1039895c8b1cSMichal Meloun error = intr_resolve_irq(map_dev, map_xref, data, &isrc); 1040895c8b1cSMichal Meloun if (error != 0) { 1041895c8b1cSMichal Meloun free(data, M_INTRNG); 1042895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 1043895c8b1cSMichal Meloun /* if (error == EINVAL) return(0); */ 1044bff6be3eSSvatopluk Kraus return (error); 1045bff6be3eSSvatopluk Kraus } 1046895c8b1cSMichal Meloun intr_map_set_isrc(res_id, isrc); 1047895c8b1cSMichal Meloun rman_set_virtual(res, data); 1048895c8b1cSMichal Meloun return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); 1049bff6be3eSSvatopluk Kraus } 1050bff6be3eSSvatopluk Kraus 1051bff6be3eSSvatopluk Kraus int 1052895c8b1cSMichal Meloun intr_deactivate_irq(device_t dev, struct resource *res) 1053bff6be3eSSvatopluk Kraus { 1054bff6be3eSSvatopluk Kraus struct intr_map_data *data; 1055bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1056895c8b1cSMichal Meloun u_int res_id; 1057895c8b1cSMichal Meloun int error; 1058bff6be3eSSvatopluk Kraus 1059bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1060bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1061bff6be3eSSvatopluk Kraus 1062895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1063895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 1064bff6be3eSSvatopluk Kraus if (isrc == NULL) 1065895c8b1cSMichal Meloun panic("Attempt to deactivate non-active resource id: %u\n", 1066895c8b1cSMichal Meloun res_id); 1067bff6be3eSSvatopluk Kraus 1068c4263292SSvatopluk Kraus data = rman_get_virtual(res); 1069895c8b1cSMichal Meloun error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); 1070895c8b1cSMichal Meloun intr_map_set_isrc(res_id, NULL); 1071895c8b1cSMichal Meloun rman_set_virtual(res, NULL); 1072895c8b1cSMichal Meloun free(data, M_INTRNG); 1073895c8b1cSMichal Meloun return (error); 1074bff6be3eSSvatopluk Kraus } 1075bff6be3eSSvatopluk Kraus 1076bff6be3eSSvatopluk Kraus int 1077bff6be3eSSvatopluk Kraus intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, 1078bff6be3eSSvatopluk Kraus driver_intr_t hand, void *arg, int flags, void **cookiep) 1079bff6be3eSSvatopluk Kraus { 1080bff6be3eSSvatopluk Kraus int error; 1081bff6be3eSSvatopluk Kraus struct intr_map_data *data; 1082bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1083bff6be3eSSvatopluk Kraus const char *name; 1084895c8b1cSMichal Meloun u_int res_id; 1085bff6be3eSSvatopluk Kraus 1086bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1087bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1088bff6be3eSSvatopluk Kraus 1089895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1090895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 1091895c8b1cSMichal Meloun if (isrc == NULL) { 1092895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 1093bff6be3eSSvatopluk Kraus return (EINVAL); 1094895c8b1cSMichal Meloun } 10952b3ad188SAdrian Chadd 1096c4263292SSvatopluk Kraus data = rman_get_virtual(res); 10972b3ad188SAdrian Chadd name = device_get_nameunit(dev); 10982b3ad188SAdrian Chadd 10992b3ad188SAdrian Chadd #ifdef INTR_SOLO 11002b3ad188SAdrian Chadd /* 1101e3043798SPedro F. Giffuni * Standard handling is done through MI interrupt framework. However, 11022b3ad188SAdrian Chadd * some interrupts could request solely own special handling. This 11032b3ad188SAdrian Chadd * non standard handling can be used for interrupt controllers without 11042b3ad188SAdrian Chadd * handler (filter only), so in case that interrupt controllers are 11052b3ad188SAdrian Chadd * chained, MI interrupt framework is called only in leaf controller. 11062b3ad188SAdrian Chadd * 11072b3ad188SAdrian Chadd * Note that root interrupt controller routine is served as well, 11082b3ad188SAdrian Chadd * however in intr_irq_handler(), i.e. main system dispatch routine. 11092b3ad188SAdrian Chadd */ 11102b3ad188SAdrian Chadd if (flags & INTR_SOLO && hand != NULL) { 11112b3ad188SAdrian Chadd debugf("irq %u cannot solo on %s\n", irq, name); 11122b3ad188SAdrian Chadd return (EINVAL); 11132b3ad188SAdrian Chadd } 11142b3ad188SAdrian Chadd 11152b3ad188SAdrian Chadd if (flags & INTR_SOLO) { 11162b3ad188SAdrian Chadd error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 11172b3ad188SAdrian Chadd arg, cookiep); 1118ce44a736SIan Lepore debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, 11192b3ad188SAdrian Chadd name); 11202b3ad188SAdrian Chadd } else 11212b3ad188SAdrian Chadd #endif 11222b3ad188SAdrian Chadd { 11232b3ad188SAdrian Chadd error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 11242b3ad188SAdrian Chadd cookiep); 1125ce44a736SIan Lepore debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); 11262b3ad188SAdrian Chadd } 11272b3ad188SAdrian Chadd if (error != 0) 11282b3ad188SAdrian Chadd return (error); 11292b3ad188SAdrian Chadd 11302b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1131bff6be3eSSvatopluk Kraus error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); 1132bff6be3eSSvatopluk Kraus if (error == 0) { 11332b3ad188SAdrian Chadd isrc->isrc_handlers++; 1134bff6be3eSSvatopluk Kraus if (isrc->isrc_handlers == 1) 11352b3ad188SAdrian Chadd PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 11362b3ad188SAdrian Chadd } 11372b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1138bff6be3eSSvatopluk Kraus if (error != 0) 1139bff6be3eSSvatopluk Kraus intr_event_remove_handler(*cookiep); 1140bff6be3eSSvatopluk Kraus return (error); 11412b3ad188SAdrian Chadd } 11422b3ad188SAdrian Chadd 11432b3ad188SAdrian Chadd int 1144bff6be3eSSvatopluk Kraus intr_teardown_irq(device_t dev, struct resource *res, void *cookie) 11452b3ad188SAdrian Chadd { 11462b3ad188SAdrian Chadd int error; 1147bff6be3eSSvatopluk Kraus struct intr_map_data *data; 1148bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1149895c8b1cSMichal Meloun u_int res_id; 11502b3ad188SAdrian Chadd 1151bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1152bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1153bff6be3eSSvatopluk Kraus 1154895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1155895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 11562b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 11572b3ad188SAdrian Chadd return (EINVAL); 1158bff6be3eSSvatopluk Kraus 1159c4263292SSvatopluk Kraus data = rman_get_virtual(res); 1160c4263292SSvatopluk Kraus 1161169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 11622b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 11632b3ad188SAdrian Chadd if (isrc != cookie) 11642b3ad188SAdrian Chadd return (EINVAL); 11652b3ad188SAdrian Chadd 11662b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 11672b3ad188SAdrian Chadd isrc->isrc_filter = NULL; 11682b3ad188SAdrian Chadd isrc->isrc_arg = NULL; 11692b3ad188SAdrian Chadd isrc->isrc_handlers = 0; 11702b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1171bff6be3eSSvatopluk Kraus PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 11722b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 11732b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 11742b3ad188SAdrian Chadd return (0); 11752b3ad188SAdrian Chadd } 1176169e6abdSSvatopluk Kraus #endif 11772b3ad188SAdrian Chadd if (isrc != intr_handler_source(cookie)) 11782b3ad188SAdrian Chadd return (EINVAL); 11792b3ad188SAdrian Chadd 11802b3ad188SAdrian Chadd error = intr_event_remove_handler(cookie); 11812b3ad188SAdrian Chadd if (error == 0) { 11822b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 11832b3ad188SAdrian Chadd isrc->isrc_handlers--; 1184bff6be3eSSvatopluk Kraus if (isrc->isrc_handlers == 0) 11852b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1186bff6be3eSSvatopluk Kraus PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 11872b3ad188SAdrian Chadd intrcnt_updatename(isrc); 11882b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 11892b3ad188SAdrian Chadd } 11902b3ad188SAdrian Chadd return (error); 11912b3ad188SAdrian Chadd } 11922b3ad188SAdrian Chadd 11932b3ad188SAdrian Chadd int 1194bff6be3eSSvatopluk Kraus intr_describe_irq(device_t dev, struct resource *res, void *cookie, 1195bff6be3eSSvatopluk Kraus const char *descr) 11962b3ad188SAdrian Chadd { 11972b3ad188SAdrian Chadd int error; 1198bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1199895c8b1cSMichal Meloun u_int res_id; 12002b3ad188SAdrian Chadd 1201bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1202bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1203bff6be3eSSvatopluk Kraus 1204895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1205895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 12062b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 12072b3ad188SAdrian Chadd return (EINVAL); 1208169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 12092b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 12102b3ad188SAdrian Chadd if (isrc != cookie) 12112b3ad188SAdrian Chadd return (EINVAL); 12122b3ad188SAdrian Chadd 12132b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 12142b3ad188SAdrian Chadd isrc_update_name(isrc, descr); 12152b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 12162b3ad188SAdrian Chadd return (0); 12172b3ad188SAdrian Chadd } 1218169e6abdSSvatopluk Kraus #endif 12192b3ad188SAdrian Chadd error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 12202b3ad188SAdrian Chadd if (error == 0) { 12212b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 12222b3ad188SAdrian Chadd intrcnt_updatename(isrc); 12232b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 12242b3ad188SAdrian Chadd } 12252b3ad188SAdrian Chadd return (error); 12262b3ad188SAdrian Chadd } 12272b3ad188SAdrian Chadd 12282b3ad188SAdrian Chadd #ifdef SMP 12292b3ad188SAdrian Chadd int 1230bff6be3eSSvatopluk Kraus intr_bind_irq(device_t dev, struct resource *res, int cpu) 12312b3ad188SAdrian Chadd { 12322b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1233895c8b1cSMichal Meloun u_int res_id; 12342b3ad188SAdrian Chadd 1235bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1236bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1237bff6be3eSSvatopluk Kraus 1238895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1239895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 12402b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 12412b3ad188SAdrian Chadd return (EINVAL); 1242169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 12432b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) 12442b3ad188SAdrian Chadd return (intr_isrc_assign_cpu(isrc, cpu)); 1245169e6abdSSvatopluk Kraus #endif 12462b3ad188SAdrian Chadd return (intr_event_bind(isrc->isrc_event, cpu)); 12472b3ad188SAdrian Chadd } 12482b3ad188SAdrian Chadd 12492b3ad188SAdrian Chadd /* 12502b3ad188SAdrian Chadd * Return the CPU that the next interrupt source should use. 12512b3ad188SAdrian Chadd * For now just returns the next CPU according to round-robin. 12522b3ad188SAdrian Chadd */ 12532b3ad188SAdrian Chadd u_int 12542b3ad188SAdrian Chadd intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 12552b3ad188SAdrian Chadd { 1256a92a2f00SAndrew Turner u_int cpu; 12572b3ad188SAdrian Chadd 1258a92a2f00SAndrew Turner KASSERT(!CPU_EMPTY(cpumask), ("%s: Empty CPU mask", __func__)); 1259a92a2f00SAndrew Turner if (!irq_assign_cpu || mp_ncpus == 1) { 1260a92a2f00SAndrew Turner cpu = PCPU_GET(cpuid); 1261a92a2f00SAndrew Turner 1262a92a2f00SAndrew Turner if (CPU_ISSET(cpu, cpumask)) 1263a92a2f00SAndrew Turner return (curcpu); 1264a92a2f00SAndrew Turner 1265a92a2f00SAndrew Turner return (CPU_FFS(cpumask) - 1); 1266a92a2f00SAndrew Turner } 12672b3ad188SAdrian Chadd 12682b3ad188SAdrian Chadd do { 12692b3ad188SAdrian Chadd last_cpu++; 12702b3ad188SAdrian Chadd if (last_cpu > mp_maxid) 12712b3ad188SAdrian Chadd last_cpu = 0; 12722b3ad188SAdrian Chadd } while (!CPU_ISSET(last_cpu, cpumask)); 12732b3ad188SAdrian Chadd return (last_cpu); 12742b3ad188SAdrian Chadd } 12752b3ad188SAdrian Chadd 1276dc425090SMitchell Horne #ifndef EARLY_AP_STARTUP 12772b3ad188SAdrian Chadd /* 12782b3ad188SAdrian Chadd * Distribute all the interrupt sources among the available 12792b3ad188SAdrian Chadd * CPUs once the AP's have been launched. 12802b3ad188SAdrian Chadd */ 12812b3ad188SAdrian Chadd static void 12822b3ad188SAdrian Chadd intr_irq_shuffle(void *arg __unused) 12832b3ad188SAdrian Chadd { 12842b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 12852b3ad188SAdrian Chadd u_int i; 12862b3ad188SAdrian Chadd 12872b3ad188SAdrian Chadd if (mp_ncpus == 1) 12882b3ad188SAdrian Chadd return; 12892b3ad188SAdrian Chadd 12902b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1291dc425090SMitchell Horne irq_assign_cpu = true; 1292248f0cabSOleksandr Tymoshenko for (i = 0; i < intr_nirq; i++) { 12932b3ad188SAdrian Chadd isrc = irq_sources[i]; 12942b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0 || 1295cf55df9fSSvatopluk Kraus isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) 12962b3ad188SAdrian Chadd continue; 12972b3ad188SAdrian Chadd 12982b3ad188SAdrian Chadd if (isrc->isrc_event != NULL && 12992b3ad188SAdrian Chadd isrc->isrc_flags & INTR_ISRCF_BOUND && 13002b3ad188SAdrian Chadd isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 13012b3ad188SAdrian Chadd panic("%s: CPU inconsistency", __func__); 13022b3ad188SAdrian Chadd 13032b3ad188SAdrian Chadd if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 13042b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); /* start again */ 13052b3ad188SAdrian Chadd 13062b3ad188SAdrian Chadd /* 13072b3ad188SAdrian Chadd * We are in wicked position here if the following call fails 13082b3ad188SAdrian Chadd * for bound ISRC. The best thing we can do is to clear 13092b3ad188SAdrian Chadd * isrc_cpu so inconsistency with ie_cpu will be detectable. 13102b3ad188SAdrian Chadd */ 1311bff6be3eSSvatopluk Kraus if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) 13122b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 13132b3ad188SAdrian Chadd } 13142b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 13152b3ad188SAdrian Chadd } 13162b3ad188SAdrian Chadd SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1317dc425090SMitchell Horne #endif /* !EARLY_AP_STARTUP */ 13182b3ad188SAdrian Chadd 13192b3ad188SAdrian Chadd #else 13202b3ad188SAdrian Chadd u_int 13212b3ad188SAdrian Chadd intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 13222b3ad188SAdrian Chadd { 13232b3ad188SAdrian Chadd 13242b3ad188SAdrian Chadd return (PCPU_GET(cpuid)); 13252b3ad188SAdrian Chadd } 1326dc425090SMitchell Horne #endif /* SMP */ 13272b3ad188SAdrian Chadd 13283fc155dcSAndrew Turner /* 1329895c8b1cSMichal Meloun * Allocate memory for new intr_map_data structure. 1330895c8b1cSMichal Meloun * Initialize common fields. 1331895c8b1cSMichal Meloun */ 1332895c8b1cSMichal Meloun struct intr_map_data * 1333895c8b1cSMichal Meloun intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) 1334895c8b1cSMichal Meloun { 1335895c8b1cSMichal Meloun struct intr_map_data *data; 1336895c8b1cSMichal Meloun 1337895c8b1cSMichal Meloun data = malloc(len, M_INTRNG, flags); 1338895c8b1cSMichal Meloun data->type = type; 1339895c8b1cSMichal Meloun data->len = len; 1340895c8b1cSMichal Meloun return (data); 1341895c8b1cSMichal Meloun } 1342895c8b1cSMichal Meloun 1343895c8b1cSMichal Meloun void intr_free_intr_map_data(struct intr_map_data *data) 1344895c8b1cSMichal Meloun { 1345895c8b1cSMichal Meloun 1346895c8b1cSMichal Meloun free(data, M_INTRNG); 1347895c8b1cSMichal Meloun } 1348895c8b1cSMichal Meloun 1349895c8b1cSMichal Meloun /* 13503fc155dcSAndrew Turner * Register a MSI/MSI-X interrupt controller 13513fc155dcSAndrew Turner */ 13523fc155dcSAndrew Turner int 13533fc155dcSAndrew Turner intr_msi_register(device_t dev, intptr_t xref) 13543fc155dcSAndrew Turner { 13553fc155dcSAndrew Turner struct intr_pic *pic; 13563fc155dcSAndrew Turner 13573fc155dcSAndrew Turner if (dev == NULL) 13583fc155dcSAndrew Turner return (EINVAL); 1359c0d52370SAndrew Turner pic = pic_create(dev, xref, FLAG_MSI); 13603fc155dcSAndrew Turner if (pic == NULL) 13613fc155dcSAndrew Turner return (ENOMEM); 13623fc155dcSAndrew Turner 13633fc155dcSAndrew Turner debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 13643fc155dcSAndrew Turner device_get_nameunit(dev), dev, (uintmax_t)xref); 13653fc155dcSAndrew Turner return (0); 13663fc155dcSAndrew Turner } 13673fc155dcSAndrew Turner 13683fc155dcSAndrew Turner int 13693fc155dcSAndrew Turner intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, 13703fc155dcSAndrew Turner int maxcount, int *irqs) 13713fc155dcSAndrew Turner { 1372e707c8beSRuslan Bukin struct iommu_domain *domain; 13733fc155dcSAndrew Turner struct intr_irqsrc **isrc; 13743fc155dcSAndrew Turner struct intr_pic *pic; 13753fc155dcSAndrew Turner device_t pdev; 1376895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 13773fc155dcSAndrew Turner int err, i; 13783fc155dcSAndrew Turner 1379c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 13803fc155dcSAndrew Turner if (pic == NULL) 13813fc155dcSAndrew Turner return (ESRCH); 13823fc155dcSAndrew Turner 1383c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 13843fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 13853fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 13863fc155dcSAndrew Turner 1387e707c8beSRuslan Bukin /* 1388e707c8beSRuslan Bukin * If this is the first time we have used this context ask the 1389e707c8beSRuslan Bukin * interrupt controller to map memory the msi source will need. 1390e707c8beSRuslan Bukin */ 1391e707c8beSRuslan Bukin err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); 1392e707c8beSRuslan Bukin if (err != 0) 1393e707c8beSRuslan Bukin return (err); 1394e707c8beSRuslan Bukin 13953fc155dcSAndrew Turner isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 13963fc155dcSAndrew Turner err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); 1397895c8b1cSMichal Meloun if (err != 0) { 1398895c8b1cSMichal Meloun free(isrc, M_INTRNG); 1399895c8b1cSMichal Meloun return (err); 14003fc155dcSAndrew Turner } 14013fc155dcSAndrew Turner 1402895c8b1cSMichal Meloun for (i = 0; i < count; i++) { 1403e707c8beSRuslan Bukin isrc[i]->isrc_iommu = domain; 1404895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1405895c8b1cSMichal Meloun INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1406895c8b1cSMichal Meloun msi-> isrc = isrc[i]; 1407e707c8beSRuslan Bukin 1408895c8b1cSMichal Meloun irqs[i] = intr_map_irq(pic->pic_dev, xref, 1409895c8b1cSMichal Meloun (struct intr_map_data *)msi); 1410895c8b1cSMichal Meloun } 14113fc155dcSAndrew Turner free(isrc, M_INTRNG); 14123fc155dcSAndrew Turner 14133fc155dcSAndrew Turner return (err); 14143fc155dcSAndrew Turner } 14153fc155dcSAndrew Turner 14163fc155dcSAndrew Turner int 14173fc155dcSAndrew Turner intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, 14183fc155dcSAndrew Turner int *irqs) 14193fc155dcSAndrew Turner { 14203fc155dcSAndrew Turner struct intr_irqsrc **isrc; 14213fc155dcSAndrew Turner struct intr_pic *pic; 1422609b0fe9SOleksandr Tymoshenko struct intr_map_data_msi *msi; 14233fc155dcSAndrew Turner int i, err; 14243fc155dcSAndrew Turner 1425c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 14263fc155dcSAndrew Turner if (pic == NULL) 14273fc155dcSAndrew Turner return (ESRCH); 14283fc155dcSAndrew Turner 1429c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 14303fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 14313fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 14323fc155dcSAndrew Turner 14333fc155dcSAndrew Turner isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 14343fc155dcSAndrew Turner 1435609b0fe9SOleksandr Tymoshenko for (i = 0; i < count; i++) { 1436609b0fe9SOleksandr Tymoshenko msi = (struct intr_map_data_msi *) 1437609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(irqs[i]); 1438609b0fe9SOleksandr Tymoshenko KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1439609b0fe9SOleksandr Tymoshenko ("%s: irq %d map data is not MSI", __func__, 1440609b0fe9SOleksandr Tymoshenko irqs[i])); 1441609b0fe9SOleksandr Tymoshenko isrc[i] = msi->isrc; 1442609b0fe9SOleksandr Tymoshenko } 14433fc155dcSAndrew Turner 1444f32f0095SRuslan Bukin MSI_IOMMU_DEINIT(pic->pic_dev, child); 1445f32f0095SRuslan Bukin 14463fc155dcSAndrew Turner err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); 1447895c8b1cSMichal Meloun 1448895c8b1cSMichal Meloun for (i = 0; i < count; i++) { 1449895c8b1cSMichal Meloun if (isrc[i] != NULL) 1450895c8b1cSMichal Meloun intr_unmap_irq(irqs[i]); 1451895c8b1cSMichal Meloun } 1452895c8b1cSMichal Meloun 14533fc155dcSAndrew Turner free(isrc, M_INTRNG); 14543fc155dcSAndrew Turner return (err); 14553fc155dcSAndrew Turner } 14563fc155dcSAndrew Turner 14573fc155dcSAndrew Turner int 14583fc155dcSAndrew Turner intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) 14593fc155dcSAndrew Turner { 1460e707c8beSRuslan Bukin struct iommu_domain *domain; 14613fc155dcSAndrew Turner struct intr_irqsrc *isrc; 14623fc155dcSAndrew Turner struct intr_pic *pic; 14633fc155dcSAndrew Turner device_t pdev; 1464895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 14653fc155dcSAndrew Turner int err; 14663fc155dcSAndrew Turner 1467c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 14683fc155dcSAndrew Turner if (pic == NULL) 14693fc155dcSAndrew Turner return (ESRCH); 14703fc155dcSAndrew Turner 1471c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 14723fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 14733fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 14743fc155dcSAndrew Turner 1475e707c8beSRuslan Bukin /* 1476e707c8beSRuslan Bukin * If this is the first time we have used this context ask the 1477e707c8beSRuslan Bukin * interrupt controller to map memory the msi source will need. 1478e707c8beSRuslan Bukin */ 1479e707c8beSRuslan Bukin err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); 1480e707c8beSRuslan Bukin if (err != 0) 1481e707c8beSRuslan Bukin return (err); 1482e707c8beSRuslan Bukin 14833fc155dcSAndrew Turner err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); 14843fc155dcSAndrew Turner if (err != 0) 14853fc155dcSAndrew Turner return (err); 14863fc155dcSAndrew Turner 1487e707c8beSRuslan Bukin isrc->isrc_iommu = domain; 1488895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1489895c8b1cSMichal Meloun INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1490895c8b1cSMichal Meloun msi->isrc = isrc; 1491895c8b1cSMichal Meloun *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); 14923fc155dcSAndrew Turner return (0); 14933fc155dcSAndrew Turner } 14943fc155dcSAndrew Turner 14953fc155dcSAndrew Turner int 14963fc155dcSAndrew Turner intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) 14973fc155dcSAndrew Turner { 14983fc155dcSAndrew Turner struct intr_irqsrc *isrc; 14993fc155dcSAndrew Turner struct intr_pic *pic; 1500609b0fe9SOleksandr Tymoshenko struct intr_map_data_msi *msi; 15013fc155dcSAndrew Turner int err; 15023fc155dcSAndrew Turner 1503c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 15043fc155dcSAndrew Turner if (pic == NULL) 15053fc155dcSAndrew Turner return (ESRCH); 15063fc155dcSAndrew Turner 1507c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 15083fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 15093fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 15103fc155dcSAndrew Turner 1511609b0fe9SOleksandr Tymoshenko msi = (struct intr_map_data_msi *) 1512609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(irq); 1513609b0fe9SOleksandr Tymoshenko KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1514609b0fe9SOleksandr Tymoshenko ("%s: irq %d map data is not MSI", __func__, 1515609b0fe9SOleksandr Tymoshenko irq)); 1516609b0fe9SOleksandr Tymoshenko isrc = msi->isrc; 1517895c8b1cSMichal Meloun if (isrc == NULL) { 1518895c8b1cSMichal Meloun intr_unmap_irq(irq); 15193fc155dcSAndrew Turner return (EINVAL); 1520895c8b1cSMichal Meloun } 15213fc155dcSAndrew Turner 1522f32f0095SRuslan Bukin MSI_IOMMU_DEINIT(pic->pic_dev, child); 1523f32f0095SRuslan Bukin 15243fc155dcSAndrew Turner err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); 1525895c8b1cSMichal Meloun intr_unmap_irq(irq); 1526895c8b1cSMichal Meloun 15273fc155dcSAndrew Turner return (err); 15283fc155dcSAndrew Turner } 15293fc155dcSAndrew Turner 15303fc155dcSAndrew Turner int 15313fc155dcSAndrew Turner intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, 15323fc155dcSAndrew Turner uint64_t *addr, uint32_t *data) 15333fc155dcSAndrew Turner { 15343fc155dcSAndrew Turner struct intr_irqsrc *isrc; 15353fc155dcSAndrew Turner struct intr_pic *pic; 15363fc155dcSAndrew Turner int err; 15373fc155dcSAndrew Turner 1538c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 15393fc155dcSAndrew Turner if (pic == NULL) 15403fc155dcSAndrew Turner return (ESRCH); 15413fc155dcSAndrew Turner 1542c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 15433fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 15443fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 15453fc155dcSAndrew Turner 1546895c8b1cSMichal Meloun isrc = intr_map_get_isrc(irq); 15473fc155dcSAndrew Turner if (isrc == NULL) 15483fc155dcSAndrew Turner return (EINVAL); 15493fc155dcSAndrew Turner 15503fc155dcSAndrew Turner err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); 1551e707c8beSRuslan Bukin 1552e707c8beSRuslan Bukin #ifdef IOMMU 1553e707c8beSRuslan Bukin if (isrc->isrc_iommu != NULL) 1554e707c8beSRuslan Bukin iommu_translate_msi(isrc->isrc_iommu, addr); 1555e707c8beSRuslan Bukin #endif 1556e707c8beSRuslan Bukin 15573fc155dcSAndrew Turner return (err); 15583fc155dcSAndrew Turner } 15593fc155dcSAndrew Turner 15602b3ad188SAdrian Chadd void dosoftints(void); 15612b3ad188SAdrian Chadd void 15622b3ad188SAdrian Chadd dosoftints(void) 15632b3ad188SAdrian Chadd { 15642b3ad188SAdrian Chadd } 15652b3ad188SAdrian Chadd 15662b3ad188SAdrian Chadd #ifdef SMP 15672b3ad188SAdrian Chadd /* 15682b3ad188SAdrian Chadd * Init interrupt controller on another CPU. 15692b3ad188SAdrian Chadd */ 15702b3ad188SAdrian Chadd void 15712b3ad188SAdrian Chadd intr_pic_init_secondary(void) 15722b3ad188SAdrian Chadd { 157385918bebSAyrton Munoz device_t dev; 15744b01a7faSKyle Evans uint32_t rootnum; 15752b3ad188SAdrian Chadd 15762b3ad188SAdrian Chadd /* 157785918bebSAyrton Munoz * QQQ: Only root PICs are aware of other CPUs ??? 15782b3ad188SAdrian Chadd */ 15792b3ad188SAdrian Chadd //mtx_lock(&isrc_table_lock); 1580*487788a6SElliott Mitchell for (rootnum = 0; rootnum < INTR_ROOT_COUNT; rootnum++) { 15814b01a7faSKyle Evans dev = intr_irq_roots[rootnum].dev; 158285918bebSAyrton Munoz if (dev != NULL) { 15834b01a7faSKyle Evans PIC_INIT_SECONDARY(dev, rootnum); 158485918bebSAyrton Munoz } 158585918bebSAyrton Munoz } 15862b3ad188SAdrian Chadd //mtx_unlock(&isrc_table_lock); 15872b3ad188SAdrian Chadd } 15882b3ad188SAdrian Chadd #endif 15892b3ad188SAdrian Chadd 15902b3ad188SAdrian Chadd #ifdef DDB 1591c84c5e00SMitchell Horne DB_SHOW_COMMAND_FLAGS(irqs, db_show_irqs, DB_CMD_MEMSAFE) 15922b3ad188SAdrian Chadd { 15932b3ad188SAdrian Chadd u_int i, irqsum; 1594bff6be3eSSvatopluk Kraus u_long num; 15952b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 15962b3ad188SAdrian Chadd 1597248f0cabSOleksandr Tymoshenko for (irqsum = 0, i = 0; i < intr_nirq; i++) { 15982b3ad188SAdrian Chadd isrc = irq_sources[i]; 15992b3ad188SAdrian Chadd if (isrc == NULL) 16002b3ad188SAdrian Chadd continue; 16012b3ad188SAdrian Chadd 1602bff6be3eSSvatopluk Kraus num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; 16032b3ad188SAdrian Chadd db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 16042b3ad188SAdrian Chadd isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1605bff6be3eSSvatopluk Kraus isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); 1606bff6be3eSSvatopluk Kraus irqsum += num; 16072b3ad188SAdrian Chadd } 16082b3ad188SAdrian Chadd db_printf("irq total %u\n", irqsum); 16092b3ad188SAdrian Chadd } 16102b3ad188SAdrian Chadd #endif 1611895c8b1cSMichal Meloun 1612895c8b1cSMichal Meloun /* 1613895c8b1cSMichal Meloun * Interrupt mapping table functions. 1614895c8b1cSMichal Meloun * 1615895c8b1cSMichal Meloun * Please, keep this part separately, it can be transformed to 1616895c8b1cSMichal Meloun * extension of standard resources. 1617895c8b1cSMichal Meloun */ 1618895c8b1cSMichal Meloun struct intr_map_entry 1619895c8b1cSMichal Meloun { 1620895c8b1cSMichal Meloun device_t dev; 1621895c8b1cSMichal Meloun intptr_t xref; 1622895c8b1cSMichal Meloun struct intr_map_data *map_data; 1623895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 1624895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 1625895c8b1cSMichal Meloun /*int flags */ 1626895c8b1cSMichal Meloun }; 1627895c8b1cSMichal Meloun 1628895c8b1cSMichal Meloun /* XXX Convert irq_map[] to dynamicaly expandable one. */ 1629248f0cabSOleksandr Tymoshenko static struct intr_map_entry **irq_map; 1630a3c7da3dSElliott Mitchell static u_int irq_map_count; 1631a3c7da3dSElliott Mitchell static u_int irq_map_first_free_idx; 1632895c8b1cSMichal Meloun static struct mtx irq_map_lock; 1633895c8b1cSMichal Meloun 1634895c8b1cSMichal Meloun static struct intr_irqsrc * 1635895c8b1cSMichal Meloun intr_map_get_isrc(u_int res_id) 1636895c8b1cSMichal Meloun { 1637895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 1638895c8b1cSMichal Meloun 1639ecc8ccb4SAndrew Turner isrc = NULL; 1640895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1641ecc8ccb4SAndrew Turner if (res_id < irq_map_count && irq_map[res_id] != NULL) 1642895c8b1cSMichal Meloun isrc = irq_map[res_id]->isrc; 1643895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1644ecc8ccb4SAndrew Turner 1645895c8b1cSMichal Meloun return (isrc); 1646895c8b1cSMichal Meloun } 1647895c8b1cSMichal Meloun 1648895c8b1cSMichal Meloun static void 1649895c8b1cSMichal Meloun intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) 1650895c8b1cSMichal Meloun { 1651895c8b1cSMichal Meloun 1652895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1653ecc8ccb4SAndrew Turner if (res_id < irq_map_count && irq_map[res_id] != NULL) 1654895c8b1cSMichal Meloun irq_map[res_id]->isrc = isrc; 1655895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1656895c8b1cSMichal Meloun } 1657895c8b1cSMichal Meloun 1658895c8b1cSMichal Meloun /* 1659895c8b1cSMichal Meloun * Get a copy of intr_map_entry data 1660895c8b1cSMichal Meloun */ 1661609b0fe9SOleksandr Tymoshenko static struct intr_map_data * 1662609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(u_int res_id) 1663609b0fe9SOleksandr Tymoshenko { 1664609b0fe9SOleksandr Tymoshenko struct intr_map_data *data; 1665609b0fe9SOleksandr Tymoshenko 1666609b0fe9SOleksandr Tymoshenko data = NULL; 1667609b0fe9SOleksandr Tymoshenko mtx_lock(&irq_map_lock); 1668609b0fe9SOleksandr Tymoshenko if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1669609b0fe9SOleksandr Tymoshenko panic("Attempt to copy invalid resource id: %u\n", res_id); 1670609b0fe9SOleksandr Tymoshenko data = irq_map[res_id]->map_data; 1671609b0fe9SOleksandr Tymoshenko mtx_unlock(&irq_map_lock); 1672609b0fe9SOleksandr Tymoshenko 1673609b0fe9SOleksandr Tymoshenko return (data); 1674609b0fe9SOleksandr Tymoshenko } 1675609b0fe9SOleksandr Tymoshenko 1676609b0fe9SOleksandr Tymoshenko /* 1677609b0fe9SOleksandr Tymoshenko * Get a copy of intr_map_entry data 1678609b0fe9SOleksandr Tymoshenko */ 1679895c8b1cSMichal Meloun static void 1680895c8b1cSMichal Meloun intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, 1681895c8b1cSMichal Meloun struct intr_map_data **data) 1682895c8b1cSMichal Meloun { 1683895c8b1cSMichal Meloun size_t len; 1684895c8b1cSMichal Meloun 1685895c8b1cSMichal Meloun len = 0; 1686895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1687895c8b1cSMichal Meloun if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1688895c8b1cSMichal Meloun panic("Attempt to copy invalid resource id: %u\n", res_id); 1689895c8b1cSMichal Meloun if (irq_map[res_id]->map_data != NULL) 1690895c8b1cSMichal Meloun len = irq_map[res_id]->map_data->len; 1691895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1692895c8b1cSMichal Meloun 1693895c8b1cSMichal Meloun if (len == 0) 1694895c8b1cSMichal Meloun *data = NULL; 1695895c8b1cSMichal Meloun else 1696895c8b1cSMichal Meloun *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); 1697895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1698895c8b1cSMichal Meloun if (irq_map[res_id] == NULL) 1699895c8b1cSMichal Meloun panic("Attempt to copy invalid resource id: %u\n", res_id); 1700895c8b1cSMichal Meloun if (len != 0) { 1701895c8b1cSMichal Meloun if (len != irq_map[res_id]->map_data->len) 1702895c8b1cSMichal Meloun panic("Resource id: %u has changed.\n", res_id); 1703895c8b1cSMichal Meloun memcpy(*data, irq_map[res_id]->map_data, len); 1704895c8b1cSMichal Meloun } 1705895c8b1cSMichal Meloun *map_dev = irq_map[res_id]->dev; 1706895c8b1cSMichal Meloun *map_xref = irq_map[res_id]->xref; 1707895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1708895c8b1cSMichal Meloun } 1709895c8b1cSMichal Meloun 1710895c8b1cSMichal Meloun /* 1711895c8b1cSMichal Meloun * Allocate and fill new entry in irq_map table. 1712895c8b1cSMichal Meloun */ 1713895c8b1cSMichal Meloun u_int 1714895c8b1cSMichal Meloun intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) 1715895c8b1cSMichal Meloun { 1716895c8b1cSMichal Meloun u_int i; 1717895c8b1cSMichal Meloun struct intr_map_entry *entry; 1718895c8b1cSMichal Meloun 1719895c8b1cSMichal Meloun /* Prepare new entry first. */ 1720895c8b1cSMichal Meloun entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); 1721895c8b1cSMichal Meloun 1722895c8b1cSMichal Meloun entry->dev = dev; 1723895c8b1cSMichal Meloun entry->xref = xref; 1724895c8b1cSMichal Meloun entry->map_data = data; 1725895c8b1cSMichal Meloun entry->isrc = NULL; 1726895c8b1cSMichal Meloun 1727895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1728895c8b1cSMichal Meloun for (i = irq_map_first_free_idx; i < irq_map_count; i++) { 1729895c8b1cSMichal Meloun if (irq_map[i] == NULL) { 1730895c8b1cSMichal Meloun irq_map[i] = entry; 1731895c8b1cSMichal Meloun irq_map_first_free_idx = i + 1; 1732895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1733895c8b1cSMichal Meloun return (i); 1734895c8b1cSMichal Meloun } 1735895c8b1cSMichal Meloun } 17369beb195fSAndrew Turner for (i = 0; i < irq_map_first_free_idx; i++) { 17379beb195fSAndrew Turner if (irq_map[i] == NULL) { 17389beb195fSAndrew Turner irq_map[i] = entry; 17399beb195fSAndrew Turner irq_map_first_free_idx = i + 1; 17409beb195fSAndrew Turner mtx_unlock(&irq_map_lock); 17419beb195fSAndrew Turner return (i); 17429beb195fSAndrew Turner } 17439beb195fSAndrew Turner } 1744895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1745895c8b1cSMichal Meloun 1746895c8b1cSMichal Meloun /* XXX Expand irq_map table */ 1747895c8b1cSMichal Meloun panic("IRQ mapping table is full."); 1748895c8b1cSMichal Meloun } 1749895c8b1cSMichal Meloun 1750895c8b1cSMichal Meloun /* 1751895c8b1cSMichal Meloun * Remove and free mapping entry. 1752895c8b1cSMichal Meloun */ 1753895c8b1cSMichal Meloun void 1754895c8b1cSMichal Meloun intr_unmap_irq(u_int res_id) 1755895c8b1cSMichal Meloun { 1756895c8b1cSMichal Meloun struct intr_map_entry *entry; 1757895c8b1cSMichal Meloun 1758895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1759895c8b1cSMichal Meloun if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) 1760895c8b1cSMichal Meloun panic("Attempt to unmap invalid resource id: %u\n", res_id); 1761895c8b1cSMichal Meloun entry = irq_map[res_id]; 1762895c8b1cSMichal Meloun irq_map[res_id] = NULL; 1763895c8b1cSMichal Meloun irq_map_first_free_idx = res_id; 1764895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1765895c8b1cSMichal Meloun intr_free_intr_map_data(entry->map_data); 1766895c8b1cSMichal Meloun free(entry, M_INTRNG); 1767895c8b1cSMichal Meloun } 1768895c8b1cSMichal Meloun 1769895c8b1cSMichal Meloun /* 1770895c8b1cSMichal Meloun * Clone mapping entry. 1771895c8b1cSMichal Meloun */ 1772895c8b1cSMichal Meloun u_int 1773895c8b1cSMichal Meloun intr_map_clone_irq(u_int old_res_id) 1774895c8b1cSMichal Meloun { 1775895c8b1cSMichal Meloun device_t map_dev; 1776895c8b1cSMichal Meloun intptr_t map_xref; 1777895c8b1cSMichal Meloun struct intr_map_data *data; 1778895c8b1cSMichal Meloun 1779895c8b1cSMichal Meloun intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); 1780895c8b1cSMichal Meloun return (intr_map_irq(map_dev, map_xref, data)); 1781895c8b1cSMichal Meloun } 1782895c8b1cSMichal Meloun 1783895c8b1cSMichal Meloun static void 1784895c8b1cSMichal Meloun intr_map_init(void *dummy __unused) 1785895c8b1cSMichal Meloun { 1786895c8b1cSMichal Meloun 1787895c8b1cSMichal Meloun mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); 1788248f0cabSOleksandr Tymoshenko 1789248f0cabSOleksandr Tymoshenko irq_map_count = 2 * intr_nirq; 1790248f0cabSOleksandr Tymoshenko irq_map = mallocarray(irq_map_count, sizeof(struct intr_map_entry*), 1791248f0cabSOleksandr Tymoshenko M_INTRNG, M_WAITOK | M_ZERO); 1792895c8b1cSMichal Meloun } 1793895c8b1cSMichal Meloun SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); 1794fae8755fSJessica Clarke 1795fae8755fSJessica Clarke #ifdef SMP 1796fae8755fSJessica Clarke /* Virtualization for interrupt source IPI counter increment. */ 1797fae8755fSJessica Clarke static inline void 1798fae8755fSJessica Clarke intr_ipi_increment_count(u_long *counter, u_int cpu) 1799fae8755fSJessica Clarke { 1800fae8755fSJessica Clarke 1801fae8755fSJessica Clarke KASSERT(cpu < mp_maxid + 1, ("%s: too big cpu %u", __func__, cpu)); 1802fae8755fSJessica Clarke counter[cpu]++; 1803fae8755fSJessica Clarke } 1804fae8755fSJessica Clarke 1805fae8755fSJessica Clarke /* 1806fae8755fSJessica Clarke * Virtualization for interrupt source IPI counters setup. 1807fae8755fSJessica Clarke */ 1808fae8755fSJessica Clarke static u_long * 1809fae8755fSJessica Clarke intr_ipi_setup_counters(const char *name) 1810fae8755fSJessica Clarke { 1811fae8755fSJessica Clarke u_int index, i; 1812fae8755fSJessica Clarke char str[INTRNAME_LEN]; 1813fae8755fSJessica Clarke 1814fae8755fSJessica Clarke mtx_lock(&isrc_table_lock); 1815fae8755fSJessica Clarke 1816fae8755fSJessica Clarke /* 1817fae8755fSJessica Clarke * We should never have a problem finding mp_maxid + 1 contiguous 1818fae8755fSJessica Clarke * counters, in practice. Interrupts will be allocated sequentially 1819fae8755fSJessica Clarke * during boot, so the array should fill from low to high index. Once 1820fae8755fSJessica Clarke * reserved, the IPI counters will never be released. Similarly, we 1821fae8755fSJessica Clarke * will not need to allocate more IPIs once the system is running. 1822fae8755fSJessica Clarke */ 1823fae8755fSJessica Clarke bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index); 1824fae8755fSJessica Clarke if (index == -1) 1825fae8755fSJessica Clarke panic("Failed to allocate %d counters. Array exhausted?", 1826fae8755fSJessica Clarke mp_maxid + 1); 1827fae8755fSJessica Clarke bit_nset(intrcnt_bitmap, index, index + mp_maxid); 1828fae8755fSJessica Clarke for (i = 0; i < mp_maxid + 1; i++) { 1829fae8755fSJessica Clarke snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 1830fae8755fSJessica Clarke intrcnt_setname(str, index + i); 1831fae8755fSJessica Clarke } 1832fae8755fSJessica Clarke mtx_unlock(&isrc_table_lock); 1833fae8755fSJessica Clarke return (&intrcnt[index]); 1834fae8755fSJessica Clarke } 1835fae8755fSJessica Clarke 1836fae8755fSJessica Clarke /* 1837fae8755fSJessica Clarke * Lookup IPI source. 1838fae8755fSJessica Clarke */ 1839fae8755fSJessica Clarke static struct intr_ipi * 1840fae8755fSJessica Clarke intr_ipi_lookup(u_int ipi) 1841fae8755fSJessica Clarke { 1842fae8755fSJessica Clarke 1843fae8755fSJessica Clarke if (ipi >= INTR_IPI_COUNT) 1844fae8755fSJessica Clarke panic("%s: no such IPI %u", __func__, ipi); 1845fae8755fSJessica Clarke 1846fae8755fSJessica Clarke return (&ipi_sources[ipi]); 1847fae8755fSJessica Clarke } 1848fae8755fSJessica Clarke 1849103d39efSJessica Clarke int 1850103d39efSJessica Clarke intr_ipi_pic_register(device_t dev, u_int priority) 1851103d39efSJessica Clarke { 1852103d39efSJessica Clarke if (intr_ipi_dev_frozen) { 1853103d39efSJessica Clarke device_printf(dev, "IPI device already frozen"); 1854103d39efSJessica Clarke return (EBUSY); 1855103d39efSJessica Clarke } 1856103d39efSJessica Clarke 18573b03e1bbSAndrew Turner if (intr_ipi_dev == NULL || priority > intr_ipi_dev_priority) { 18583b03e1bbSAndrew Turner intr_ipi_dev_priority = priority; 1859103d39efSJessica Clarke intr_ipi_dev = dev; 18603b03e1bbSAndrew Turner } 1861103d39efSJessica Clarke 1862103d39efSJessica Clarke return (0); 1863103d39efSJessica Clarke } 1864103d39efSJessica Clarke 1865fae8755fSJessica Clarke /* 1866fae8755fSJessica Clarke * Setup IPI handler on interrupt controller. 1867fae8755fSJessica Clarke * 1868fae8755fSJessica Clarke * Not SMP coherent. 1869fae8755fSJessica Clarke */ 1870fae8755fSJessica Clarke void 1871fae8755fSJessica Clarke intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, 1872fae8755fSJessica Clarke void *arg) 1873fae8755fSJessica Clarke { 1874fae8755fSJessica Clarke struct intr_irqsrc *isrc; 1875fae8755fSJessica Clarke struct intr_ipi *ii; 1876fae8755fSJessica Clarke int error; 1877fae8755fSJessica Clarke 1878103d39efSJessica Clarke if (!intr_ipi_dev_frozen) { 1879103d39efSJessica Clarke if (intr_ipi_dev == NULL) 1880103d39efSJessica Clarke panic("%s: no IPI PIC attached", __func__); 1881103d39efSJessica Clarke 1882103d39efSJessica Clarke intr_ipi_dev_frozen = true; 1883103d39efSJessica Clarke device_printf(intr_ipi_dev, "using for IPIs\n"); 1884103d39efSJessica Clarke } 1885103d39efSJessica Clarke 1886fae8755fSJessica Clarke KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); 1887fae8755fSJessica Clarke 1888103d39efSJessica Clarke error = PIC_IPI_SETUP(intr_ipi_dev, ipi, &isrc); 1889fae8755fSJessica Clarke if (error != 0) 1890fae8755fSJessica Clarke return; 1891fae8755fSJessica Clarke 1892fae8755fSJessica Clarke isrc->isrc_handlers++; 1893fae8755fSJessica Clarke 1894fae8755fSJessica Clarke ii = intr_ipi_lookup(ipi); 1895fae8755fSJessica Clarke KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); 1896fae8755fSJessica Clarke 1897fae8755fSJessica Clarke ii->ii_handler = hand; 1898fae8755fSJessica Clarke ii->ii_handler_arg = arg; 1899fae8755fSJessica Clarke ii->ii_isrc = isrc; 1900fae8755fSJessica Clarke strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); 1901fae8755fSJessica Clarke ii->ii_count = intr_ipi_setup_counters(name); 1902fae8755fSJessica Clarke 1903103d39efSJessica Clarke PIC_ENABLE_INTR(intr_ipi_dev, isrc); 1904fae8755fSJessica Clarke } 1905fae8755fSJessica Clarke 1906fae8755fSJessica Clarke void 1907fae8755fSJessica Clarke intr_ipi_send(cpuset_t cpus, u_int ipi) 1908fae8755fSJessica Clarke { 1909fae8755fSJessica Clarke struct intr_ipi *ii; 1910fae8755fSJessica Clarke 1911103d39efSJessica Clarke KASSERT(intr_ipi_dev_frozen, 1912103d39efSJessica Clarke ("%s: IPI device not yet frozen", __func__)); 1913fae8755fSJessica Clarke 1914fae8755fSJessica Clarke ii = intr_ipi_lookup(ipi); 1915fae8755fSJessica Clarke if (ii->ii_count == NULL) 1916fae8755fSJessica Clarke panic("%s: not setup IPI %u", __func__, ipi); 1917fae8755fSJessica Clarke 1918fae8755fSJessica Clarke /* 1919fae8755fSJessica Clarke * XXX: Surely needed on other architectures too? Either way should be 1920fae8755fSJessica Clarke * some kind of MI hook defined in an MD header, or the responsibility 1921fae8755fSJessica Clarke * of the MD caller if not widespread. 1922fae8755fSJessica Clarke */ 1923fae8755fSJessica Clarke #ifdef __aarch64__ 1924fae8755fSJessica Clarke /* 1925fae8755fSJessica Clarke * Ensure that this CPU's stores will be visible to IPI 1926fae8755fSJessica Clarke * recipients before starting to send the interrupts. 1927fae8755fSJessica Clarke */ 1928fae8755fSJessica Clarke dsb(ishst); 1929fae8755fSJessica Clarke #endif 1930fae8755fSJessica Clarke 1931103d39efSJessica Clarke PIC_IPI_SEND(intr_ipi_dev, ii->ii_isrc, cpus, ipi); 1932fae8755fSJessica Clarke } 1933fae8755fSJessica Clarke 1934fae8755fSJessica Clarke /* 1935fae8755fSJessica Clarke * interrupt controller dispatch function for IPIs. It should 1936fae8755fSJessica Clarke * be called straight from the interrupt controller, when associated 1937fae8755fSJessica Clarke * interrupt source is learned. Or from anybody who has an interrupt 1938fae8755fSJessica Clarke * source mapped. 1939fae8755fSJessica Clarke */ 1940fae8755fSJessica Clarke void 1941fae8755fSJessica Clarke intr_ipi_dispatch(u_int ipi) 1942fae8755fSJessica Clarke { 1943fae8755fSJessica Clarke struct intr_ipi *ii; 1944fae8755fSJessica Clarke 1945fae8755fSJessica Clarke ii = intr_ipi_lookup(ipi); 1946fae8755fSJessica Clarke if (ii->ii_count == NULL) 1947fae8755fSJessica Clarke panic("%s: not setup IPI %u", __func__, ipi); 1948fae8755fSJessica Clarke 1949fae8755fSJessica Clarke intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); 1950fae8755fSJessica Clarke 1951fae8755fSJessica Clarke ii->ii_handler(ii->ii_handler_arg); 1952fae8755fSJessica Clarke } 1953fae8755fSJessica Clarke #endif 1954