1 /* $NetBSD: gic_v2m.c,v 1.9 2020/05/07 16:20:40 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jared McNeill <jmcneill@invisible.ca>. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _INTR_PRIVATE 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: gic_v2m.c,v 1.9 2020/05/07 16:20:40 jmcneill Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/kmem.h> 39 #include <sys/bitops.h> 40 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #include <arm/pic/picvar.h> 45 #include <arm/cortex/gic_v2m.h> 46 47 static uint64_t 48 gic_v2m_msi_addr(struct gic_v2m_frame *frame, int spi) 49 { 50 if ((frame->frame_flags & GIC_V2M_FLAG_GRAVITON) != 0) 51 return frame->frame_reg + ((spi - 32) << 3); 52 53 return frame->frame_reg + GIC_MSI_SETSPI; 54 } 55 56 static uint32_t 57 gic_v2m_msi_data(struct gic_v2m_frame *frame, int spi) 58 { 59 if ((frame->frame_flags & GIC_V2M_FLAG_GRAVITON) != 0) 60 return 0; 61 62 return spi; 63 } 64 65 static int 66 gic_v2m_msi_alloc_spi(struct gic_v2m_frame *frame, int count, 67 const struct pci_attach_args *pa) 68 { 69 struct pci_attach_args *new_pa; 70 int spi, n; 71 72 for (spi = frame->frame_base; 73 spi < frame->frame_base + frame->frame_count; ) { 74 if (frame->frame_pa[spi] == NULL) { 75 for (n = 1; n < count; n++) 76 if (frame->frame_pa[spi + n] != NULL) 77 goto next_spi; 78 79 for (n = 0; n < count; n++) { 80 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 81 memcpy(new_pa, pa, sizeof(*new_pa)); 82 frame->frame_pa[spi + n] = new_pa; 83 } 84 85 return spi; 86 } 87 next_spi: 88 spi += count; 89 } 90 91 return -1; 92 } 93 94 static void 95 gic_v2m_msi_free_spi(struct gic_v2m_frame *frame, int spi) 96 { 97 struct pci_attach_args *pa; 98 99 pa = frame->frame_pa[spi]; 100 frame->frame_pa[spi] = NULL; 101 102 if (pa != NULL) 103 kmem_free(pa, sizeof(*pa)); 104 } 105 106 static int 107 gic_v2m_msi_available_spi(struct gic_v2m_frame *frame) 108 { 109 int spi, n; 110 111 for (spi = frame->frame_base, n = 0; 112 spi < frame->frame_base + frame->frame_count; 113 spi++) { 114 if (frame->frame_pa[spi] == NULL) 115 n++; 116 } 117 118 return n; 119 } 120 121 static void 122 gic_v2m_msi_enable(struct gic_v2m_frame *frame, int spi, int count) 123 { 124 const struct pci_attach_args *pa = frame->frame_pa[spi]; 125 pci_chipset_tag_t pc = pa->pa_pc; 126 pcitag_t tag = pa->pa_tag; 127 pcireg_t ctl; 128 int off; 129 130 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 131 panic("gic_v2m_msi_enable: device is not MSI-capable"); 132 133 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 134 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 135 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 136 137 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 138 ctl &= ~PCI_MSI_CTL_MME_MASK; 139 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 140 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 141 142 const uint64_t addr = gic_v2m_msi_addr(frame, spi); 143 const uint32_t data = gic_v2m_msi_data(frame, spi); 144 145 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 146 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 147 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 148 addr & 0xffffffff); 149 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 150 (addr >> 32) & 0xffffffff); 151 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); 152 } else { 153 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 154 addr & 0xffffffff); 155 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); 156 } 157 ctl |= PCI_MSI_CTL_MSI_ENABLE; 158 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 159 } 160 161 static void 162 gic_v2m_msi_disable(struct gic_v2m_frame *frame, int spi) 163 { 164 const struct pci_attach_args *pa = frame->frame_pa[spi]; 165 pci_chipset_tag_t pc = pa->pa_pc; 166 pcitag_t tag = pa->pa_tag; 167 pcireg_t ctl; 168 int off; 169 170 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 171 panic("gic_v2m_msi_disable: device is not MSI-capable"); 172 173 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 174 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 175 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 176 } 177 178 static void 179 gic_v2m_msix_enable(struct gic_v2m_frame *frame, int spi, int msix_vec, 180 bus_space_tag_t bst, bus_space_handle_t bsh) 181 { 182 const struct pci_attach_args *pa = frame->frame_pa[spi]; 183 pci_chipset_tag_t pc = pa->pa_pc; 184 pcitag_t tag = pa->pa_tag; 185 pcireg_t ctl; 186 int off; 187 188 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 189 panic("gic_v2m_msix_enable: device is not MSI-X-capable"); 190 191 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 192 ctl &= ~PCI_MSIX_CTL_ENABLE; 193 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 194 195 const uint64_t addr = gic_v2m_msi_addr(frame, spi); 196 const uint32_t data = gic_v2m_msi_data(frame, spi); 197 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 198 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr); 199 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32)); 200 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data); 201 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0); 202 203 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 204 ctl |= PCI_MSIX_CTL_ENABLE; 205 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 206 } 207 208 static void 209 gic_v2m_msix_disable(struct gic_v2m_frame *frame, int spi) 210 { 211 const struct pci_attach_args *pa = frame->frame_pa[spi]; 212 pci_chipset_tag_t pc = pa->pa_pc; 213 pcitag_t tag = pa->pa_tag; 214 pcireg_t ctl; 215 int off; 216 217 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 218 panic("gic_v2m_msix_disable: device is not MSI-X-capable"); 219 220 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 221 ctl &= ~PCI_MSIX_CTL_ENABLE; 222 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 223 } 224 225 static pci_intr_handle_t * 226 gic_v2m_msi_alloc(struct arm_pci_msi *msi, int *count, 227 const struct pci_attach_args *pa, bool exact) 228 { 229 struct gic_v2m_frame * const frame = msi->msi_priv; 230 pci_intr_handle_t *vectors; 231 int n, off; 232 233 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 234 return NULL; 235 236 const int avail = gic_v2m_msi_available_spi(frame); 237 if (exact && *count > avail) 238 return NULL; 239 240 while (*count > avail) { 241 if (avail < *count) 242 (*count) >>= 1; 243 } 244 if (*count == 0) 245 return NULL; 246 247 const int spi_base = gic_v2m_msi_alloc_spi(frame, *count, pa); 248 if (spi_base == -1) 249 return NULL; 250 251 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 252 for (n = 0; n < *count; n++) { 253 const int spi = spi_base + n; 254 vectors[n] = ARM_PCI_INTR_MSI | 255 __SHIFTIN(spi, ARM_PCI_INTR_IRQ) | 256 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | 257 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 258 } 259 260 gic_v2m_msi_enable(frame, spi_base, *count); 261 262 return vectors; 263 } 264 265 static pci_intr_handle_t * 266 gic_v2m_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, 267 const struct pci_attach_args *pa, bool exact) 268 { 269 struct gic_v2m_frame * const frame = msi->msi_priv; 270 pci_intr_handle_t *vectors; 271 bus_space_tag_t bst; 272 bus_space_handle_t bsh; 273 bus_size_t bsz; 274 uint32_t table_offset, table_size; 275 int n, off, bar, error; 276 pcireg_t tbl; 277 278 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 279 return NULL; 280 281 const int avail = gic_v2m_msi_available_spi(frame); 282 if (exact && *count > avail) 283 return NULL; 284 285 while (*count > avail) { 286 if (avail < *count) 287 (*count) >>= 1; 288 } 289 if (*count == 0) 290 return NULL; 291 292 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 293 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 294 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 295 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 296 if (table_size == 0) 297 return NULL; 298 299 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 300 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 301 &bst, &bsh, NULL, &bsz); 302 if (error) 303 return NULL; 304 305 const int spi_base = gic_v2m_msi_alloc_spi(frame, *count, pa); 306 if (spi_base == -1) { 307 bus_space_unmap(bst, bsh, bsz); 308 return NULL; 309 } 310 311 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 312 for (n = 0; n < *count; n++) { 313 const int spi = spi_base + n; 314 const int msix_vec = table_indexes ? table_indexes[n] : n; 315 vectors[msix_vec] = ARM_PCI_INTR_MSIX | 316 __SHIFTIN(spi, ARM_PCI_INTR_IRQ) | 317 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | 318 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 319 320 gic_v2m_msix_enable(frame, spi, msix_vec, bst, bsh); 321 } 322 323 bus_space_unmap(bst, bsh, bsz); 324 325 return vectors; 326 } 327 328 static void * 329 gic_v2m_msi_intr_establish(struct arm_pci_msi *msi, 330 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 331 { 332 struct gic_v2m_frame * const frame = msi->msi_priv; 333 334 const int spi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); 335 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0; 336 337 return pic_establish_intr(frame->frame_pic, spi, ipl, 338 IST_EDGE | mpsafe, func, arg, xname); 339 } 340 341 static void 342 gic_v2m_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, 343 int count) 344 { 345 struct gic_v2m_frame * const frame = msi->msi_priv; 346 int n; 347 348 for (n = 0; n < count; n++) { 349 const int spi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); 350 if (pih[n] & ARM_PCI_INTR_MSIX) 351 gic_v2m_msix_disable(frame, spi); 352 if (pih[n] & ARM_PCI_INTR_MSI) 353 gic_v2m_msi_disable(frame, spi); 354 gic_v2m_msi_free_spi(frame, spi); 355 struct intrsource * const is = 356 frame->frame_pic->pic_sources[spi]; 357 if (is != NULL) 358 pic_disestablish_source(is); 359 } 360 } 361 362 int 363 gic_v2m_init(struct gic_v2m_frame *frame, device_t dev, uint32_t frame_id) 364 { 365 struct arm_pci_msi *msi = &frame->frame_msi; 366 367 msi->msi_dev = dev; 368 msi->msi_priv = frame; 369 msi->msi_alloc = gic_v2m_msi_alloc; 370 msi->msix_alloc = gic_v2m_msix_alloc; 371 msi->msi_intr_establish = gic_v2m_msi_intr_establish; 372 msi->msi_intr_release = gic_v2m_msi_intr_release; 373 374 return arm_pci_msi_add(msi); 375 } 376