1 /* $NetBSD: gicv3_its.c,v 1.27 2020/02/13 02:12:06 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jared McNeill <jmcneill@invisible.ca>. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _INTR_PRIVATE 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.27 2020/02/13 02:12:06 jmcneill Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/kmem.h> 39 #include <sys/bus.h> 40 #include <sys/cpu.h> 41 #include <sys/bitops.h> 42 43 #include <uvm/uvm.h> 44 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 48 #include <arm/pic/picvar.h> 49 #include <arm/cortex/gicv3_its.h> 50 51 /* 52 * ITS translation table sizes 53 */ 54 #define GITS_COMMANDS_SIZE 0x1000 55 #define GITS_COMMANDS_ALIGN 0x10000 56 57 #define GITS_ITT_ALIGN 0x100 58 59 /* 60 * IIDR values used for errata 61 */ 62 #define GITS_IIDR_PID_CAVIUM_THUNDERX 0xa1 63 #define GITS_IIDR_IMP_CAVIUM 0x34c 64 #define GITS_IIDR_CAVIUM_ERRATA_MASK (GITS_IIDR_Implementor|GITS_IIDR_ProductID|GITS_IIDR_Variant) 65 #define GITS_IIDR_CAVIUM_ERRATA_VALUE \ 66 (__SHIFTIN(GITS_IIDR_IMP_CAVIUM, GITS_IIDR_Implementor) | \ 67 __SHIFTIN(GITS_IIDR_PID_CAVIUM_THUNDERX, GITS_IIDR_ProductID) | \ 68 __SHIFTIN(0, GITS_IIDR_Variant)) 69 70 static const char * gits_cache_type[] = { 71 [GITS_Cache_DEVICE_nGnRnE] = "Device-nGnRnE", 72 [GITS_Cache_NORMAL_NC] = "Non-cacheable", 73 [GITS_Cache_NORMAL_RA_WT] = "Cacheable RA WT", 74 [GITS_Cache_NORMAL_RA_WB] = "Cacheable RA WB", 75 [GITS_Cache_NORMAL_WA_WT] = "Cacheable WA WT", 76 [GITS_Cache_NORMAL_WA_WB] = "Cacheable WA WB", 77 [GITS_Cache_NORMAL_RA_WA_WT] = "Cacheable RA WA WT", 78 [GITS_Cache_NORMAL_RA_WA_WB] = "Cacheable RA WA WB", 79 }; 80 81 static const char * gits_share_type[] = { 82 [GITS_Shareability_NS] = "Non-shareable", 83 [GITS_Shareability_IS] = "Inner shareable", 84 [GITS_Shareability_OS] = "Outer shareable", 85 [3] = "(Reserved)", 86 }; 87 88 static inline uint32_t 89 gits_read_4(struct gicv3_its *its, bus_size_t reg) 90 { 91 return bus_space_read_4(its->its_bst, its->its_bsh, reg); 92 } 93 94 static inline void 95 gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val) 96 { 97 bus_space_write_4(its->its_bst, its->its_bsh, reg, val); 98 } 99 100 static inline uint64_t 101 gits_read_8(struct gicv3_its *its, bus_size_t reg) 102 { 103 return bus_space_read_8(its->its_bst, its->its_bsh, reg); 104 } 105 106 static inline void 107 gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val) 108 { 109 bus_space_write_8(its->its_bst, its->its_bsh, reg, val); 110 } 111 112 static inline void 113 gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd) 114 { 115 uint64_t cwriter; 116 u_int woff; 117 118 cwriter = gits_read_8(its, GITS_CWRITER); 119 woff = cwriter & GITS_CWRITER_Offset; 120 121 memcpy(its->its_cmd.base + woff, cmd->dw, sizeof(cmd->dw)); 122 bus_dmamap_sync(its->its_dmat, its->its_cmd.map, woff, sizeof(cmd->dw), BUS_DMASYNC_PREWRITE); 123 124 woff += sizeof(cmd->dw); 125 if (woff == its->its_cmd.len) 126 woff = 0; 127 128 gits_write_8(its, GITS_CWRITER, woff); 129 } 130 131 static inline void 132 gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v) 133 { 134 struct gicv3_its_command cmd; 135 136 KASSERT((rdbase & 0xffff) == 0); 137 138 /* 139 * Map a collection table entry (ICID) to the target redistributor (RDbase). 140 */ 141 memset(&cmd, 0, sizeof(cmd)); 142 cmd.dw[0] = GITS_CMD_MAPC; 143 cmd.dw[2] = icid; 144 if (v) { 145 cmd.dw[2] |= rdbase; 146 cmd.dw[2] |= __BIT(63); 147 } 148 149 gits_command(its, &cmd); 150 } 151 152 static inline void 153 gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v) 154 { 155 struct gicv3_its_command cmd; 156 157 KASSERT((itt_addr & 0xff) == 0); 158 159 /* 160 * Map a device table entry (DeviceID) to its associated ITT (ITT_addr). 161 */ 162 memset(&cmd, 0, sizeof(cmd)); 163 cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32); 164 cmd.dw[1] = size; 165 if (v) { 166 cmd.dw[2] = itt_addr | __BIT(63); 167 } 168 169 gits_command(its, &cmd); 170 } 171 172 static inline void 173 gits_command_mapti(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint32_t pintid, uint16_t icid) 174 { 175 struct gicv3_its_command cmd; 176 177 /* 178 * Map the event defined by EventID and DeviceID to its associated ITE, defined by ICID and pINTID 179 * in the ITT associated with DeviceID. 180 */ 181 memset(&cmd, 0, sizeof(cmd)); 182 cmd.dw[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32); 183 cmd.dw[1] = eventid | ((uint64_t)pintid << 32); 184 cmd.dw[2] = icid; 185 186 gits_command(its, &cmd); 187 } 188 189 static inline void 190 gits_command_movi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid) 191 { 192 struct gicv3_its_command cmd; 193 194 /* 195 * Update the ICID field in the ITT entry for the event defined by DeviceID and 196 * EventID. 197 */ 198 memset(&cmd, 0, sizeof(cmd)); 199 cmd.dw[0] = GITS_CMD_MOVI | ((uint64_t)deviceid << 32); 200 cmd.dw[1] = eventid; 201 cmd.dw[2] = icid; 202 203 gits_command(its, &cmd); 204 } 205 206 static inline void 207 gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) 208 { 209 struct gicv3_its_command cmd; 210 211 /* 212 * Ensure any caching in the redistributors associated with the specified 213 * EventID is consistent with the LPI configuration tables. 214 */ 215 memset(&cmd, 0, sizeof(cmd)); 216 cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32); 217 cmd.dw[1] = eventid; 218 219 gits_command(its, &cmd); 220 } 221 222 static inline void 223 gits_command_invall(struct gicv3_its *its, uint16_t icid) 224 { 225 struct gicv3_its_command cmd; 226 227 /* 228 * Ensure any caching associated with this ICID is consistent with LPI 229 * configuration tables for all redistributors. 230 */ 231 memset(&cmd, 0, sizeof(cmd)); 232 cmd.dw[0] = GITS_CMD_INVALL; 233 cmd.dw[2] = icid; 234 235 gits_command(its, &cmd); 236 } 237 238 static inline void 239 gits_command_sync(struct gicv3_its *its, uint64_t rdbase) 240 { 241 struct gicv3_its_command cmd; 242 243 KASSERT((rdbase & 0xffff) == 0); 244 245 /* 246 * Ensure all outstanding ITS operations associated with physical interrupts 247 * for the specified redistributor (RDbase) are globally observed before 248 * further ITS commands are executed. 249 */ 250 memset(&cmd, 0, sizeof(cmd)); 251 cmd.dw[0] = GITS_CMD_SYNC; 252 cmd.dw[2] = rdbase; 253 254 gits_command(its, &cmd); 255 } 256 257 static inline int 258 gits_wait(struct gicv3_its *its) 259 { 260 u_int woff, roff; 261 int retry = 100000; 262 263 /* 264 * The ITS command queue is empty when CWRITER and CREADR specify the 265 * same base address offset value. 266 */ 267 for (retry = 1000; retry > 0; retry--) { 268 woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset; 269 roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset; 270 if (woff == roff) 271 break; 272 delay(100); 273 } 274 if (retry == 0) { 275 device_printf(its->its_gic->sc_dev, "ITS command queue timeout\n"); 276 return ETIMEDOUT; 277 } 278 279 return 0; 280 } 281 282 static int 283 gicv3_its_msi_alloc_lpi(struct gicv3_its *its, 284 const struct pci_attach_args *pa) 285 { 286 struct pci_attach_args *new_pa; 287 vmem_addr_t n; 288 289 KASSERT(its->its_gic->sc_lpi_pool != NULL); 290 291 if (vmem_alloc(its->its_gic->sc_lpi_pool, 1, VM_INSTANTFIT|VM_SLEEP, &n) != 0) 292 return -1; 293 294 KASSERT(its->its_pa[n] == NULL); 295 296 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 297 memcpy(new_pa, pa, sizeof(*new_pa)); 298 its->its_pa[n] = new_pa; 299 return n + its->its_pic->pic_irqbase; 300 } 301 302 static void 303 gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi) 304 { 305 struct pci_attach_args *pa; 306 307 KASSERT(its->its_gic->sc_lpi_pool != NULL); 308 KASSERT(lpi >= its->its_pic->pic_irqbase); 309 310 pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 311 its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL; 312 kmem_free(pa, sizeof(*pa)); 313 314 vmem_free(its->its_gic->sc_lpi_pool, lpi - its->its_pic->pic_irqbase, 1); 315 } 316 317 static uint32_t 318 gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag) 319 { 320 uint32_t devid; 321 int b, d, f; 322 323 pci_decompose_tag(pc, tag, &b, &d, &f); 324 325 devid = (b << 8) | (d << 3) | f; 326 327 return pci_get_devid(pc, devid); 328 } 329 330 static int 331 gicv3_its_device_map(struct gicv3_its *its, uint32_t devid, u_int count) 332 { 333 struct gicv3_its_device *dev; 334 u_int vectors; 335 336 vectors = MAX(2, count); 337 while (!powerof2(vectors)) 338 vectors++; 339 340 const uint64_t typer = gits_read_8(its, GITS_TYPER); 341 const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1; 342 const u_int itt_size = roundup(vectors * itt_entry_size, GITS_ITT_ALIGN); 343 344 LIST_FOREACH(dev, &its->its_devices, dev_list) 345 if (dev->dev_id == devid) { 346 return itt_size <= dev->dev_size ? 0 : EEXIST; 347 } 348 349 dev = kmem_alloc(sizeof(*dev), KM_SLEEP); 350 dev->dev_id = devid; 351 dev->dev_size = itt_size; 352 gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN); 353 LIST_INSERT_HEAD(&its->its_devices, dev, dev_list); 354 355 /* 356 * Map the device to the ITT 357 */ 358 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 359 gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true); 360 gits_wait(its); 361 362 return 0; 363 } 364 365 static void 366 gicv3_its_msi_enable(struct gicv3_its *its, int lpi, int count) 367 { 368 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 369 pci_chipset_tag_t pc = pa->pa_pc; 370 pcitag_t tag = pa->pa_tag; 371 pcireg_t ctl; 372 int off; 373 374 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 375 panic("gicv3_its_msi_enable: device is not MSI-capable"); 376 377 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 378 ctl &= ~PCI_MSI_CTL_MME_MASK; 379 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 380 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 381 382 const uint64_t addr = its->its_base + GITS_TRANSLATER; 383 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 384 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 385 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 386 addr & 0xffffffff); 387 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 388 (addr >> 32) & 0xffffffff); 389 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, 390 lpi - its->its_pic->pic_irqbase); 391 } else { 392 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 393 addr & 0xffffffff); 394 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, 395 lpi - its->its_pic->pic_irqbase); 396 } 397 ctl |= PCI_MSI_CTL_MSI_ENABLE; 398 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 399 } 400 401 static void 402 gicv3_its_msi_disable(struct gicv3_its *its, int lpi) 403 { 404 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 405 pci_chipset_tag_t pc = pa->pa_pc; 406 pcitag_t tag = pa->pa_tag; 407 pcireg_t ctl; 408 int off; 409 410 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 411 panic("gicv3_its_msi_enable: device is not MSI-capable"); 412 413 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 414 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 415 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 416 } 417 418 static void 419 gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec, 420 bus_space_tag_t bst, bus_space_handle_t bsh) 421 { 422 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 423 pci_chipset_tag_t pc = pa->pa_pc; 424 pcitag_t tag = pa->pa_tag; 425 pcireg_t ctl; 426 int off; 427 428 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 429 panic("gicv3_its_msix_enable: device is not MSI-X-capable"); 430 431 const uint64_t addr = its->its_base + GITS_TRANSLATER; 432 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 433 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr); 434 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32)); 435 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, lpi - its->its_pic->pic_irqbase); 436 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0); 437 438 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 439 ctl |= PCI_MSIX_CTL_ENABLE; 440 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 441 } 442 443 static void 444 gicv3_its_msix_disable(struct gicv3_its *its, int lpi) 445 { 446 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 447 pci_chipset_tag_t pc = pa->pa_pc; 448 pcitag_t tag = pa->pa_tag; 449 pcireg_t ctl; 450 int off; 451 452 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 453 panic("gicv3_its_msix_disable: device is not MSI-X-capable"); 454 455 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 456 ctl &= ~PCI_MSIX_CTL_ENABLE; 457 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 458 } 459 460 static pci_intr_handle_t * 461 gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count, 462 const struct pci_attach_args *pa, bool exact) 463 { 464 struct gicv3_its * const its = msi->msi_priv; 465 struct cpu_info * const ci = cpu_lookup(0); 466 pci_intr_handle_t *vectors; 467 int n, off; 468 469 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 470 return NULL; 471 472 const uint64_t typer = gits_read_8(its, GITS_TYPER); 473 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 474 if (*count == 0 || *count > (1 << id_bits)) 475 return NULL; 476 477 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 478 479 if (gicv3_its_device_map(its, devid, *count) != 0) 480 return NULL; 481 482 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 483 for (n = 0; n < *count; n++) { 484 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 485 vectors[n] = ARM_PCI_INTR_MSI | 486 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | 487 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | 488 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 489 490 if (n == 0) 491 gicv3_its_msi_enable(its, lpi, *count); 492 493 /* 494 * Record target PE 495 */ 496 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 497 498 /* 499 * Map event 500 */ 501 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci)); 502 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 503 } 504 gits_wait(its); 505 506 return vectors; 507 } 508 509 static pci_intr_handle_t * 510 gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, 511 const struct pci_attach_args *pa, bool exact) 512 { 513 struct gicv3_its * const its = msi->msi_priv; 514 struct cpu_info *ci = cpu_lookup(0); 515 pci_intr_handle_t *vectors; 516 bus_space_tag_t bst; 517 bus_space_handle_t bsh; 518 bus_size_t bsz; 519 uint32_t table_offset, table_size; 520 int n, off, bar, error; 521 pcireg_t tbl; 522 523 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 524 return NULL; 525 526 const uint64_t typer = gits_read_8(its, GITS_TYPER); 527 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 528 if (*count == 0 || *count > (1 << id_bits)) 529 return NULL; 530 531 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 532 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 533 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 534 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 535 if (table_size == 0) 536 return NULL; 537 538 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 539 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 540 &bst, &bsh, NULL, &bsz); 541 if (error) 542 return NULL; 543 544 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 545 546 if (gicv3_its_device_map(its, devid, *count) != 0) { 547 bus_space_unmap(bst, bsh, bsz); 548 return NULL; 549 } 550 551 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 552 for (n = 0; n < *count; n++) { 553 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 554 const int msix_vec = table_indexes ? table_indexes[n] : n; 555 vectors[msix_vec] = ARM_PCI_INTR_MSIX | 556 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | 557 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | 558 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 559 560 gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh); 561 562 /* 563 * Record target PE 564 */ 565 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 566 567 /* 568 * Map event 569 */ 570 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci)); 571 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 572 } 573 gits_wait(its); 574 575 bus_space_unmap(bst, bsh, bsz); 576 577 return vectors; 578 } 579 580 static void * 581 gicv3_its_msi_intr_establish(struct arm_pci_msi *msi, 582 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 583 { 584 struct gicv3_its * const its = msi->msi_priv; 585 const struct pci_attach_args *pa; 586 void *intrh; 587 588 const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); 589 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0; 590 591 intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl, 592 IST_EDGE | mpsafe, func, arg, xname); 593 if (intrh == NULL) 594 return NULL; 595 596 /* Invalidate LPI configuration tables */ 597 pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 598 KASSERT(pa != NULL); 599 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 600 gits_command_inv(its, devid, lpi - its->its_pic->pic_irqbase); 601 602 return intrh; 603 } 604 605 static void 606 gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, 607 int count) 608 { 609 struct gicv3_its * const its = msi->msi_priv; 610 int n; 611 612 for (n = 0; n < count; n++) { 613 const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); 614 KASSERT(lpi >= its->its_pic->pic_irqbase); 615 if (pih[n] & ARM_PCI_INTR_MSIX) 616 gicv3_its_msix_disable(its, lpi); 617 if (pih[n] & ARM_PCI_INTR_MSI) 618 gicv3_its_msi_disable(its, lpi); 619 gicv3_its_msi_free_lpi(its, lpi); 620 its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL; 621 struct intrsource * const is = 622 its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase]; 623 if (is != NULL) 624 pic_disestablish_source(is); 625 } 626 } 627 628 static void 629 gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its) 630 { 631 uint64_t cbaser; 632 633 gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN); 634 635 cbaser = its->its_cmd.segs[0].ds_addr; 636 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache); 637 cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability); 638 cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size); 639 cbaser |= GITS_CBASER_Valid; 640 641 gits_write_8(its, GITS_CBASER, cbaser); 642 gits_write_8(its, GITS_CWRITER, 0); 643 } 644 645 static void 646 gicv3_its_table_params(struct gicv3_softc *sc, struct gicv3_its *its, 647 u_int *devbits, u_int *innercache, u_int *share) 648 { 649 650 const uint64_t typer = gits_read_8(its, GITS_TYPER); 651 const uint32_t iidr = gits_read_4(its, GITS_IIDR); 652 653 /* Default values */ 654 *devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1; 655 *innercache = GITS_Cache_NORMAL_WA_WB; 656 *share = GITS_Shareability_IS; 657 658 /* Cavium ThunderX errata */ 659 if ((iidr & GITS_IIDR_CAVIUM_ERRATA_MASK) == GITS_IIDR_CAVIUM_ERRATA_VALUE) { 660 *devbits = 20; /* 8Mb */ 661 *innercache = GITS_Cache_DEVICE_nGnRnE; 662 aprint_normal_dev(sc->sc_dev, "Cavium ThunderX errata detected\n"); 663 } 664 } 665 666 static void 667 gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its) 668 { 669 u_int table_size, page_size, table_align; 670 u_int devbits, innercache, share; 671 const char *table_type; 672 uint64_t baser; 673 int tab; 674 675 gicv3_its_table_params(sc, its, &devbits, &innercache, &share); 676 677 for (tab = 0; tab < 8; tab++) { 678 baser = gits_read_8(its, GITS_BASERn(tab)); 679 680 const u_int entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1; 681 682 switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) { 683 case GITS_Page_Size_4KB: 684 page_size = 4096; 685 table_align = 4096; 686 break; 687 case GITS_Page_Size_16KB: 688 page_size = 16384; 689 table_align = 4096; 690 break; 691 case GITS_Page_Size_64KB: 692 default: 693 page_size = 65536; 694 table_align = 65536; 695 break; 696 } 697 698 switch (__SHIFTOUT(baser, GITS_BASER_Type)) { 699 case GITS_Type_Devices: 700 /* 701 * Table size scales with the width of the DeviceID. 702 */ 703 table_size = roundup(entry_size * (1 << devbits), page_size); 704 table_type = "Devices"; 705 break; 706 case GITS_Type_InterruptCollections: 707 /* 708 * Allocate space for one interrupt collection per CPU. 709 */ 710 table_size = roundup(entry_size * MAXCPUS, page_size); 711 table_type = "Collections"; 712 break; 713 default: 714 table_size = 0; 715 break; 716 } 717 718 if (table_size == 0) 719 continue; 720 721 gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align); 722 723 baser &= ~GITS_BASER_Size; 724 baser |= __SHIFTIN(table_size / page_size - 1, GITS_BASER_Size); 725 baser &= ~GITS_BASER_Physical_Address; 726 baser |= its->its_tab[tab].segs[0].ds_addr; 727 baser &= ~GITS_BASER_InnerCache; 728 baser |= __SHIFTIN(innercache, GITS_BASER_InnerCache); 729 baser &= ~GITS_BASER_Shareability; 730 baser |= __SHIFTIN(share, GITS_BASER_Shareability); 731 baser |= GITS_BASER_Valid; 732 733 gits_write_8(its, GITS_BASERn(tab), baser); 734 735 baser = gits_read_8(its, GITS_BASERn(tab)); 736 if (__SHIFTOUT(baser, GITS_BASER_Shareability) == GITS_Shareability_NS) { 737 baser &= ~GITS_BASER_InnerCache; 738 baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache); 739 740 gits_write_8(its, GITS_BASERn(tab), baser); 741 } 742 743 baser = gits_read_8(its, GITS_BASERn(tab)); 744 aprint_normal_dev(sc->sc_dev, "ITS [#%d] %s table @ %#lx/%#x, %s, %s\n", 745 tab, table_type, its->its_tab[tab].segs[0].ds_addr, table_size, 746 gits_cache_type[__SHIFTOUT(baser, GITS_BASER_InnerCache)], 747 gits_share_type[__SHIFTOUT(baser, GITS_BASER_Shareability)]); 748 } 749 } 750 751 static void 752 gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its) 753 { 754 uint32_t ctlr; 755 756 ctlr = gits_read_4(its, GITS_CTLR); 757 ctlr |= GITS_CTLR_Enabled; 758 gits_write_4(its, GITS_CTLR, ctlr); 759 } 760 761 static void 762 gicv3_its_cpu_init(void *priv, struct cpu_info *ci) 763 { 764 struct gicv3_its * const its = priv; 765 struct gicv3_softc * const sc = its->its_gic; 766 const struct pci_attach_args *pa; 767 uint64_t rdbase; 768 size_t irq; 769 770 const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER); 771 if (typer & GITS_TYPER_PTA) { 772 void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]); 773 rdbase = vtophys((vaddr_t)va); 774 } else { 775 rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16; 776 } 777 its->its_rdbase[cpu_index(ci)] = rdbase; 778 779 /* 780 * Map collection ID of this CPU's index to this CPU's redistributor. 781 */ 782 gits_command_mapc(its, cpu_index(ci), rdbase, true); 783 gits_command_invall(its, cpu_index(ci)); 784 gits_wait(its); 785 786 /* 787 * Update routing for LPIs targetting this CPU 788 */ 789 for (irq = 0; irq < its->its_pic->pic_maxsources; irq++) { 790 if (its->its_targets[irq] != ci) 791 continue; 792 pa = its->its_pa[irq]; 793 KASSERT(pa != NULL); 794 795 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 796 gits_command_movi(its, devid, irq, cpu_index(ci)); 797 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 798 } 799 800 its->its_cpuonline[cpu_index(ci)] = true; 801 } 802 803 static void 804 gicv3_its_get_affinity(void *priv, size_t irq, kcpuset_t *affinity) 805 { 806 struct gicv3_its * const its = priv; 807 struct cpu_info *ci; 808 809 ci = its->its_targets[irq]; 810 if (ci) 811 kcpuset_set(affinity, cpu_index(ci)); 812 } 813 814 static int 815 gicv3_its_set_affinity(void *priv, size_t irq, const kcpuset_t *affinity) 816 { 817 struct gicv3_its * const its = priv; 818 const struct pci_attach_args *pa; 819 struct cpu_info *ci; 820 821 const int set = kcpuset_countset(affinity); 822 if (set != 1) 823 return EINVAL; 824 825 pa = its->its_pa[irq]; 826 if (pa == NULL) 827 return EPASSTHROUGH; 828 829 ci = cpu_lookup(kcpuset_ffs(affinity) - 1); 830 its->its_targets[irq] = ci; 831 832 if (its->its_cpuonline[cpu_index(ci)] == true) { 833 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 834 gits_command_movi(its, devid, irq, cpu_index(ci)); 835 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 836 } 837 838 return 0; 839 } 840 841 int 842 gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, 843 uint64_t its_base, uint32_t its_id) 844 { 845 struct gicv3_its *its; 846 struct arm_pci_msi *msi; 847 848 const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER); 849 if ((typer & GITS_TYPER_Physical) == 0) 850 return ENXIO; 851 852 its = kmem_zalloc(sizeof(*its), KM_SLEEP); 853 its->its_id = its_id; 854 its->its_bst = sc->sc_bst; 855 its->its_bsh = bsh; 856 its->its_dmat = sc->sc_dmat; 857 its->its_base = its_base; 858 its->its_pic = &sc->sc_lpi; 859 snprintf(its->its_pic->pic_name, sizeof(its->its_pic->pic_name), "gicv3-its"); 860 KASSERT(its->its_pic->pic_maxsources > 0); 861 its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP); 862 its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP); 863 its->its_gic = sc; 864 its->its_cb.cpu_init = gicv3_its_cpu_init; 865 its->its_cb.get_affinity = gicv3_its_get_affinity; 866 its->its_cb.set_affinity = gicv3_its_set_affinity; 867 its->its_cb.priv = its; 868 LIST_INIT(&its->its_devices); 869 LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list); 870 871 gicv3_its_command_init(sc, its); 872 gicv3_its_table_init(sc, its); 873 874 gicv3_its_enable(sc, its); 875 876 gicv3_its_cpu_init(its, curcpu()); 877 878 msi = &its->its_msi; 879 msi->msi_id = its_id; 880 msi->msi_dev = sc->sc_dev; 881 msi->msi_priv = its; 882 msi->msi_alloc = gicv3_its_msi_alloc; 883 msi->msix_alloc = gicv3_its_msix_alloc; 884 msi->msi_intr_establish = gicv3_its_msi_intr_establish; 885 msi->msi_intr_release = gicv3_its_msi_intr_release; 886 887 return arm_pci_msi_add(msi); 888 } 889