1 /* $NetBSD: gicv3_its.c,v 1.35 2023/11/11 17:35:45 tnn Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jared McNeill <jmcneill@invisible.ca>. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _INTR_PRIVATE 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.35 2023/11/11 17:35:45 tnn Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/kmem.h> 39 #include <sys/bus.h> 40 #include <sys/cpu.h> 41 #include <sys/bitops.h> 42 43 #include <uvm/uvm.h> 44 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 48 #include <arm/pic/picvar.h> 49 #include <arm/cortex/gicv3_its.h> 50 51 /* 52 * ITS translation table sizes 53 */ 54 #define GITS_COMMANDS_SIZE 0x1000 55 #define GITS_COMMANDS_ALIGN 0x10000 56 57 #define GITS_ITT_ALIGN 0x100 58 59 /* 60 * IIDR values used for errata 61 */ 62 #define GITS_IIDR_PID_CAVIUM_THUNDERX 0xa1 63 #define GITS_IIDR_IMP_CAVIUM 0x34c 64 #define GITS_IIDR_CAVIUM_ERRATA_MASK (GITS_IIDR_Implementor|GITS_IIDR_ProductID|GITS_IIDR_Variant) 65 #define GITS_IIDR_CAVIUM_ERRATA_VALUE \ 66 (__SHIFTIN(GITS_IIDR_IMP_CAVIUM, GITS_IIDR_Implementor) | \ 67 __SHIFTIN(GITS_IIDR_PID_CAVIUM_THUNDERX, GITS_IIDR_ProductID) | \ 68 __SHIFTIN(0, GITS_IIDR_Variant)) 69 70 static const char * gits_cache_type[] = { 71 [GITS_Cache_DEVICE_nGnRnE] = "Device-nGnRnE", 72 [GITS_Cache_NORMAL_NC] = "Non-cacheable", 73 [GITS_Cache_NORMAL_RA_WT] = "Cacheable RA WT", 74 [GITS_Cache_NORMAL_RA_WB] = "Cacheable RA WB", 75 [GITS_Cache_NORMAL_WA_WT] = "Cacheable WA WT", 76 [GITS_Cache_NORMAL_WA_WB] = "Cacheable WA WB", 77 [GITS_Cache_NORMAL_RA_WA_WT] = "Cacheable RA WA WT", 78 [GITS_Cache_NORMAL_RA_WA_WB] = "Cacheable RA WA WB", 79 }; 80 81 static const char * gits_share_type[] = { 82 [GITS_Shareability_NS] = "Non-shareable", 83 [GITS_Shareability_IS] = "Inner shareable", 84 [GITS_Shareability_OS] = "Outer shareable", 85 [3] = "(Reserved)", 86 }; 87 88 static inline uint32_t 89 gits_read_4(struct gicv3_its *its, bus_size_t reg) 90 { 91 return bus_space_read_4(its->its_bst, its->its_bsh, reg); 92 } 93 94 static inline void 95 gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val) 96 { 97 bus_space_write_4(its->its_bst, its->its_bsh, reg, val); 98 } 99 100 static inline uint64_t 101 gits_read_8(struct gicv3_its *its, bus_size_t reg) 102 { 103 return bus_space_read_8(its->its_bst, its->its_bsh, reg); 104 } 105 106 static inline void 107 gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val) 108 { 109 bus_space_write_8(its->its_bst, its->its_bsh, reg, val); 110 } 111 112 static inline void 113 gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd) 114 { 115 uint64_t cwriter; 116 u_int woff; 117 118 cwriter = gits_read_8(its, GITS_CWRITER); 119 woff = cwriter & GITS_CWRITER_Offset; 120 #ifdef DIAGNOSTIC 121 uint64_t creadr = gits_read_8(its, GITS_CREADR); 122 KASSERT(!ISSET(creadr, GITS_CREADR_Stalled)); 123 KASSERT(((woff + sizeof(cmd->dw)) & (its->its_cmd.len - 1)) != (creadr & GITS_CREADR_Offset)); 124 #endif 125 126 uint64_t *dw = (uint64_t *)(its->its_cmd.base + woff); 127 for (int i = 0; i < __arraycount(cmd->dw); i++) 128 dw[i] = htole64(cmd->dw[i]); 129 bus_dmamap_sync(its->its_dmat, its->its_cmd.map, woff, sizeof(cmd->dw), BUS_DMASYNC_PREWRITE); 130 131 woff += sizeof(cmd->dw); 132 if (woff == its->its_cmd.len) 133 woff = 0; 134 135 gits_write_8(its, GITS_CWRITER, woff); 136 } 137 138 static inline void 139 gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v) 140 { 141 struct gicv3_its_command cmd; 142 143 KASSERT((rdbase & 0xffff) == 0); 144 145 /* 146 * Map a collection table entry (ICID) to the target redistributor (RDbase). 147 */ 148 memset(&cmd, 0, sizeof(cmd)); 149 cmd.dw[0] = GITS_CMD_MAPC; 150 cmd.dw[2] = icid; 151 if (v) { 152 cmd.dw[2] |= rdbase; 153 cmd.dw[2] |= __BIT(63); 154 } 155 156 gits_command(its, &cmd); 157 } 158 159 static inline void 160 gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v) 161 { 162 struct gicv3_its_command cmd; 163 164 KASSERT((itt_addr & 0xff) == 0); 165 166 /* 167 * Map a device table entry (DeviceID) to its associated ITT (ITT_addr). 168 */ 169 memset(&cmd, 0, sizeof(cmd)); 170 cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32); 171 cmd.dw[1] = size; 172 if (v) { 173 cmd.dw[2] = itt_addr | __BIT(63); 174 } 175 176 gits_command(its, &cmd); 177 } 178 179 static inline void 180 gits_command_mapti(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint32_t pintid, uint16_t icid) 181 { 182 struct gicv3_its_command cmd; 183 184 /* 185 * Map the event defined by EventID and DeviceID to its associated ITE, defined by ICID and pINTID 186 * in the ITT associated with DeviceID. 187 */ 188 memset(&cmd, 0, sizeof(cmd)); 189 cmd.dw[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32); 190 cmd.dw[1] = eventid | ((uint64_t)pintid << 32); 191 cmd.dw[2] = icid; 192 193 gits_command(its, &cmd); 194 } 195 196 static inline void 197 gits_command_movi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid) 198 { 199 struct gicv3_its_command cmd; 200 201 /* 202 * Update the ICID field in the ITT entry for the event defined by DeviceID and 203 * EventID. 204 */ 205 memset(&cmd, 0, sizeof(cmd)); 206 cmd.dw[0] = GITS_CMD_MOVI | ((uint64_t)deviceid << 32); 207 cmd.dw[1] = eventid; 208 cmd.dw[2] = icid; 209 210 gits_command(its, &cmd); 211 } 212 213 static inline void 214 gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) 215 { 216 struct gicv3_its_command cmd; 217 218 /* 219 * Ensure any caching in the redistributors associated with the specified 220 * EventID is consistent with the LPI configuration tables. 221 */ 222 memset(&cmd, 0, sizeof(cmd)); 223 cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32); 224 cmd.dw[1] = eventid; 225 226 gits_command(its, &cmd); 227 } 228 229 static inline void 230 gits_command_invall(struct gicv3_its *its, uint16_t icid) 231 { 232 struct gicv3_its_command cmd; 233 234 /* 235 * Ensure any caching associated with this ICID is consistent with LPI 236 * configuration tables for all redistributors. 237 */ 238 memset(&cmd, 0, sizeof(cmd)); 239 cmd.dw[0] = GITS_CMD_INVALL; 240 cmd.dw[2] = icid; 241 242 gits_command(its, &cmd); 243 } 244 245 static inline void 246 gits_command_sync(struct gicv3_its *its, uint64_t rdbase) 247 { 248 struct gicv3_its_command cmd; 249 250 KASSERT((rdbase & 0xffff) == 0); 251 252 /* 253 * Ensure all outstanding ITS operations associated with physical interrupts 254 * for the specified redistributor (RDbase) are globally observed before 255 * further ITS commands are executed. 256 */ 257 memset(&cmd, 0, sizeof(cmd)); 258 cmd.dw[0] = GITS_CMD_SYNC; 259 cmd.dw[2] = rdbase; 260 261 gits_command(its, &cmd); 262 } 263 264 #if 0 265 static inline void 266 gits_command_int(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) 267 { 268 struct gicv3_its_command cmd; 269 270 /* 271 * Translate the deviceid and eventid into an icid and pintid through 272 * the device table and ITT. Mark the pintid as pending 273 * on the redistributor. 274 * If the interrupt is not configured the command queue stalls. 275 */ 276 memset(&cmd, 0, sizeof(cmd)); 277 cmd.dw[0] = GITS_CMD_INT | ((uint64_t)deviceid << 32); 278 cmd.dw[1] = eventid; 279 280 gits_command(its, &cmd); 281 } 282 #endif 283 284 static inline int 285 gits_wait(struct gicv3_its *its) 286 { 287 u_int woff, roff; 288 int retry = 100000; 289 290 /* 291 * The ITS command queue is empty when CWRITER and CREADR specify the 292 * same base address offset value. 293 */ 294 for (retry = 1000; retry > 0; retry--) { 295 woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset; 296 roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset; 297 if (woff == roff) 298 break; 299 delay(100); 300 } 301 if (retry == 0) { 302 device_printf(its->its_gic->sc_dev, "ITS command queue timeout\n"); 303 return ETIMEDOUT; 304 } 305 306 return 0; 307 } 308 309 static int 310 gicv3_its_msi_alloc_lpi(struct gicv3_its *its, 311 const struct pci_attach_args *pa) 312 { 313 struct pci_attach_args *new_pa; 314 vmem_addr_t n; 315 316 KASSERT(its->its_gic->sc_lpi_pool != NULL); 317 318 if (vmem_alloc(its->its_gic->sc_lpi_pool, 1, VM_INSTANTFIT|VM_SLEEP, &n) != 0) 319 return -1; 320 321 KASSERT(its->its_pa[n] == NULL); 322 323 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 324 memcpy(new_pa, pa, sizeof(*new_pa)); 325 its->its_pa[n] = new_pa; 326 return n + its->its_pic->pic_irqbase; 327 } 328 329 static void 330 gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi) 331 { 332 struct pci_attach_args *pa; 333 334 KASSERT(its->its_gic->sc_lpi_pool != NULL); 335 KASSERT(lpi >= its->its_pic->pic_irqbase); 336 337 pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 338 its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL; 339 kmem_free(pa, sizeof(*pa)); 340 341 vmem_free(its->its_gic->sc_lpi_pool, lpi - its->its_pic->pic_irqbase, 1); 342 } 343 344 static uint32_t 345 gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag) 346 { 347 uint32_t devid; 348 int b, d, f; 349 350 pci_decompose_tag(pc, tag, &b, &d, &f); 351 352 devid = (b << 8) | (d << 3) | f; 353 354 return pci_get_devid(pc, devid); 355 } 356 357 static int 358 gicv3_its_device_map(struct gicv3_its *its, uint32_t devid, u_int count) 359 { 360 struct gicv3_its_device *dev; 361 u_int vectors; 362 363 vectors = MAX(2, count); 364 while (!powerof2(vectors)) 365 vectors++; 366 367 const uint64_t typer = gits_read_8(its, GITS_TYPER); 368 const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1; 369 const u_int itt_size = roundup(vectors * itt_entry_size, GITS_ITT_ALIGN); 370 371 LIST_FOREACH(dev, &its->its_devices, dev_list) 372 if (dev->dev_id == devid) { 373 return itt_size <= dev->dev_size ? 0 : EEXIST; 374 } 375 376 dev = kmem_alloc(sizeof(*dev), KM_SLEEP); 377 dev->dev_id = devid; 378 dev->dev_size = itt_size; 379 gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN); 380 LIST_INSERT_HEAD(&its->its_devices, dev, dev_list); 381 382 /* 383 * Map the device to the ITT 384 */ 385 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 386 mutex_enter(its->its_lock); 387 gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true); 388 gits_wait(its); 389 mutex_exit(its->its_lock); 390 391 return 0; 392 } 393 394 static void 395 gicv3_its_msi_enable(struct gicv3_its *its, int lpi, int count) 396 { 397 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 398 pci_chipset_tag_t pc = pa->pa_pc; 399 pcitag_t tag = pa->pa_tag; 400 pcireg_t ctl; 401 int off; 402 403 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 404 panic("gicv3_its_msi_enable: device is not MSI-capable"); 405 406 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 407 ctl &= ~PCI_MSI_CTL_MME_MASK; 408 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 409 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 410 411 const uint64_t addr = its->its_base + GITS_TRANSLATER; 412 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 413 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 414 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 415 addr & 0xffffffff); 416 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 417 (addr >> 32) & 0xffffffff); 418 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, 419 lpi - its->its_pic->pic_irqbase); 420 } else { 421 KASSERT((addr >> 32) == 0); 422 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 423 addr & 0xffffffff); 424 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, 425 lpi - its->its_pic->pic_irqbase); 426 } 427 ctl |= PCI_MSI_CTL_MSI_ENABLE; 428 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 429 } 430 431 static void 432 gicv3_its_msi_disable(struct gicv3_its *its, int lpi) 433 { 434 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 435 pci_chipset_tag_t pc = pa->pa_pc; 436 pcitag_t tag = pa->pa_tag; 437 pcireg_t ctl; 438 int off; 439 440 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 441 panic("gicv3_its_msi_enable: device is not MSI-capable"); 442 443 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 444 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 445 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 446 } 447 448 static void 449 gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec, 450 bus_space_tag_t bst, bus_space_handle_t bsh) 451 { 452 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 453 pci_chipset_tag_t pc = pa->pa_pc; 454 pcitag_t tag = pa->pa_tag; 455 pcireg_t ctl; 456 uint32_t val; 457 int off; 458 459 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 460 panic("gicv3_its_msix_enable: device is not MSI-X-capable"); 461 462 const uint64_t addr = its->its_base + GITS_TRANSLATER; 463 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 464 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr); 465 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32)); 466 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, lpi - its->its_pic->pic_irqbase); 467 val = bus_space_read_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); 468 val &= ~PCI_MSIX_VECTCTL_MASK; 469 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, val); 470 471 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 472 ctl |= PCI_MSIX_CTL_ENABLE; 473 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 474 } 475 476 static void 477 gicv3_its_msix_disable(struct gicv3_its *its, int lpi) 478 { 479 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 480 pci_chipset_tag_t pc = pa->pa_pc; 481 pcitag_t tag = pa->pa_tag; 482 pcireg_t ctl; 483 int off; 484 485 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 486 panic("gicv3_its_msix_disable: device is not MSI-X-capable"); 487 488 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 489 ctl &= ~PCI_MSIX_CTL_ENABLE; 490 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 491 } 492 493 static pci_intr_handle_t * 494 gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count, 495 const struct pci_attach_args *pa, bool exact) 496 { 497 struct gicv3_its * const its = msi->msi_priv; 498 struct cpu_info * const ci = cpu_lookup(0); 499 pci_intr_handle_t *vectors; 500 int n, off; 501 502 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 503 return NULL; 504 505 const uint64_t typer = gits_read_8(its, GITS_TYPER); 506 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 507 if (*count == 0 || *count > (1 << id_bits)) 508 return NULL; 509 510 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 511 512 if (gicv3_its_device_map(its, devid, *count) != 0) 513 return NULL; 514 515 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 516 mutex_enter(its->its_lock); 517 for (n = 0; n < *count; n++) { 518 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 519 KASSERT(lpi >= 0); 520 vectors[n] = ARM_PCI_INTR_MSI | 521 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | 522 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | 523 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 524 525 if (n == 0) 526 gicv3_its_msi_enable(its, lpi, *count); 527 528 /* 529 * Record devid and target PE 530 */ 531 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; 532 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 533 534 /* 535 * Map event 536 */ 537 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci)); 538 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 539 } 540 gits_wait(its); 541 mutex_exit(its->its_lock); 542 543 return vectors; 544 } 545 546 static pci_intr_handle_t * 547 gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, 548 const struct pci_attach_args *pa, bool exact) 549 { 550 struct gicv3_its * const its = msi->msi_priv; 551 struct cpu_info *ci = cpu_lookup(0); 552 pci_intr_handle_t *vectors; 553 bus_space_tag_t bst; 554 bus_space_handle_t bsh; 555 bus_size_t bsz; 556 uint32_t table_offset, table_size; 557 int n, off, bar, error; 558 pcireg_t tbl; 559 560 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 561 return NULL; 562 563 const uint64_t typer = gits_read_8(its, GITS_TYPER); 564 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 565 if (*count == 0 || *count > (1 << id_bits)) 566 return NULL; 567 568 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 569 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 570 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 571 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 572 if (table_size == 0) 573 return NULL; 574 575 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 576 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 577 &bst, &bsh, NULL, &bsz); 578 if (error) 579 return NULL; 580 581 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 582 583 if (gicv3_its_device_map(its, devid, *count) != 0) { 584 bus_space_unmap(bst, bsh, bsz); 585 return NULL; 586 } 587 588 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 589 mutex_enter(its->its_lock); 590 for (n = 0; n < *count; n++) { 591 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 592 KASSERT(lpi >= 0); 593 const int msix_vec = table_indexes ? table_indexes[n] : n; 594 vectors[msix_vec] = ARM_PCI_INTR_MSIX | 595 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | 596 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | 597 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 598 599 gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh); 600 601 /* 602 * Record devid and target PE 603 */ 604 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; 605 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 606 607 /* 608 * Map event 609 */ 610 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci)); 611 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 612 } 613 gits_wait(its); 614 mutex_exit(its->its_lock); 615 616 bus_space_unmap(bst, bsh, bsz); 617 618 return vectors; 619 } 620 621 static void * 622 gicv3_its_msi_intr_establish(struct arm_pci_msi *msi, 623 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 624 { 625 struct gicv3_its * const its = msi->msi_priv; 626 void *intrh; 627 628 const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); 629 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0; 630 631 intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl, 632 IST_EDGE | mpsafe, func, arg, xname); 633 if (intrh == NULL) 634 return NULL; 635 636 /* Invalidate LPI configuration tables */ 637 KASSERT(its->its_pa[lpi - its->its_pic->pic_irqbase] != NULL); 638 const uint32_t devid = its->its_devid[lpi - its->its_pic->pic_irqbase]; 639 gits_command_inv(its, devid, lpi - its->its_pic->pic_irqbase); 640 641 return intrh; 642 } 643 644 static void 645 gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, 646 int count) 647 { 648 struct gicv3_its * const its = msi->msi_priv; 649 int n; 650 651 for (n = 0; n < count; n++) { 652 const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); 653 KASSERT(lpi >= its->its_pic->pic_irqbase); 654 if (pih[n] & ARM_PCI_INTR_MSIX) 655 gicv3_its_msix_disable(its, lpi); 656 if (pih[n] & ARM_PCI_INTR_MSI) 657 gicv3_its_msi_disable(its, lpi); 658 gicv3_its_msi_free_lpi(its, lpi); 659 its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL; 660 its->its_devid[lpi - its->its_pic->pic_irqbase] = 0; 661 struct intrsource * const is = 662 its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase]; 663 if (is != NULL) 664 pic_disestablish_source(is); 665 } 666 } 667 668 static void 669 gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its) 670 { 671 uint64_t cbaser; 672 673 gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN); 674 675 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Enabled) == 0); 676 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Quiescent) != 0); 677 678 cbaser = its->its_cmd.segs[0].ds_addr; 679 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache); 680 cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability); 681 cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size); 682 cbaser |= GITS_CBASER_Valid; 683 684 gits_write_8(its, GITS_CWRITER, 0); 685 gits_write_8(its, GITS_CBASER, cbaser); 686 } 687 688 static void 689 gicv3_its_table_params(struct gicv3_softc *sc, struct gicv3_its *its, 690 u_int *devbits, u_int *innercache, u_int *share) 691 { 692 693 const uint64_t typer = gits_read_8(its, GITS_TYPER); 694 const uint32_t iidr = gits_read_4(its, GITS_IIDR); 695 696 /* Default values */ 697 *devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1; 698 *innercache = GITS_Cache_NORMAL_WA_WB; 699 *share = GITS_Shareability_IS; 700 701 /* Cavium ThunderX errata */ 702 if ((iidr & GITS_IIDR_CAVIUM_ERRATA_MASK) == GITS_IIDR_CAVIUM_ERRATA_VALUE) { 703 *devbits = 20; /* 8Mb */ 704 *innercache = GITS_Cache_DEVICE_nGnRnE; 705 aprint_normal_dev(sc->sc_dev, "Cavium ThunderX errata detected\n"); 706 } 707 } 708 709 static void 710 gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its) 711 { 712 u_int table_size, page_size, table_align; 713 u_int devbits, innercache, share; 714 const char *table_type; 715 uint64_t baser; 716 int tab; 717 718 gicv3_its_table_params(sc, its, &devbits, &innercache, &share); 719 720 for (tab = 0; tab < 8; tab++) { 721 baser = gits_read_8(its, GITS_BASERn(tab)); 722 723 const u_int entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1; 724 725 switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) { 726 case GITS_Page_Size_4KB: 727 page_size = 4096; 728 table_align = 4096; 729 break; 730 case GITS_Page_Size_16KB: 731 page_size = 16384; 732 table_align = 16384; 733 break; 734 case GITS_Page_Size_64KB: 735 default: 736 page_size = 65536; 737 table_align = 65536; 738 break; 739 } 740 741 switch (__SHIFTOUT(baser, GITS_BASER_Type)) { 742 case GITS_Type_Devices: 743 /* 744 * Table size scales with the width of the DeviceID. 745 */ 746 table_size = roundup(entry_size * (1 << devbits), page_size); 747 table_type = "Devices"; 748 break; 749 case GITS_Type_InterruptCollections: 750 /* 751 * Allocate space for one interrupt collection per CPU. 752 */ 753 table_size = roundup(entry_size * ncpu, page_size); 754 table_type = "Collections"; 755 break; 756 default: 757 table_size = 0; 758 break; 759 } 760 761 if (table_size == 0) 762 continue; 763 764 gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align); 765 766 baser &= ~GITS_BASER_Size; 767 baser |= __SHIFTIN(table_size / page_size - 1, GITS_BASER_Size); 768 baser &= ~GITS_BASER_Physical_Address; 769 baser |= its->its_tab[tab].segs[0].ds_addr; 770 baser &= ~GITS_BASER_InnerCache; 771 baser |= __SHIFTIN(innercache, GITS_BASER_InnerCache); 772 baser &= ~GITS_BASER_Shareability; 773 baser |= __SHIFTIN(share, GITS_BASER_Shareability); 774 baser |= GITS_BASER_Valid; 775 776 gits_write_8(its, GITS_BASERn(tab), baser); 777 778 baser = gits_read_8(its, GITS_BASERn(tab)); 779 if (__SHIFTOUT(baser, GITS_BASER_Shareability) == GITS_Shareability_NS) { 780 baser &= ~GITS_BASER_InnerCache; 781 baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache); 782 783 gits_write_8(its, GITS_BASERn(tab), baser); 784 } 785 786 baser = gits_read_8(its, GITS_BASERn(tab)); 787 aprint_normal_dev(sc->sc_dev, "ITS [#%d] %s table @ %#lx/%#x, %s, %s\n", 788 tab, table_type, its->its_tab[tab].segs[0].ds_addr, table_size, 789 gits_cache_type[__SHIFTOUT(baser, GITS_BASER_InnerCache)], 790 gits_share_type[__SHIFTOUT(baser, GITS_BASER_Shareability)]); 791 } 792 } 793 794 static void 795 gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its) 796 { 797 uint32_t ctlr; 798 799 ctlr = gits_read_4(its, GITS_CTLR); 800 ctlr |= GITS_CTLR_Enabled; 801 gits_write_4(its, GITS_CTLR, ctlr); 802 } 803 804 static void 805 gicv3_its_cpu_init(void *priv, struct cpu_info *ci) 806 { 807 struct gicv3_its * const its = priv; 808 struct gicv3_softc * const sc = its->its_gic; 809 uint64_t rdbase; 810 size_t irq; 811 812 const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER); 813 if (typer & GITS_TYPER_PTA) { 814 void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]); 815 rdbase = vtophys((vaddr_t)va); 816 } else { 817 rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16; 818 } 819 its->its_rdbase[cpu_index(ci)] = rdbase; 820 821 /* 822 * Map collection ID of this CPU's index to this CPU's redistributor. 823 */ 824 mutex_enter(its->its_lock); 825 gits_command_mapc(its, cpu_index(ci), rdbase, true); 826 gits_command_invall(its, cpu_index(ci)); 827 gits_wait(its); 828 829 /* 830 * Update routing for LPIs targetting this CPU 831 */ 832 for (irq = 0; irq < its->its_pic->pic_maxsources; irq++) { 833 if (its->its_targets[irq] != ci) 834 continue; 835 KASSERT(its->its_pa[irq] != NULL); 836 837 const uint32_t devid = its->its_devid[irq]; 838 gits_command_movi(its, devid, irq, cpu_index(ci)); 839 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 840 } 841 gits_wait(its); 842 mutex_exit(its->its_lock); 843 844 its->its_cpuonline[cpu_index(ci)] = true; 845 } 846 847 static void 848 gicv3_its_get_affinity(void *priv, size_t irq, kcpuset_t *affinity) 849 { 850 struct gicv3_its * const its = priv; 851 struct cpu_info *ci; 852 853 ci = its->its_targets[irq]; 854 if (ci) 855 kcpuset_set(affinity, cpu_index(ci)); 856 } 857 858 static int 859 gicv3_its_set_affinity(void *priv, size_t irq, const kcpuset_t *affinity) 860 { 861 struct gicv3_its * const its = priv; 862 const struct pci_attach_args *pa; 863 struct cpu_info *ci; 864 865 const int set = kcpuset_countset(affinity); 866 if (set != 1) 867 return EINVAL; 868 869 pa = its->its_pa[irq]; 870 if (pa == NULL) 871 return EPASSTHROUGH; 872 873 ci = cpu_lookup(kcpuset_ffs(affinity) - 1); 874 its->its_targets[irq] = ci; 875 876 if (its->its_cpuonline[cpu_index(ci)] == true) { 877 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 878 mutex_enter(its->its_lock); 879 gits_command_movi(its, devid, irq, cpu_index(ci)); 880 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 881 mutex_exit(its->its_lock); 882 } 883 884 return 0; 885 } 886 887 int 888 gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, 889 uint64_t its_base, uint32_t its_id) 890 { 891 struct gicv3_its *its; 892 struct arm_pci_msi *msi; 893 894 const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER); 895 if ((typer & GITS_TYPER_Physical) == 0) 896 return ENXIO; 897 898 its = kmem_zalloc(sizeof(*its), KM_SLEEP); 899 its->its_id = its_id; 900 its->its_bst = sc->sc_bst; 901 its->its_bsh = bsh; 902 its->its_dmat = sc->sc_dmat; 903 its->its_base = its_base; 904 its->its_pic = &sc->sc_lpi; 905 snprintf(its->its_pic->pic_name, sizeof(its->its_pic->pic_name), "gicv3-its"); 906 KASSERT(its->its_pic->pic_maxsources > 0); 907 its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP); 908 its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP); 909 its->its_devid = kmem_zalloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP); 910 its->its_gic = sc; 911 its->its_rdbase = kmem_zalloc(sizeof(*its->its_rdbase) * ncpu, KM_SLEEP); 912 its->its_cpuonline = kmem_zalloc(sizeof(*its->its_cpuonline) * ncpu, KM_SLEEP); 913 its->its_cb.cpu_init = gicv3_its_cpu_init; 914 its->its_cb.get_affinity = gicv3_its_get_affinity; 915 its->its_cb.set_affinity = gicv3_its_set_affinity; 916 its->its_cb.priv = its; 917 LIST_INIT(&its->its_devices); 918 LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list); 919 its->its_lock = mutex_obj_alloc(MUTEX_SPIN, IPL_NONE); 920 921 gicv3_its_command_init(sc, its); 922 gicv3_its_table_init(sc, its); 923 924 gicv3_its_enable(sc, its); 925 926 gicv3_its_cpu_init(its, curcpu()); 927 928 msi = &its->its_msi; 929 msi->msi_id = its_id; 930 msi->msi_dev = sc->sc_dev; 931 msi->msi_priv = its; 932 msi->msi_alloc = gicv3_its_msi_alloc; 933 msi->msix_alloc = gicv3_its_msix_alloc; 934 msi->msi_intr_establish = gicv3_its_msi_intr_establish; 935 msi->msi_intr_release = gicv3_its_msi_intr_release; 936 937 return arm_pci_msi_add(msi); 938 } 939