1 /* $NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $ */
2 /* $OpenBSD: aplpcie.c,v 1.13 2022/04/06 18:59:26 naddy Exp $ */
3
4 /*-
5 * Copyright (c) 2021 Jared McNeill <jmcneill@invisible.ca>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org>
32 *
33 * Permission to use, copy, modify, and distribute this software for any
34 * purpose with or without fee is hereby granted, provided that the above
35 * copyright notice and this permission notice appear in all copies.
36 *
37 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
38 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
39 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
40 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
41 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
42 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
43 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
44 */
45
46
47 #include <sys/cdefs.h>
48 __KERNEL_RCSID(0, "$NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $");
49
50 #include <sys/param.h>
51 #include <sys/device.h>
52 #include <sys/kernel.h>
53 #include <sys/systm.h>
54 #include <sys/bus.h>
55 #include <sys/kmem.h>
56 #include <sys/bitops.h>
57
58 #include <dev/pci/pcireg.h>
59 #include <dev/pci/pcivar.h>
60 #include <dev/pci/pciconf.h>
61
62 #include <dev/fdt/fdtvar.h>
63
64 #include <arm/pci/pci_msi_machdep.h>
65 #include <arm/fdt/pcihost_fdtvar.h>
66
67 #define PCIE_CORE_LANE_CONF(port) (0x84000 + (port) * 0x4000)
68 #define PCIE_CORE_LANE_CONF_REFCLK0REQ __BIT(0)
69 #define PCIE_CORE_LANE_CONF_REFCLK1REQ __BIT(1)
70 #define PCIE_CORE_LANE_CONF_REFCLK0ACK __BIT(2)
71 #define PCIE_CORE_LANE_CONF_REFCLK1ACK __BIT(3)
72 #define PCIE_CORE_LANE_CONF_REFCLK0EN __BIT(9)
73 #define PCIE_CORE_LANE_CONF_REFCLK1EN __BIT(10)
74 #define PCIE_CORE_LANE_CTRL(port) (0x84004 + (port) * 0x4000)
75 #define PCIE_CORE_LANE_CTRL_CFGACC __BIT(15)
76
77 #define PCIE_PORT_LTSSM_CTRL 0x0080
78 #define PCIE_PORT_LTSSM_CTRL_START __BIT(0)
79 #define PCIE_PORT_MSI_CTRL 0x0124
80 #define PCIE_PORT_MSI_CTRL_EN __BIT(0)
81 #define PCIE_PORT_MSI_CTRL_32 __SHIFTIN(5U, __BITS(7,4))
82 #define PCIE_PORT_MSI_REMAP 0x0128
83 #define PCIE_PORT_MSI_DOORBELL 0x0168
84 #define PCIE_PORT_LINK_STAT 0x0208
85 #define PCIE_PORT_LINK_STAT_UP __BIT(0)
86 #define PCIE_PORT_APPCLK 0x0800
87 #define PCIE_PORT_APPCLK_EN __BIT(0)
88 #define PCIE_PORT_APPCLK_CGDIS __BIT(8)
89 #define PCIE_PORT_STAT 0x0804
90 #define PCIE_PORT_STAT_READY __BIT(0)
91 #define PCIE_PORT_REFCLK 0x0810
92 #define PCIE_PORT_REFCLK_EN __BIT(0)
93 #define PCIE_PORT_REFCLK_CGDIS __BIT(8)
94 #define PCIE_PORT_PERST 0x0814
95 #define PCIE_PORT_PERST_DIS __BIT(0)
96
97 extern struct bus_space arm_generic_bs_tag;
98
99 struct apple_pcie_softc {
100 struct pcihost_softc sc_pcihost;
101
102 bus_space_tag_t sc_rc_bst;
103 bus_space_handle_t sc_rc_bsh;
104
105 int sc_phandle;
106 struct arm_pci_msi sc_msi;
107 u_int sc_msi_start;
108 u_int sc_nmsi;
109 struct pci_attach_args **sc_msi_pa;
110 void **sc_msi_ih;
111 uint64_t sc_msi_addr;
112 };
113
114 static int apple_pcie_match(device_t, cfdata_t, void *);
115 static void apple_pcie_attach(device_t, device_t, void *);
116
117 static void apple_pcie_attach_hook(device_t, device_t,
118 struct pcibus_attach_args *);
119 static int apple_pcie_msi_init(struct apple_pcie_softc *);
120
121 CFATTACH_DECL_NEW(apple_pcie, sizeof(struct apple_pcie_softc),
122 apple_pcie_match, apple_pcie_attach, NULL, NULL);
123
124 static const struct device_compatible_entry compat_data[] = {
125 { .compat = "apple,pcie" },
126 DEVICE_COMPAT_EOL
127 };
128
129 #define RREAD4(sc, reg) \
130 (bus_space_read_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg)))
131 #define RWRITE4(sc, reg, val) \
132 bus_space_write_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg), (val))
133 #define RSET4(sc, reg, bits) \
134 RWRITE4((sc), (reg), RREAD4((sc), (reg)) | (bits))
135 #define RCLR4(sc, reg, bits) \
136 RWRITE4((sc), (reg), RREAD4((sc), (reg)) & ~(bits))
137
138
139 static void
apple_pcie_setup_port(struct apple_pcie_softc * sc,int phandle)140 apple_pcie_setup_port(struct apple_pcie_softc *sc, int phandle)
141 {
142 const bus_space_tag_t bst = sc->sc_pcihost.sc_bst;
143 const device_t dev = sc->sc_pcihost.sc_dev;
144 const int parent = sc->sc_pcihost.sc_phandle;
145 char regname[sizeof("portX")];
146 bus_space_handle_t bsh;
147 bus_addr_t addr;
148 bus_size_t size;
149 int error;
150 int timo;
151 int len;
152
153 const u_int *reg = fdtbus_get_prop(phandle, "reg", &len);
154 if (len != 5 * sizeof(uint32_t)) {
155 aprint_error(": couldn't get port number\n");
156 }
157
158 u_int portno = __SHIFTOUT(be32toh(reg[0]), __BITS(13,11));
159 snprintf(regname, sizeof(regname), "port%u", portno);
160
161 if (fdtbus_get_reg_byname(parent, regname, &addr, &size) != 0) {
162 aprint_error(": couldn't get %s regs\n", regname);
163 return;
164 }
165 error = bus_space_map(bst, addr, size, 0, &bsh);
166 if (error != 0) {
167 aprint_error(": couldn't map %s regs\n", regname);
168 return;
169 }
170
171 #define PREAD4(bst, bsh, reg) \
172 bus_space_read_4((bst), (bsh), (reg))
173 #define PWRITE4(bst, bsh, reg, val) \
174 bus_space_write_4((bst), (bsh), (reg), (val))
175 #define PSET4(bst, bsh, reg, bits) \
176 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) | (bits))
177 #define PCLR4(bst, bsh, reg, bits) \
178 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) & ~(bits))
179
180 /* Doorbell address must be below 4GB */
181 KASSERT((sc->sc_msi_addr & ~0xffffffffUL) == 0);
182
183 int pwren_gpiolen, reset_gpiolen;
184
185 pwren_gpiolen = OF_getproplen(phandle, "pwren-gpios");
186 reset_gpiolen = OF_getproplen(phandle, "reset-gpios");
187 if (reset_gpiolen <= 0)
188 return;
189
190 /*
191 * Set things up such that we can share the 32 available MSIs
192 * across all ports.
193 */
194 PWRITE4(bst, bsh, PCIE_PORT_MSI_CTRL,
195 PCIE_PORT_MSI_CTRL_32 | PCIE_PORT_MSI_CTRL_EN);
196 PWRITE4(bst, bsh, PCIE_PORT_MSI_REMAP, 0);
197 PWRITE4(bst, bsh, PCIE_PORT_MSI_DOORBELL,
198 __SHIFTOUT(sc->sc_msi_addr, __BITS(31, 0)));
199
200 /* Check if the link is already up. */
201 uint32_t stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT);
202 if (stat & PCIE_PORT_LINK_STAT_UP) {
203 aprint_debug_dev(dev, "link already up\n");
204 return;
205 }
206 aprint_debug_dev(dev, "bringing link up\n");
207
208 PSET4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_EN);
209
210 struct fdtbus_gpio_pin *gpio_reset = fdtbus_gpio_acquire(phandle,
211 "reset-gpios", GPIO_PIN_OUTPUT);
212
213 if (gpio_reset == NULL) {
214 aprint_debug_dev(dev, "failed to get reset-gpios\n");
215 return;
216 }
217
218 fdtbus_gpio_write(gpio_reset, 1);
219
220 /* Power up the device if necessary. */
221 if (pwren_gpiolen > 0) {
222 struct fdtbus_gpio_pin *gpio_pwren = fdtbus_gpio_acquire(phandle,
223 "pwren-gpios", GPIO_PIN_OUTPUT);
224
225 if (gpio_pwren == NULL) {
226 aprint_debug_dev(dev, "failed to get pwren-gpios\n");
227 return;
228 }
229
230 fdtbus_gpio_write(gpio_pwren, 1);
231 }
232
233 /* Setup Refclk. */
234 RSET4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC);
235 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK0REQ);
236 for (timo = 500; timo > 0; timo--) {
237 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno));
238 if (stat & PCIE_CORE_LANE_CONF_REFCLK0ACK)
239 break;
240 delay(100);
241 }
242 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK1REQ);
243 for (timo = 500; timo > 0; timo--) {
244 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno));
245 if (stat & PCIE_CORE_LANE_CONF_REFCLK1ACK)
246 break;
247 delay(100);
248 }
249 RCLR4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC);
250 RSET4(sc, PCIE_CORE_LANE_CONF(portno),
251 PCIE_CORE_LANE_CONF_REFCLK0EN | PCIE_CORE_LANE_CONF_REFCLK1EN);
252 PSET4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_EN);
253
254 /*
255 * PERST# must remain asserted for at least 100us after the
256 * reference clock becomes stable. But also has to remain
257 * active at least 100ms after power up.
258 */
259 if (pwren_gpiolen > 0)
260 delay(100000);
261 else
262 delay(100);
263
264 /* Deassert PERST#. */
265 PSET4(bst, bsh, PCIE_PORT_PERST, PCIE_PORT_PERST_DIS);
266 fdtbus_gpio_write(gpio_reset, 0);
267
268 for (timo = 2500; timo > 0; timo--) {
269 stat = PREAD4(bst, bsh, PCIE_PORT_STAT);
270 if (stat & PCIE_PORT_STAT_READY)
271 break;
272 delay(100);
273 }
274 if ((stat & PCIE_PORT_STAT_READY) == 0) {
275 aprint_debug_dev(dev, "link up\n");
276 return;
277 }
278
279 PCLR4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_CGDIS);
280 PCLR4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_CGDIS);
281
282 /* Bring up the link. */
283 PWRITE4(bst, bsh, PCIE_PORT_LTSSM_CTRL, PCIE_PORT_LTSSM_CTRL_START);
284 for (timo = 1000; timo > 0; timo--) {
285 stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT);
286 if (stat & PCIE_PORT_LINK_STAT_UP)
287 break;
288 delay(100);
289 }
290
291 #undef PREAD4
292 #undef PWRITE4
293 #undef PCLR4
294 #undef PSET4
295
296 bus_space_unmap(bst, bsh, size);
297 }
298
299 static int
apple_pcie_match(device_t parent,cfdata_t cf,void * aux)300 apple_pcie_match(device_t parent, cfdata_t cf, void *aux)
301 {
302 struct fdt_attach_args * const faa = aux;
303
304 return of_compatible_match(faa->faa_phandle, compat_data);
305 }
306
307 static void
apple_pcie_attach(device_t parent,device_t self,void * aux)308 apple_pcie_attach(device_t parent, device_t self, void *aux)
309 {
310 struct apple_pcie_softc * const asc = device_private(self);
311 struct pcihost_softc * const sc = &asc->sc_pcihost;
312 struct fdt_attach_args * const faa = aux;
313 const int phandle = faa->faa_phandle;
314 bus_addr_t cs_addr, rc_addr;
315 bus_size_t cs_size, rc_size;
316 int error;
317
318 if (fdtbus_get_reg_byname(phandle, "config", &cs_addr, &cs_size) != 0) {
319 aprint_error(": couldn't get registers (%s)\n", "config");
320 return;
321 }
322
323 if (fdtbus_get_reg_byname(phandle, "rc", &rc_addr, &rc_size) != 0) {
324 aprint_error(": couldn't get registers (%s)\n", "rc");
325 return;
326 }
327
328 sc->sc_dev = self;
329 sc->sc_dmat = faa->faa_dmat;
330 sc->sc_bst = asc->sc_rc_bst = faa->faa_bst;
331 /*
332 * Create a new bus tag for PCIe devices that does not inherit the
333 * nonposted MMIO flag from the host controller.
334 */
335 sc->sc_pci_bst = &arm_generic_bs_tag;
336 sc->sc_phandle = phandle;
337 error = bus_space_map(faa->faa_bst, cs_addr, cs_size, 0, &sc->sc_bsh);
338 if (error) {
339 aprint_error(": couldn't map registers (%s): %d\n", "config",
340 error);
341 return;
342 }
343 error = bus_space_map(asc->sc_rc_bst, rc_addr, rc_size, 0,
344 &asc->sc_rc_bsh);
345 if (error) {
346 aprint_error(": couldn't map registers (%s): %d\n", "rc",
347 error);
348 return;
349 }
350 sc->sc_type = PCIHOST_ECAM;
351
352 if (apple_pcie_msi_init(asc) == 0) {
353 sc->sc_pci_flags |= PCI_FLAGS_MSI_OKAY;
354 #if notyet
355 sc->sc_pci_flags |= PCI_FLAGS_MSIX_OKAY;
356 #endif
357 }
358
359 aprint_naive("\n");
360 aprint_normal(": Apple PCIe host controller\n");
361
362 for (int node = OF_child(phandle); node; node = OF_peer(node))
363 apple_pcie_setup_port(asc, node);
364
365 /*
366 * Must wait at least 100ms after link training completes
367 * before sending a configuration request to a device
368 * immediately below a port.
369 */
370 delay(100000);
371
372 pcihost_init(&sc->sc_pc, sc);
373
374 sc->sc_pc.pc_attach_hook = apple_pcie_attach_hook;
375 pcihost_init2(sc);
376 }
377
378
379
380 static void
apple_pcie_attach_hook(device_t parent,device_t self,struct pcibus_attach_args * pba)381 apple_pcie_attach_hook(device_t parent, device_t self,
382 struct pcibus_attach_args *pba)
383 {
384 struct apple_pcie_softc *sc = pba->pba_pc->pc_conf_v;
385 const int phandle = sc->sc_pcihost.sc_phandle;
386 bus_dma_tag_t dmat;
387
388 KASSERT(device_is_a(sc->sc_pcihost.sc_dev, "applepcie"));
389
390 /* XXX this should be per-device, not per-bus */
391 const uint32_t rid = pba->pba_bus << 8;
392
393 dmat = fdtbus_iommu_map_pci(phandle, rid, sc->sc_pcihost.sc_dmat);
394
395 pba->pba_dmat = pba->pba_dmat64 = dmat;
396 }
397
398 static int
apple_pcie_msi_alloc_msi(struct apple_pcie_softc * sc,int count,const struct pci_attach_args * pa)399 apple_pcie_msi_alloc_msi(struct apple_pcie_softc *sc, int count,
400 const struct pci_attach_args *pa)
401 {
402 struct pci_attach_args *new_pa;
403 int msi, n;
404
405 for (msi = 0; msi < sc->sc_nmsi; msi += n) {
406 /* Look for first empty slot */
407 if (sc->sc_msi_pa[msi] != NULL) {
408 /* skip the used entry */
409 n = 1;
410 continue;
411 }
412
413 /* Now check that 'count' entries are also empty */
414 for (n = 1; n < count && msi + n < sc->sc_nmsi; n++) {
415 if (sc->sc_msi_pa[msi + n] != NULL) {
416 break;
417 }
418 }
419 /*
420 * If 'count' empty entries weren't found then the search
421 * continues.
422 */
423 if (n != count)
424 continue;
425 for (n = 0; n < count; n++) {
426 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP);
427 memcpy(new_pa, pa, sizeof(*new_pa));
428 sc->sc_msi_pa[msi + n] = new_pa;
429 }
430
431 return msi;
432 }
433
434 return -1;
435 }
436
437 static void
apple_pcie_msi_free_msi(struct apple_pcie_softc * sc,int msi)438 apple_pcie_msi_free_msi(struct apple_pcie_softc *sc, int msi)
439 {
440 struct pci_attach_args *pa;
441
442 pa = sc->sc_msi_pa[msi];
443 sc->sc_msi_pa[msi] = NULL;
444
445 if (pa != NULL) {
446 kmem_free(pa, sizeof(*pa));
447 }
448 }
449
450 static int
apple_pcie_msi_available_msi(struct apple_pcie_softc * sc)451 apple_pcie_msi_available_msi(struct apple_pcie_softc *sc)
452 {
453 int msi, n;
454
455 for (n = 0, msi = 0; msi < sc->sc_nmsi; msi++) {
456 if (sc->sc_msi_pa[msi] == NULL) {
457 n++;
458 }
459 }
460
461 return n;
462 }
463
464 static void
apple_pcie_msi_msi_enable(struct apple_pcie_softc * sc,int msi,int count)465 apple_pcie_msi_msi_enable(struct apple_pcie_softc *sc, int msi, int count)
466 {
467 const struct pci_attach_args *pa = sc->sc_msi_pa[msi];
468 pci_chipset_tag_t pc = pa->pa_pc;
469 pcitag_t tag = pa->pa_tag;
470 pcireg_t ctl;
471 int off;
472
473 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
474 panic("apple_pcie_msi_msi_enable: device is not MSI-capable");
475
476 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
477 ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
478 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
479
480 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
481 ctl &= ~PCI_MSI_CTL_MME_MASK;
482 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
483 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
484
485 const uint64_t addr = sc->sc_msi_addr;
486 const uint32_t data = msi;
487
488 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
489 if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
490 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO,
491 __SHIFTOUT(addr, __BITS(31, 0)));
492 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI,
493 __SHIFTOUT(addr, __BITS(63, 32)));
494 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
495 } else {
496 pci_conf_write(pc, tag, off + PCI_MSI_MADDR,
497 __SHIFTOUT(addr, __BITS(31, 0)));
498 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
499 }
500 ctl |= PCI_MSI_CTL_MSI_ENABLE;
501 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
502 }
503
504 static void
apple_pcie_msi_msi_disable(struct apple_pcie_softc * sc,int msi)505 apple_pcie_msi_msi_disable(struct apple_pcie_softc *sc, int msi)
506 {
507 const struct pci_attach_args *pa = sc->sc_msi_pa[msi];
508 pci_chipset_tag_t pc = pa->pa_pc;
509 pcitag_t tag = pa->pa_tag;
510 pcireg_t ctl;
511 int off;
512
513 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
514 panic("apple_pcie_msi_msi_disable: device is not MSI-capable");
515
516 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
517 ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
518 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
519 }
520
521 static void
apple_pcie_msi_msix_enable(struct apple_pcie_softc * sc,int msi,int msix_vec,bus_space_tag_t bst,bus_space_handle_t bsh)522 apple_pcie_msi_msix_enable(struct apple_pcie_softc *sc, int msi, int msix_vec,
523 bus_space_tag_t bst, bus_space_handle_t bsh)
524 {
525 const struct pci_attach_args *pa = sc->sc_msi_pa[msi];
526 pci_chipset_tag_t pc = pa->pa_pc;
527 pcitag_t tag = pa->pa_tag;
528 pcireg_t ctl;
529 uint32_t val;
530 int off;
531
532 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
533 panic("apple_pcie_msi_msix_enable: device is not MSI-X-capable");
534
535 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
536 ctl &= ~PCI_MSIX_CTL_ENABLE;
537 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
538
539 const uint64_t addr = sc->sc_msi_addr;
540 const uint32_t data = msi;
541 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
542 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO,
543 __SHIFTOUT(addr, __BITS(31, 0)));
544 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI,
545 __SHIFTOUT(addr, __BITS(63, 32)));
546 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA,
547 data);
548 val = bus_space_read_4(bst, bsh,
549 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
550 val &= ~PCI_MSIX_VECTCTL_MASK;
551 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL,
552 val);
553
554 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
555 ctl |= PCI_MSIX_CTL_ENABLE;
556 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
557 }
558
559 static void
apple_pcie_msi_msix_disable(struct apple_pcie_softc * sc,int msi)560 apple_pcie_msi_msix_disable(struct apple_pcie_softc *sc, int msi)
561 {
562 const struct pci_attach_args *pa = sc->sc_msi_pa[msi];
563 pci_chipset_tag_t pc = pa->pa_pc;
564 pcitag_t tag = pa->pa_tag;
565 pcireg_t ctl;
566 int off;
567
568 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
569 panic("apple_pcie_msi_msix_disable: device is not MSI-X-capable");
570
571 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
572 ctl &= ~PCI_MSIX_CTL_ENABLE;
573 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
574 }
575
576 static pci_intr_handle_t *
apple_pcie_msi_msi_alloc(struct arm_pci_msi * msi,int * count,const struct pci_attach_args * pa,bool exact)577 apple_pcie_msi_msi_alloc(struct arm_pci_msi *msi, int *count,
578 const struct pci_attach_args *pa, bool exact)
579 {
580 struct apple_pcie_softc * const sc = msi->msi_priv;
581 pci_intr_handle_t *vectors;
582 int n, off;
583
584 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL))
585 return NULL;
586
587 const int avail = apple_pcie_msi_available_msi(sc);
588 if (avail == 0)
589 return NULL;
590
591 if (exact && *count > avail)
592 return NULL;
593
594 while (*count > avail) {
595 (*count) >>= 1;
596 }
597 if (*count == 0)
598 return NULL;
599
600 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa);
601 if (msi_base == -1)
602 return NULL;
603
604 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
605 for (n = 0; n < *count; n++) {
606 const int msino = msi_base + n;
607 vectors[n] = ARM_PCI_INTR_MSI |
608 __SHIFTIN(msino, ARM_PCI_INTR_IRQ) |
609 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) |
610 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
611 }
612
613 apple_pcie_msi_msi_enable(sc, msi_base, *count);
614
615 return vectors;
616 }
617
618 static pci_intr_handle_t *
apple_pcie_msi_msix_alloc(struct arm_pci_msi * msi,u_int * table_indexes,int * count,const struct pci_attach_args * pa,bool exact)619 apple_pcie_msi_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes,
620 int *count, const struct pci_attach_args *pa, bool exact)
621 {
622 struct apple_pcie_softc * const sc = msi->msi_priv;
623 pci_intr_handle_t *vectors;
624 bus_space_tag_t bst;
625 bus_space_handle_t bsh;
626 bus_size_t bsz;
627 uint32_t table_offset, table_size;
628 int n, off, bar, error;
629 pcireg_t tbl;
630
631 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL))
632 return NULL;
633
634 const int avail = apple_pcie_msi_available_msi(sc);
635 if (exact && *count > avail)
636 return NULL;
637
638 while (*count > avail) {
639 (*count) >>= 1;
640 }
641 if (*count == 0)
642 return NULL;
643
644 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET);
645 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK));
646 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
647 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE;
648 if (table_size == 0)
649 return NULL;
650
651 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar),
652 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset,
653 &bst, &bsh, NULL, &bsz);
654 if (error)
655 return NULL;
656
657 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa);
658 if (msi_base == -1) {
659 bus_space_unmap(bst, bsh, bsz);
660 return NULL;
661 }
662
663 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
664 for (n = 0; n < *count; n++) {
665 const int msino = msi_base + n;
666 const int msix_vec = table_indexes ? table_indexes[n] : n;
667 vectors[msix_vec] = ARM_PCI_INTR_MSIX |
668 __SHIFTIN(msino, ARM_PCI_INTR_IRQ) |
669 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) |
670 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
671
672 apple_pcie_msi_msix_enable(sc, msino, msix_vec, bst, bsh);
673 }
674
675 bus_space_unmap(bst, bsh, bsz);
676
677 return vectors;
678 }
679
680 static void *
apple_pcie_msi_intr_establish(struct arm_pci_msi * msi,pci_intr_handle_t ih,int ipl,int (* func)(void *),void * arg,const char * xname)681 apple_pcie_msi_intr_establish(struct arm_pci_msi *msi,
682 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname)
683 {
684 struct apple_pcie_softc * const sc = msi->msi_priv;
685
686 const int msino = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ);
687 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? FDT_INTR_MPSAFE : 0;
688
689 KASSERT(sc->sc_msi_ih[msino] == NULL);
690 sc->sc_msi_ih[msino] = intr_establish_xname(sc->sc_msi_start + msino,
691 ipl, IST_LEVEL | (mpsafe ? IST_MPSAFE : 0), func, arg, xname);
692
693 return sc->sc_msi_ih[msino];
694 }
695
696 static void
apple_pcie_msi_intr_release(struct arm_pci_msi * msi,pci_intr_handle_t * pih,int count)697 apple_pcie_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih,
698 int count)
699 {
700 struct apple_pcie_softc * const sc = msi->msi_priv;
701 int n;
702
703 for (n = 0; n < count; n++) {
704 const int msino = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ);
705 if (pih[n] & ARM_PCI_INTR_MSIX)
706 apple_pcie_msi_msix_disable(sc, msino);
707 if (pih[n] & ARM_PCI_INTR_MSI)
708 apple_pcie_msi_msi_disable(sc, msino);
709 apple_pcie_msi_free_msi(sc, msino);
710 if (sc->sc_msi_ih[msino] != NULL) {
711 intr_disestablish(sc->sc_msi_ih[msino]);
712 sc->sc_msi_ih[msino] = NULL;
713 }
714 }
715 }
716
717 static int
apple_pcie_msi_init(struct apple_pcie_softc * sc)718 apple_pcie_msi_init(struct apple_pcie_softc *sc)
719 {
720 struct arm_pci_msi *msi = &sc->sc_msi;
721 const int phandle = sc->sc_pcihost.sc_phandle;
722 int len;
723
724 const u_int *data = fdtbus_get_prop(phandle, "msi-ranges", &len);
725 switch (len) {
726 case 8:
727 /* two cells: start and count */
728 sc->sc_msi_start = be32toh(data[0]);
729 sc->sc_nmsi = be32toh(data[1]);
730 break;
731 case 20:
732 /* 5 cells: xref, specifier (3 cells), and count */
733 sc->sc_msi_start = be32toh(data[2]);
734 sc->sc_nmsi = be32toh(data[4]);
735 break;
736 default:
737 aprint_error_dev(sc->sc_pcihost.sc_dev,
738 "WARNING: bad msi-ranges property, MSI not enabled!\n");
739 return ENXIO;
740 }
741 sc->sc_msi_pa = kmem_zalloc(sizeof(*sc->sc_msi_pa) * sc->sc_nmsi,
742 KM_SLEEP);
743 sc->sc_msi_ih = kmem_zalloc(sizeof(*sc->sc_msi_ih) * sc->sc_nmsi,
744 KM_SLEEP);
745
746 if (of_getprop_uint64(phandle, "msi-doorbell", &sc->sc_msi_addr)) {
747 sc->sc_msi_addr = 0xffff000ULL;
748 }
749
750 msi->msi_dev = sc->sc_pcihost.sc_dev;
751 msi->msi_priv = sc;
752 msi->msi_alloc = apple_pcie_msi_msi_alloc;
753 msi->msix_alloc = apple_pcie_msi_msix_alloc;
754 msi->msi_intr_establish = apple_pcie_msi_intr_establish;
755 msi->msi_intr_release = apple_pcie_msi_intr_release;
756
757 return arm_pci_msi_add(msi);
758 }
759