xref: /openbsd-src/sys/arch/powerpc64/dev/xive.c (revision 471aeecfc619bc9b69519928152daf993376c2a1)

/*	$OpenBSD: xive.c,v 1.17 2022/04/06 18:59:27 naddy Exp $	*/
/*
 * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/evcount.h>
#include <sys/malloc.h>
#include <sys/queue.h>

#include <machine/bus.h>
#include <machine/fdt.h>
#include <machine/opal.h>

#include <dev/ofw/openfirm.h>
#include <dev/ofw/fdt.h>

#define XIVE_NUM_PRIORITIES	8
#define XIVE_NUM_IRQS		1024

#define XIVE_EQ_SIZE		PAGE_SHIFT
#define XIVE_EQ_IDX_MASK	((1 << (PAGE_SHIFT - 2)) - 1)
#define XIVE_EQ_GEN_MASK	0x80000000

#define XIVE_TM_CPPR_HV		0x031

#define XIVE_TM_SPC_ACK_HV	0x830
#define  XIVE_TM_SPC_ACK_HE_MASK	0xc000
#define  XIVE_TM_SPC_ACK_HE_NONE	0x0000
#define  XIVE_TM_SPC_ACK_HE_PHYS	0x8000

#define XIVE_ESB_STORE_TRIGGER	0x000
#define XIVE_ESB_LOAD_EOI	0x000
#define XIVE_ESB_STORE_EOI	0x400
#define XIVE_ESB_SET_PQ_00	0xc00
#define XIVE_ESB_SET_PQ_01	0xd00
#define XIVE_ESB_SET_PQ_10	0xe00
#define XIVE_ESB_SET_PQ_11	0xf00

#define XIVE_ESB_VAL_P	0x2
#define XIVE_ESB_VAL_Q	0x1

static inline uint8_t
xive_prio(int ipl)
{
	return ((IPL_IPI - ipl) > 7 ? 0xff : IPL_IPI - ipl);
}

static inline int
xive_ipl(uint8_t prio)
{
	return (IPL_IPI - prio);
}

struct intrhand {
	TAILQ_ENTRY(intrhand)	ih_list;
	int			(*ih_func)(void *);
	void			*ih_arg;
	int			ih_ipl;
	int			ih_flags;
	uint32_t		ih_girq;
	struct evcount		ih_count;
	const char		*ih_name;

	bus_space_handle_t	ih_esb_eoi;
	bus_space_handle_t	ih_esb_trig;
	uint64_t		ih_xive_flags;
};

struct xive_eq {
	struct xive_dmamem	*eq_queue;
	uint32_t		eq_idx;
	uint32_t		eq_gen;
};

struct xive_softc {
	struct device		sc_dev;
	bus_space_tag_t		sc_iot;
	bus_space_handle_t	sc_ioh;
	bus_dma_tag_t		sc_dmat;

	struct intrhand		*sc_handler[XIVE_NUM_IRQS];
	struct xive_eq		sc_eq[MAXCPUS][XIVE_NUM_PRIORITIES];

	uint32_t		sc_page_size;
	uint32_t		sc_lirq;
};

struct xive_softc *xive_sc;

struct xive_dmamem {
	bus_dmamap_t		xdm_map;
	bus_dma_segment_t	xdm_seg;
	size_t			xdm_size;
	caddr_t			xdm_kva;
};

#define XIVE_DMA_MAP(_xdm)	((_xdm)->xdm_map)
#define XIVE_DMA_LEN(_xdm)	((_xdm)->xdm_size)
#define XIVE_DMA_DVA(_xdm)	((_xdm)->xdm_map->dm_segs[0].ds_addr)
#define XIVE_DMA_KVA(_xdm)	((void *)(_xdm)->xdm_kva)

struct xive_dmamem *xive_dmamem_alloc(bus_dma_tag_t, bus_size_t,
	    bus_size_t);
void	xive_dmamem_free(bus_dma_tag_t, struct xive_dmamem *);

static inline void
xive_write_1(struct xive_softc *sc, bus_size_t off, uint8_t val)
{
	bus_space_write_1(sc->sc_iot, sc->sc_ioh, off, val);
}

static inline uint16_t
xive_read_2(struct xive_softc *sc, bus_size_t off)
{
	return bus_space_read_2(sc->sc_iot, sc->sc_ioh, off);
}

static inline void
xive_unmask(struct xive_softc *sc, struct intrhand *ih)
{
	bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, XIVE_ESB_SET_PQ_00);
}

int	xive_match(struct device *, void *, void *);
void	xive_attach(struct device *, struct device *, void *);
int	xive_activate(struct device *, int);

const struct cfattach xive_ca = {
	sizeof (struct xive_softc), xive_match, xive_attach, NULL,
	xive_activate
};

struct cfdriver xive_cd = {
	NULL, "xive", DV_DULL
};

void	xive_hvi(struct trapframe *);
void 	*xive_intr_establish(uint32_t, int, int, struct cpu_info *,
	    int (*)(void *), void *, const char *);
void	xive_intr_send_ipi(void *);
void	xive_setipl(int);

int
xive_match(struct device *parent, void *match, void *aux)
{
	struct fdt_attach_args *faa = aux;

	return OF_is_compatible(faa->fa_node, "ibm,opal-xive-pe");
}

void
xive_attach(struct device *parent, struct device *self, void *aux)
{
	struct xive_softc *sc = (struct xive_softc *)self;
	struct fdt_attach_args *faa = aux;
	struct cpu_info *ci;
	CPU_INFO_ITERATOR cii;
	int64_t error;
	int i;

	if (faa->fa_nreg < 2) {
		printf(": no registers\n");
		return;
	}

	sc->sc_iot = faa->fa_iot;
	if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr,
	    faa->fa_reg[1].size, 0, &sc->sc_ioh)) {
		printf(": can't map registers\n");
		return;
	}

	sc->sc_dmat = faa->fa_dmat;

	sc->sc_page_size = OF_getpropint(faa->fa_node,
	    "ibm,xive-provision-page-size", 0);

	error = opal_xive_reset(OPAL_XIVE_MODE_EXPL);
	if (error != OPAL_SUCCESS)
		printf(": can't enable exploitation mode\n");

	printf("\n");

	CPU_INFO_FOREACH(cii, ci) {
		for (i = 0; i < XIVE_NUM_PRIORITIES; i++) {
			sc->sc_eq[ci->ci_cpuid][i].eq_queue =
			    xive_dmamem_alloc(sc->sc_dmat,
			    1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE);
			if (sc->sc_eq[ci->ci_cpuid][i].eq_queue == NULL) {
				printf("%s: can't allocate event queue\n",
				    sc->sc_dev.dv_xname);
				return;
			}

			error = opal_xive_set_queue_info(ci->ci_pir, i,
			    XIVE_DMA_DVA(sc->sc_eq[ci->ci_cpuid][i].eq_queue),
			    XIVE_EQ_SIZE, OPAL_XIVE_EQ_ENABLED |
			    OPAL_XIVE_EQ_ALWAYS_NOTIFY);
			if (error != OPAL_SUCCESS) {
				printf("%s: can't enable event queue\n",
				    sc->sc_dev.dv_xname);
				return;
			}

			sc->sc_eq[ci->ci_cpuid][i].eq_gen = XIVE_EQ_GEN_MASK;
		}
	}

	/* There can be only one. */
	KASSERT(xive_sc == NULL);
	xive_sc = sc;

	_hvi = xive_hvi;
	_intr_establish = xive_intr_establish;
	_intr_send_ipi = xive_intr_send_ipi;
	_setipl = xive_setipl;

	/* Synchronize hardware state to software state. */
	xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(curcpu()->ci_cpl));
	eieio();
}

int
xive_activate(struct device *self, int act)
{
	switch (act) {
	case DVACT_POWERDOWN:
		opal_xive_reset(OPAL_XIVE_MODE_EMU);
		break;
	}

	return 0;
}

void *
xive_intr_establish(uint32_t girq, int type, int level, struct cpu_info *ci,
    int (*func)(void *), void *arg, const char *name)
{
	struct xive_softc *sc = xive_sc;
	struct intrhand *ih;
	bus_space_handle_t eoi, trig;
	bus_size_t page_size;
	uint64_t flags, eoi_page, trig_page;
	uint32_t esb_shift, lirq;
	int64_t error;

	if (ci == NULL)
		ci = cpu_info_primary;

	/* Allocate a logical IRQ. */
	if (sc->sc_lirq >= XIVE_NUM_IRQS)
		return NULL;
	lirq = sc->sc_lirq++;

	error = opal_xive_get_irq_info(girq, opal_phys(&flags),
	    opal_phys(&eoi_page), opal_phys(&trig_page),
	    opal_phys(&esb_shift), NULL);
	if (error != OPAL_SUCCESS)
		return NULL;
	page_size = 1 << esb_shift;

	/* Map EOI page. */
	if (bus_space_map(sc->sc_iot, eoi_page, page_size, 0, &eoi))
		return NULL;

	/* Map trigger page. */
	if (trig_page == eoi_page)
		trig = eoi;
	else if (trig_page == 0)
		trig = 0;
	else if (bus_space_map(sc->sc_iot, trig_page, page_size, 0, &trig)) {
		bus_space_unmap(sc->sc_iot, trig, page_size);
		return NULL;
	}

	error = opal_xive_set_irq_config(girq, ci->ci_pir,
	    xive_prio(level & IPL_IRQMASK), lirq);
	if (error != OPAL_SUCCESS) {
		if (trig != eoi && trig != 0)
			bus_space_unmap(sc->sc_iot, trig, page_size);
		bus_space_unmap(sc->sc_iot, eoi, page_size);
		return NULL;
	}

	ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK);
	ih->ih_func = func;
	ih->ih_arg = arg;
	ih->ih_ipl = level & IPL_IRQMASK;
	ih->ih_flags = level & IPL_FLAGMASK;
	ih->ih_girq = girq;
	ih->ih_name = name;
	ih->ih_esb_eoi = eoi;
	ih->ih_esb_trig = trig;
	ih->ih_xive_flags = flags;
	sc->sc_handler[lirq] = ih;

	if (name != NULL)
		evcount_attach(&ih->ih_count, name, &ih->ih_girq);

	xive_unmask(sc, ih);

	return ih;
}

void
xive_intr_send_ipi(void *cookie)
{
	struct xive_softc *sc = xive_sc;
	struct intrhand *ih = cookie;

	if (ih && ih->ih_esb_trig)
		bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
		    XIVE_ESB_STORE_TRIGGER, 0);
}

void
xive_eoi(struct xive_softc *sc, struct intrhand *ih)
{
	uint64_t eoi;

	if (ih->ih_xive_flags & OPAL_XIVE_IRQ_STORE_EOI) {
		bus_space_write_8(sc->sc_iot, ih->ih_esb_eoi,
		    XIVE_ESB_STORE_EOI, 0);
	} else if (ih->ih_xive_flags & OPAL_XIVE_IRQ_LSI) {
		eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi,
		    XIVE_ESB_LOAD_EOI);
	} else {
		eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi,
		    XIVE_ESB_SET_PQ_00);
		if ((eoi & XIVE_ESB_VAL_Q) && ih->ih_esb_trig != 0)
			bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
			    XIVE_ESB_STORE_TRIGGER, 0);
	}
}

void
xive_setipl(int new)
{
	struct xive_softc *sc = xive_sc;
	struct cpu_info *ci = curcpu();
	uint8_t oldprio = xive_prio(ci->ci_cpl);
	uint8_t newprio = xive_prio(new);
	u_long msr;

	msr = intr_disable();
	ci->ci_cpl = new;
	if (newprio != oldprio) {
		xive_write_1(sc, XIVE_TM_CPPR_HV, newprio);
		eieio();
	}
	intr_restore(msr);
}

void
xive_run_handler(struct intrhand *ih)
{
	int handled;

#ifdef MULTIPROCESSOR
	int need_lock;

	if (ih->ih_flags & IPL_MPSAFE)
		need_lock = 0;
	else
		need_lock = (ih->ih_ipl < IPL_SCHED);

	if (need_lock)
		KERNEL_LOCK();
#endif
	handled = ih->ih_func(ih->ih_arg);
	if (handled)
		ih->ih_count.ec_count++;
#ifdef MULTIPROCESSOR
	if (need_lock)
		KERNEL_UNLOCK();
#endif
}

void
xive_hvi(struct trapframe *frame)
{
	struct xive_softc *sc = xive_sc;
	struct cpu_info *ci = curcpu();
	struct intrhand *ih;
	struct xive_eq *eq;
	uint32_t *event;
	uint32_t lirq;
	int old, new;
	uint16_t ack, he;
	uint8_t cppr;

	old = ci->ci_cpl;

	while (1) {
		ack = xive_read_2(sc, XIVE_TM_SPC_ACK_HV);

		he = (ack & XIVE_TM_SPC_ACK_HE_MASK);
		if (he == XIVE_TM_SPC_ACK_HE_NONE)
			break;
		KASSERT(he == XIVE_TM_SPC_ACK_HE_PHYS);

		eieio();

		/* Synchronize software state to hardware state. */
		cppr = ack;
		new = xive_ipl(cppr);
		if (new <= old) {
			/*
			 * QEMU generates spurious interrupts.  It is
			 * unclear whether this can happen on real
			 * hardware as well.  We just ignore the
			 * interrupt, but we need to reset the CPPR
			 * register since we did accept the interrupt.
			 */
			goto spurious;
		}
		ci->ci_cpl = new;

		KASSERT(cppr < XIVE_NUM_PRIORITIES);
		eq = &sc->sc_eq[ci->ci_cpuid][cppr];
		event = XIVE_DMA_KVA(eq->eq_queue);
		while ((event[eq->eq_idx] & XIVE_EQ_GEN_MASK) == eq->eq_gen) {
			lirq = event[eq->eq_idx] & ~XIVE_EQ_GEN_MASK;
			KASSERT(lirq < XIVE_NUM_IRQS);
			ih = sc->sc_handler[lirq];
			if (ih != NULL) {
				intr_enable();
				xive_run_handler(ih);
				intr_disable();
				xive_eoi(sc, ih);
			}
			eq->eq_idx = (eq->eq_idx + 1) & XIVE_EQ_IDX_MASK;

			/* Toggle generation on wrap around. */
			if (eq->eq_idx == 0)
				eq->eq_gen ^= XIVE_EQ_GEN_MASK;
		}

		ci->ci_cpl = old;
	spurious:
		xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(old));
		eieio();
	}
}

struct xive_dmamem *
xive_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align)
{
	struct xive_dmamem *xdm;
	int nsegs;

	xdm = malloc(sizeof(*xdm), M_DEVBUF, M_WAITOK | M_ZERO);
	xdm->xdm_size = size;

	if (bus_dmamap_create(dmat, size, 1, size, 0,
	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &xdm->xdm_map) != 0)
		goto xdmfree;

	if (bus_dmamem_alloc(dmat, size, align, 0, &xdm->xdm_seg, 1,
	    &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
		goto destroy;

	if (bus_dmamem_map(dmat, &xdm->xdm_seg, nsegs, size,
	    &xdm->xdm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0)
		goto free;

	if (bus_dmamap_load_raw(dmat, xdm->xdm_map, &xdm->xdm_seg,
	    nsegs, size, BUS_DMA_WAITOK) != 0)
		goto unmap;

	return xdm;

unmap:
	bus_dmamem_unmap(dmat, xdm->xdm_kva, size);
free:
	bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
destroy:
	bus_dmamap_destroy(dmat, xdm->xdm_map);
xdmfree:
	free(xdm, M_DEVBUF, sizeof(*xdm));

	return NULL;
}

void
xive_dmamem_free(bus_dma_tag_t dmat, struct xive_dmamem *xdm)
{
	bus_dmamem_unmap(dmat, xdm->xdm_kva, xdm->xdm_size);
	bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
	bus_dmamap_destroy(dmat, xdm->xdm_map);
	free(xdm, M_DEVBUF, sizeof(*xdm));
}