xref: /openbsd-src/sys/dev/ic/ahci.c (revision 1cfc6e0b1eece719c373d1485b8624522bbc4ea3)
1*1cfc6e0bSjan /*	$OpenBSD: ahci.c,v 1.43 2024/11/22 09:29:41 jan Exp $ */
27acdc34dSpatrick 
37acdc34dSpatrick /*
47acdc34dSpatrick  * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
57acdc34dSpatrick  * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
67acdc34dSpatrick  * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
77acdc34dSpatrick  *
87acdc34dSpatrick  * Permission to use, copy, modify, and distribute this software for any
97acdc34dSpatrick  * purpose with or without fee is hereby granted, provided that the above
107acdc34dSpatrick  * copyright notice and this permission notice appear in all copies.
117acdc34dSpatrick  *
127acdc34dSpatrick  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
137acdc34dSpatrick  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
147acdc34dSpatrick  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
157acdc34dSpatrick  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
167acdc34dSpatrick  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
177acdc34dSpatrick  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
187acdc34dSpatrick  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
197acdc34dSpatrick  */
207acdc34dSpatrick 
217acdc34dSpatrick #include <sys/param.h>
227acdc34dSpatrick #include <sys/systm.h>
237acdc34dSpatrick #include <sys/buf.h>
247acdc34dSpatrick #include <sys/kernel.h>
257acdc34dSpatrick #include <sys/malloc.h>
267acdc34dSpatrick #include <sys/device.h>
277acdc34dSpatrick #include <sys/queue.h>
287acdc34dSpatrick #include <sys/mutex.h>
297acdc34dSpatrick #include <sys/pool.h>
307acdc34dSpatrick 
317acdc34dSpatrick #include <machine/bus.h>
327acdc34dSpatrick 
337acdc34dSpatrick #include <dev/ic/ahcireg.h>
3406133b5aSdlg #include <dev/ic/ahcivar.h>
357acdc34dSpatrick 
361b1d835dSjmatthew #ifdef AHCI_DEBUG
371b1d835dSjmatthew #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
381b1d835dSjmatthew     while (0)
391b1d835dSjmatthew #define AHCI_D_TIMEOUT		0x00
401b1d835dSjmatthew #define AHCI_D_VERBOSE		0x01
411b1d835dSjmatthew #define AHCI_D_INTR		0x02
421b1d835dSjmatthew #define AHCI_D_XFER		0x08
431b1d835dSjmatthew int ahcidebug = AHCI_D_VERBOSE;
441b1d835dSjmatthew #else
451b1d835dSjmatthew #define DPRINTF(m, f...)
461b1d835dSjmatthew #endif
471b1d835dSjmatthew 
4876d3d104Sderaadt #ifdef HIBERNATE
494bb7a4f6Smpi #include <uvm/uvm_extern.h>
5076d3d104Sderaadt #include <sys/hibernate.h>
5176d3d104Sderaadt #include <sys/disk.h>
5276d3d104Sderaadt #include <sys/disklabel.h>
5376d3d104Sderaadt 
5476d3d104Sderaadt #include <scsi/scsi_all.h>
5576d3d104Sderaadt #include <scsi/scsiconf.h>
561b1d835dSjmatthew 
577acdc34dSpatrick void			ahci_hibernate_io_start(struct ahci_port *,
587acdc34dSpatrick 			    struct ahci_ccb *);
597acdc34dSpatrick int			ahci_hibernate_io_poll(struct ahci_port *,
607acdc34dSpatrick 			    struct ahci_ccb *);
617acdc34dSpatrick void			ahci_hibernate_load_prdt(struct ahci_ccb *);
627acdc34dSpatrick 
637acdc34dSpatrick int			ahci_hibernate_io(dev_t dev, daddr_t blkno,
647acdc34dSpatrick 			    vaddr_t addr, size_t size, int wr, void *page);
657acdc34dSpatrick #endif
667acdc34dSpatrick 
677acdc34dSpatrick struct cfdriver ahci_cd = {
687acdc34dSpatrick 	NULL, "ahci", DV_DULL
697acdc34dSpatrick };
707acdc34dSpatrick 
717acdc34dSpatrick void			ahci_enable_interrupts(struct ahci_port *);
727acdc34dSpatrick 
737acdc34dSpatrick int			ahci_init(struct ahci_softc *);
747acdc34dSpatrick int			ahci_port_alloc(struct ahci_softc *, u_int);
7596739e86Sjmatthew void			ahci_port_detect(struct ahci_softc *, u_int);
767acdc34dSpatrick void			ahci_port_free(struct ahci_softc *, u_int);
777acdc34dSpatrick int			ahci_port_init(struct ahci_softc *, u_int);
787acdc34dSpatrick 
7993c0f120Sjsg int			ahci_default_port_start(struct ahci_port *, int);
807acdc34dSpatrick int			ahci_port_stop(struct ahci_port *, int);
817acdc34dSpatrick int			ahci_port_clo(struct ahci_port *);
827acdc34dSpatrick int			ahci_port_softreset(struct ahci_port *);
8396739e86Sjmatthew void			ahci_port_comreset(struct ahci_port *);
847acdc34dSpatrick int			ahci_port_portreset(struct ahci_port *, int);
8596739e86Sjmatthew void			ahci_port_portreset_start(struct ahci_port *);
8696739e86Sjmatthew int			ahci_port_portreset_poll(struct ahci_port *);
8796739e86Sjmatthew void			ahci_port_portreset_wait(struct ahci_port *);
8896739e86Sjmatthew int			ahci_port_portreset_finish(struct ahci_port *, int);
897acdc34dSpatrick int			ahci_port_signature(struct ahci_port *);
907acdc34dSpatrick int			ahci_pmp_port_softreset(struct ahci_port *, int);
917acdc34dSpatrick int			ahci_pmp_port_portreset(struct ahci_port *, int);
927acdc34dSpatrick int			ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
937acdc34dSpatrick 
947acdc34dSpatrick int			ahci_load_prdt(struct ahci_ccb *);
95b1514dfeSdlg void			ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96b1514dfeSdlg 			    u_int32_t, u_int32_t);
977acdc34dSpatrick void			ahci_unload_prdt(struct ahci_ccb *);
987acdc34dSpatrick 
997acdc34dSpatrick int			ahci_poll(struct ahci_ccb *, int, void (*)(void *));
1007acdc34dSpatrick void			ahci_start(struct ahci_ccb *);
1017acdc34dSpatrick 
1027acdc34dSpatrick void			ahci_issue_pending_ncq_commands(struct ahci_port *);
1037acdc34dSpatrick void			ahci_issue_pending_commands(struct ahci_port *, int);
1047acdc34dSpatrick 
1057acdc34dSpatrick int			ahci_intr(void *);
1067acdc34dSpatrick u_int32_t		ahci_port_intr(struct ahci_port *, u_int32_t);
1077acdc34dSpatrick 
1087acdc34dSpatrick struct ahci_ccb		*ahci_get_ccb(struct ahci_port *);
1097acdc34dSpatrick void			ahci_put_ccb(struct ahci_ccb *);
1107acdc34dSpatrick 
1117acdc34dSpatrick struct ahci_ccb		*ahci_get_err_ccb(struct ahci_port *);
1127acdc34dSpatrick void			ahci_put_err_ccb(struct ahci_ccb *);
1137acdc34dSpatrick 
1147acdc34dSpatrick struct ahci_ccb		*ahci_get_pmp_ccb(struct ahci_port *);
1157acdc34dSpatrick void			ahci_put_pmp_ccb(struct ahci_ccb *);
1167acdc34dSpatrick 
1177acdc34dSpatrick int			ahci_port_read_ncq_error(struct ahci_port *, int *, int);
1187acdc34dSpatrick 
1197acdc34dSpatrick struct ahci_dmamem	*ahci_dmamem_alloc(struct ahci_softc *, size_t);
1207acdc34dSpatrick void			ahci_dmamem_free(struct ahci_softc *,
1217acdc34dSpatrick 			    struct ahci_dmamem *);
1227acdc34dSpatrick 
1237acdc34dSpatrick u_int32_t		ahci_read(struct ahci_softc *, bus_size_t);
1247acdc34dSpatrick void			ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
1257acdc34dSpatrick int			ahci_wait_ne(struct ahci_softc *, bus_size_t,
1267acdc34dSpatrick 			    u_int32_t, u_int32_t);
1277acdc34dSpatrick 
1287acdc34dSpatrick u_int32_t		ahci_pread(struct ahci_port *, bus_size_t);
1297acdc34dSpatrick void			ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
1307acdc34dSpatrick int			ahci_pwait_eq(struct ahci_port *, bus_size_t,
1317acdc34dSpatrick 			    u_int32_t, u_int32_t, int);
1327acdc34dSpatrick void			ahci_flush_tfd(struct ahci_port *ap);
1337acdc34dSpatrick u_int32_t		ahci_active_mask(struct ahci_port *);
1347acdc34dSpatrick int			ahci_port_detect_pmp(struct ahci_port *);
1357acdc34dSpatrick void			ahci_pmp_probe_timeout(void *);
1367acdc34dSpatrick 
1377acdc34dSpatrick /* pmp operations */
1387acdc34dSpatrick int			ahci_pmp_read(struct ahci_port *, int, int,
1397acdc34dSpatrick 			    u_int32_t *);
1407acdc34dSpatrick int			ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
1417acdc34dSpatrick int			ahci_pmp_phy_status(struct ahci_port *, int,
1427acdc34dSpatrick 			    u_int32_t *);
1437acdc34dSpatrick int 			ahci_pmp_identify(struct ahci_port *, int *);
1447acdc34dSpatrick 
1457acdc34dSpatrick 
1467acdc34dSpatrick /* Wait for all bits in _b to be cleared */
1477acdc34dSpatrick #define ahci_pwait_clr(_ap, _r, _b, _n) \
1487acdc34dSpatrick    ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
1497acdc34dSpatrick 
1507acdc34dSpatrick /* Wait for all bits in _b to be set */
1517acdc34dSpatrick #define ahci_pwait_set(_ap, _r, _b, _n) \
1527acdc34dSpatrick    ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
1537acdc34dSpatrick 
1547acdc34dSpatrick 
1557acdc34dSpatrick 
1567acdc34dSpatrick /* provide methods for atascsi to call */
1577acdc34dSpatrick int			ahci_ata_probe(void *, int, int);
1587acdc34dSpatrick void			ahci_ata_free(void *, int, int);
1597acdc34dSpatrick struct ata_xfer *	ahci_ata_get_xfer(void *, int);
1607acdc34dSpatrick void			ahci_ata_put_xfer(struct ata_xfer *);
1617acdc34dSpatrick void			ahci_ata_cmd(struct ata_xfer *);
1627acdc34dSpatrick 
16351d3b00dSnaddy const struct atascsi_methods ahci_atascsi_methods = {
1647acdc34dSpatrick 	ahci_ata_probe,
1657acdc34dSpatrick 	ahci_ata_free,
1667acdc34dSpatrick 	ahci_ata_get_xfer,
1677acdc34dSpatrick 	ahci_ata_put_xfer,
1687acdc34dSpatrick 	ahci_ata_cmd
1697acdc34dSpatrick };
1707acdc34dSpatrick 
1717acdc34dSpatrick /* ccb completions */
1727acdc34dSpatrick void			ahci_ata_cmd_done(struct ahci_ccb *);
1737acdc34dSpatrick void			ahci_pmp_cmd_done(struct ahci_ccb *);
1747acdc34dSpatrick void			ahci_ata_cmd_timeout(void *);
1757acdc34dSpatrick void			ahci_empty_done(struct ahci_ccb *);
1767acdc34dSpatrick 
1777acdc34dSpatrick int
1787acdc34dSpatrick ahci_attach(struct ahci_softc *sc)
1797acdc34dSpatrick {
1807acdc34dSpatrick 	struct atascsi_attach_args	aaa;
1817acdc34dSpatrick 	u_int32_t			pi;
18296739e86Sjmatthew 	int				i, j, done;
1837acdc34dSpatrick 
18493c0f120Sjsg 	if (sc->sc_port_start == NULL)
18593c0f120Sjsg 		sc->sc_port_start = ahci_default_port_start;
18693c0f120Sjsg 
1877acdc34dSpatrick 	if (ahci_init(sc) != 0) {
1887acdc34dSpatrick 		/* error already printed by ahci_init */
1897acdc34dSpatrick 		goto unmap;
1907acdc34dSpatrick 	}
1917acdc34dSpatrick 
1927acdc34dSpatrick 	printf("\n");
1937acdc34dSpatrick 
1947acdc34dSpatrick 	sc->sc_cap = ahci_read(sc, AHCI_REG_CAP);
1957acdc34dSpatrick 	sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap);
1967acdc34dSpatrick #ifdef AHCI_DEBUG
1977acdc34dSpatrick 	if (ahcidebug & AHCI_D_VERBOSE) {
1987acdc34dSpatrick 		const char *gen;
1997acdc34dSpatrick 
2007acdc34dSpatrick 		switch (sc->sc_cap & AHCI_REG_CAP_ISS) {
2017acdc34dSpatrick 		case AHCI_REG_CAP_ISS_G1:
2027acdc34dSpatrick 			gen = "1 (1.5Gbps)";
2037acdc34dSpatrick 			break;
20471e33dbaSpelikan 		case AHCI_REG_CAP_ISS_G2:
2059cd47e4cSmpi 			gen = "2 (3.0Gb/s)";
206b049ffa5Spelikan 			break;
20771e33dbaSpelikan 		case AHCI_REG_CAP_ISS_G3:
2089cd47e4cSmpi 			gen = "3 (6.0Gb/s)";
2097acdc34dSpatrick 			break;
2107acdc34dSpatrick 		default:
2117acdc34dSpatrick 			gen = "unknown";
2127acdc34dSpatrick 			break;
2137acdc34dSpatrick 		}
2147acdc34dSpatrick 
2157acdc34dSpatrick 		printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
2167acdc34dSpatrick 		    DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP,
2177acdc34dSpatrick 		    AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen);
2181ec8db95Sjmatthew 		printf("%s: extended capabilities 0x%b\n", DEVNAME(sc),
2191ec8db95Sjmatthew 		    ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2);
2207acdc34dSpatrick 	}
2217acdc34dSpatrick #endif
2227acdc34dSpatrick 
2237acdc34dSpatrick 	pi = ahci_read(sc, AHCI_REG_PI);
2247acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
2257acdc34dSpatrick 	    DEVNAME(sc), pi);
2267acdc34dSpatrick 
2277acdc34dSpatrick #ifdef AHCI_COALESCE
2287acdc34dSpatrick 	/* Naive coalescing support - enable for all ports. */
2297acdc34dSpatrick 	if (sc->sc_cap & AHCI_REG_CAP_CCCS) {
2307acdc34dSpatrick 		u_int16_t		ccc_timeout = 20;
2317acdc34dSpatrick 		u_int8_t		ccc_numcomplete = 12;
2327acdc34dSpatrick 		u_int32_t		ccc_ctl;
2337acdc34dSpatrick 
2347acdc34dSpatrick 		/* disable coalescing during reconfiguration. */
2357acdc34dSpatrick 		ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
2367acdc34dSpatrick 		ccc_ctl &= ~0x00000001;
2377acdc34dSpatrick 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
2387acdc34dSpatrick 
2397acdc34dSpatrick 		sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
2407acdc34dSpatrick 		if (pi & sc->sc_ccc_mask) {
2417acdc34dSpatrick 			/* A conflict with the implemented port list? */
2427acdc34dSpatrick 			printf("%s: coalescing interrupt/implemented port list "
2437acdc34dSpatrick 			    "conflict, PI: %08x, ccc_mask: %08x\n",
2447acdc34dSpatrick 			    DEVNAME(sc), pi, sc->sc_ccc_mask);
2457acdc34dSpatrick 			sc->sc_ccc_mask = 0;
2467acdc34dSpatrick 			goto noccc;
2477acdc34dSpatrick 		}
2487acdc34dSpatrick 
2497acdc34dSpatrick 		/* ahci_port_start will enable each port when it starts. */
2507acdc34dSpatrick 		sc->sc_ccc_ports = pi;
2517acdc34dSpatrick 		sc->sc_ccc_ports_cur = 0;
2527acdc34dSpatrick 
2537acdc34dSpatrick 		/* program thresholds and enable overall coalescing. */
2547acdc34dSpatrick 		ccc_ctl &= ~0xffffff00;
2557acdc34dSpatrick 		ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
2567acdc34dSpatrick 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
2577acdc34dSpatrick 		ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
2587acdc34dSpatrick 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
2597acdc34dSpatrick 	}
2607acdc34dSpatrick noccc:
2617acdc34dSpatrick #endif
2629ae3568aSpatrick 	/*
2639ae3568aSpatrick 	 * Given that ahci_port_alloc() will grab one CCB for error recovery
2649ae3568aSpatrick 	 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
2659ae3568aSpatrick 	 * pretend at least 2 command slots for devices without NCQ support.
2669ae3568aSpatrick 	 * That way, also at least 1 slot is made available for atascsi(4).
2679ae3568aSpatrick 	 */
2689ae3568aSpatrick 	sc->sc_ncmds = max(2, sc->sc_ncmds);
2697acdc34dSpatrick 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
2706b06da21Sjsg 		if (!ISSET(pi, 1U << i)) {
2719593dc34Smglocker 			/* don't allocate stuff if the port isn't implemented */
2727acdc34dSpatrick 			continue;
2737acdc34dSpatrick 		}
2747acdc34dSpatrick 
2757acdc34dSpatrick 		if (ahci_port_alloc(sc, i) == ENOMEM)
2767acdc34dSpatrick 			goto freeports;
27796739e86Sjmatthew 
27896739e86Sjmatthew 		if (sc->sc_ports[i] != NULL)
27996739e86Sjmatthew 			ahci_port_portreset_start(sc->sc_ports[i]);
28096739e86Sjmatthew 	}
28196739e86Sjmatthew 
28296739e86Sjmatthew 	/*
28396739e86Sjmatthew 	 * Poll for device detection until all ports report a device, or one
28496739e86Sjmatthew 	 * second has elapsed.
28596739e86Sjmatthew 	 */
28696739e86Sjmatthew 	for (i = 0; i < 1000; i++) {
28796739e86Sjmatthew 		done = 1;
28896739e86Sjmatthew 		for (j = 0; j < AHCI_MAX_PORTS; j++) {
28996739e86Sjmatthew 			if (sc->sc_ports[j] == NULL)
29096739e86Sjmatthew 				continue;
29196739e86Sjmatthew 
29296739e86Sjmatthew 			if (ahci_port_portreset_poll(sc->sc_ports[j]))
29396739e86Sjmatthew 				done = 0;
29496739e86Sjmatthew 		}
29596739e86Sjmatthew 
29696739e86Sjmatthew 		if (done)
29796739e86Sjmatthew 			break;
29896739e86Sjmatthew 
29996739e86Sjmatthew 		delay(1000);
30096739e86Sjmatthew 	}
30196739e86Sjmatthew 
30296739e86Sjmatthew 	/*
30396739e86Sjmatthew 	 * Finish device detection on all ports that initialized.
30496739e86Sjmatthew 	 */
30596739e86Sjmatthew 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
30696739e86Sjmatthew 		if (sc->sc_ports[i] != NULL)
30796739e86Sjmatthew 			ahci_port_detect(sc, i);
3087acdc34dSpatrick 	}
3097acdc34dSpatrick 
310414fa24dSdlg 	memset(&aaa, 0, sizeof(aaa));
3117acdc34dSpatrick 	aaa.aaa_cookie = sc;
3127acdc34dSpatrick 	aaa.aaa_methods = &ahci_atascsi_methods;
3137acdc34dSpatrick 	aaa.aaa_minphys = NULL;
3147acdc34dSpatrick 	aaa.aaa_nports = AHCI_MAX_PORTS;
3157acdc34dSpatrick 	aaa.aaa_ncmds = sc->sc_ncmds - 1;
3167acdc34dSpatrick 	if (!(sc->sc_flags & AHCI_F_NO_NCQ) &&
317fd58cd83Spatrick 	    sc->sc_ncmds > 2 &&
3187acdc34dSpatrick 	    (sc->sc_cap & AHCI_REG_CAP_SNCQ)) {
3197acdc34dSpatrick 		aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ;
3207acdc34dSpatrick 	}
3217acdc34dSpatrick 
3227acdc34dSpatrick 	sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
3237acdc34dSpatrick 
324*1cfc6e0bSjan 	/* Flush all residual bits of the interrupt status register */
325*1cfc6e0bSjan 	ahci_write(sc, AHCI_REG_IS, ahci_read(sc, AHCI_REG_IS));
326*1cfc6e0bSjan 
3277acdc34dSpatrick 	/* Enable interrupts */
3287acdc34dSpatrick 	ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
3297acdc34dSpatrick 
3307acdc34dSpatrick 	return 0;
3317acdc34dSpatrick 
3327acdc34dSpatrick freeports:
3337acdc34dSpatrick 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
3347acdc34dSpatrick 		if (sc->sc_ports[i] != NULL)
3357acdc34dSpatrick 			ahci_port_free(sc, i);
3367acdc34dSpatrick 	}
3377acdc34dSpatrick unmap:
3387acdc34dSpatrick 	/* Disable controller */
3397acdc34dSpatrick 	ahci_write(sc, AHCI_REG_GHC, 0);
3407acdc34dSpatrick 	return 1;
3417acdc34dSpatrick }
3427acdc34dSpatrick 
3437acdc34dSpatrick int
3447acdc34dSpatrick ahci_detach(struct ahci_softc *sc, int flags)
3457acdc34dSpatrick {
3467acdc34dSpatrick 	int				 rv, i;
3477acdc34dSpatrick 
3487acdc34dSpatrick 	if (sc->sc_atascsi != NULL) {
3497acdc34dSpatrick 		rv = atascsi_detach(sc->sc_atascsi, flags);
3507acdc34dSpatrick 		if (rv != 0)
3517acdc34dSpatrick 			return (rv);
3527acdc34dSpatrick 	}
3537acdc34dSpatrick 
3547acdc34dSpatrick 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
3557acdc34dSpatrick 		if (sc->sc_ports[i] != NULL)
3567acdc34dSpatrick 			ahci_port_free(sc, i);
3577acdc34dSpatrick 	}
3587acdc34dSpatrick 
3597acdc34dSpatrick 	return (0);
3607acdc34dSpatrick }
3617acdc34dSpatrick 
3627acdc34dSpatrick int
3637acdc34dSpatrick ahci_activate(struct device *self, int act)
3647acdc34dSpatrick {
3657acdc34dSpatrick 	struct ahci_softc		*sc = (struct ahci_softc *)self;
3667acdc34dSpatrick 	int				 i, rv = 0;
3677acdc34dSpatrick 
3687acdc34dSpatrick 	switch (act) {
3697acdc34dSpatrick 	case DVACT_RESUME:
3707acdc34dSpatrick 		/* enable ahci (global interrupts disabled) */
3717acdc34dSpatrick 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
3727acdc34dSpatrick 
3737acdc34dSpatrick 		/* restore BIOS initialised parameters */
3747acdc34dSpatrick 		ahci_write(sc, AHCI_REG_CAP, sc->sc_cap);
3757acdc34dSpatrick 
3767acdc34dSpatrick 		for (i = 0; i < AHCI_MAX_PORTS; i++) {
3777acdc34dSpatrick 			if (sc->sc_ports[i] != NULL)
3787acdc34dSpatrick 				ahci_port_init(sc, i);
3797acdc34dSpatrick 		}
3807acdc34dSpatrick 
3817acdc34dSpatrick 		/* Enable interrupts */
3827acdc34dSpatrick 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
3837acdc34dSpatrick 
3847acdc34dSpatrick 		rv = config_activate_children(self, act);
3857acdc34dSpatrick 		break;
38637ecb596Sderaadt 	case DVACT_POWERDOWN:
38737ecb596Sderaadt 		rv = config_activate_children(self, act);
38837ecb596Sderaadt 		for (i = 0; i < AHCI_MAX_PORTS; i++) {
38937ecb596Sderaadt 			if (sc->sc_ports[i] != NULL)
39037ecb596Sderaadt 				ahci_port_stop(sc->sc_ports[i], 1);
39137ecb596Sderaadt 		}
39237ecb596Sderaadt 		break;
39337ecb596Sderaadt 	default:
39437ecb596Sderaadt 		rv = config_activate_children(self, act);
39537ecb596Sderaadt 		break;
3967acdc34dSpatrick 	}
3977acdc34dSpatrick 	return (rv);
3987acdc34dSpatrick }
3997acdc34dSpatrick 
4007acdc34dSpatrick int
4017acdc34dSpatrick ahci_init(struct ahci_softc *sc)
4027acdc34dSpatrick {
4037acdc34dSpatrick 	u_int32_t			reg, cap, pi;
4047acdc34dSpatrick 	const char			*revision;
4057acdc34dSpatrick 
4067acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
4077acdc34dSpatrick 	    AHCI_FMT_GHC);
4087acdc34dSpatrick 
4097acdc34dSpatrick 	/* save BIOS initialised parameters, enable staggered spin up */
4107acdc34dSpatrick 	cap = ahci_read(sc, AHCI_REG_CAP);
4117acdc34dSpatrick 	cap &= AHCI_REG_CAP_SMPS;
4127acdc34dSpatrick 	cap |= AHCI_REG_CAP_SSS;
4137acdc34dSpatrick 	pi = ahci_read(sc, AHCI_REG_PI);
4147acdc34dSpatrick 
4157acdc34dSpatrick 	if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
4167acdc34dSpatrick 		/* reset the controller */
4177acdc34dSpatrick 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
4187acdc34dSpatrick 		if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
4197acdc34dSpatrick 		    AHCI_REG_GHC_HR) != 0) {
4207acdc34dSpatrick 			printf(" unable to reset controller\n");
4217acdc34dSpatrick 			return (1);
4227acdc34dSpatrick 		}
4237acdc34dSpatrick 	}
4247acdc34dSpatrick 
4257acdc34dSpatrick 	/* enable ahci (global interrupts disabled) */
4267acdc34dSpatrick 	ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
4277acdc34dSpatrick 
4287acdc34dSpatrick 	/* restore parameters */
4297acdc34dSpatrick 	ahci_write(sc, AHCI_REG_CAP, cap);
4307acdc34dSpatrick 	ahci_write(sc, AHCI_REG_PI, pi);
4317acdc34dSpatrick 
4327acdc34dSpatrick 	/* check the revision */
4337acdc34dSpatrick 	reg = ahci_read(sc, AHCI_REG_VS);
4347acdc34dSpatrick 	switch (reg) {
4357acdc34dSpatrick 	case AHCI_REG_VS_0_95:
4367acdc34dSpatrick 		revision = "0.95";
4377acdc34dSpatrick 		break;
4387acdc34dSpatrick 	case AHCI_REG_VS_1_0:
4397acdc34dSpatrick 		revision = "1.0";
4407acdc34dSpatrick 		break;
4417acdc34dSpatrick 	case AHCI_REG_VS_1_1:
4427acdc34dSpatrick 		revision = "1.1";
4437acdc34dSpatrick 		break;
4447acdc34dSpatrick 	case AHCI_REG_VS_1_2:
4457acdc34dSpatrick 		revision = "1.2";
4467acdc34dSpatrick 		break;
4477acdc34dSpatrick 	case AHCI_REG_VS_1_3:
4487acdc34dSpatrick 		revision = "1.3";
4497acdc34dSpatrick 		break;
450a4da9f86Sjmatthew 	case AHCI_REG_VS_1_3_1:
451a4da9f86Sjmatthew 		revision = "1.3.1";
452a4da9f86Sjmatthew 		break;
4537acdc34dSpatrick 
4547acdc34dSpatrick 	default:
4557acdc34dSpatrick 		printf(" unsupported AHCI revision 0x%08x\n", reg);
4567acdc34dSpatrick 		return (1);
4577acdc34dSpatrick 	}
4587acdc34dSpatrick 
4597acdc34dSpatrick 	printf(" AHCI %s", revision);
4607acdc34dSpatrick 
4617acdc34dSpatrick 	return (0);
4627acdc34dSpatrick }
4637acdc34dSpatrick 
4647acdc34dSpatrick void
4657acdc34dSpatrick ahci_enable_interrupts(struct ahci_port *ap)
4667acdc34dSpatrick {
4677acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
4687acdc34dSpatrick 	    AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
4697acdc34dSpatrick 	    AHCI_PREG_IE_UFE |
4707acdc34dSpatrick 	    ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) |
4717acdc34dSpatrick #ifdef AHCI_COALESCE
4727acdc34dSpatrick 	    ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
4737acdc34dSpatrick 	     (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE))
4747acdc34dSpatrick #else
4757acdc34dSpatrick 	    AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
4767acdc34dSpatrick #endif
4777acdc34dSpatrick 	    );
4787acdc34dSpatrick }
4797acdc34dSpatrick 
4807acdc34dSpatrick int
4817acdc34dSpatrick ahci_port_alloc(struct ahci_softc *sc, u_int port)
4827acdc34dSpatrick {
4837acdc34dSpatrick 	struct ahci_port		*ap;
4847acdc34dSpatrick 	struct ahci_ccb			*ccb;
4857acdc34dSpatrick 	u_int64_t			dva;
4867acdc34dSpatrick 	u_int32_t			cmd;
4877acdc34dSpatrick 	struct ahci_cmd_hdr		*hdr;
4887acdc34dSpatrick 	struct ahci_cmd_table		*table;
4897acdc34dSpatrick 	int				i, rc = ENOMEM;
4907acdc34dSpatrick 
4917acdc34dSpatrick 	ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO);
4927acdc34dSpatrick 	if (ap == NULL) {
4937acdc34dSpatrick 		printf("%s: unable to allocate memory for port %d\n",
4947acdc34dSpatrick 		    DEVNAME(sc), port);
4957acdc34dSpatrick 		goto reterr;
4967acdc34dSpatrick 	}
4977acdc34dSpatrick 	ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO);
4987acdc34dSpatrick 	if (ap->ap_err_scratch == NULL) {
4997acdc34dSpatrick 		printf("%s: unable to allocate DMA scratch buf for port %d\n",
5007acdc34dSpatrick 		    DEVNAME(sc), port);
5014ba7bf3bSjmatthew 		free(ap, M_DEVBUF, sizeof(*ap));
5024ba7bf3bSjmatthew 		goto reterr;
5037acdc34dSpatrick 	}
5047acdc34dSpatrick 
5057acdc34dSpatrick #ifdef AHCI_DEBUG
5067acdc34dSpatrick 	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
5077acdc34dSpatrick 	    DEVNAME(sc), port);
5087acdc34dSpatrick #endif
5097acdc34dSpatrick 	ap->ap_port = port;
5107acdc34dSpatrick 	sc->sc_ports[port] = ap;
5117acdc34dSpatrick 
5127acdc34dSpatrick 	if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
5137acdc34dSpatrick 	    AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
5147acdc34dSpatrick 		printf("%s: unable to create register window for port %d\n",
5157acdc34dSpatrick 		    DEVNAME(sc), port);
5167acdc34dSpatrick 		goto freeport;
5177acdc34dSpatrick 	}
5187acdc34dSpatrick 
5197acdc34dSpatrick 	ap->ap_sc = sc;
5207acdc34dSpatrick #ifdef AHCI_COALESCE
5217acdc34dSpatrick 	ap->ap_num = port;
5227acdc34dSpatrick #endif
5237acdc34dSpatrick 	TAILQ_INIT(&ap->ap_ccb_free);
5247acdc34dSpatrick 	TAILQ_INIT(&ap->ap_ccb_pending);
5257acdc34dSpatrick 	mtx_init(&ap->ap_ccb_mtx, IPL_BIO);
5267acdc34dSpatrick 
5277acdc34dSpatrick 	/* Disable port interrupts */
5287acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IE, 0);
5297acdc34dSpatrick 
5307acdc34dSpatrick 	/* Sec 10.1.2 - deinitialise port if it is already running */
5317acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD);
5327acdc34dSpatrick 	if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
5337acdc34dSpatrick 	    AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
5347acdc34dSpatrick 	    ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
5357acdc34dSpatrick 		int r;
5367acdc34dSpatrick 
5377acdc34dSpatrick 		r = ahci_port_stop(ap, 1);
5387acdc34dSpatrick 		if (r) {
5397acdc34dSpatrick 			printf("%s: unable to disable %s, ignoring port %d\n",
5407acdc34dSpatrick 			    DEVNAME(sc), r == 2 ? "CR" : "FR", port);
5417acdc34dSpatrick 			rc = ENXIO;
5427acdc34dSpatrick 			goto freeport;
5437acdc34dSpatrick 		}
5447acdc34dSpatrick 
5457acdc34dSpatrick 		/* Write DET to zero */
5467acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
5477acdc34dSpatrick 	}
5487acdc34dSpatrick 
5497acdc34dSpatrick 	/* Allocate RFIS */
5507acdc34dSpatrick 	ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
5517acdc34dSpatrick 	if (ap->ap_dmamem_rfis == NULL)
5527acdc34dSpatrick 		goto nomem;
5537acdc34dSpatrick 
5547acdc34dSpatrick 	/* Setup RFIS base address */
5557acdc34dSpatrick 	ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
5567acdc34dSpatrick 	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
5577acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
5587acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
5597acdc34dSpatrick 
5607acdc34dSpatrick 	/* Enable FIS reception and activate port. */
5617acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
5627acdc34dSpatrick 	cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
5637acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
5647acdc34dSpatrick 
5657acdc34dSpatrick 	/* Check whether port activated.  Skip it if not. */
5667acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
5677acdc34dSpatrick 	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
5687acdc34dSpatrick 		rc = ENXIO;
5697acdc34dSpatrick 		goto freeport;
5707acdc34dSpatrick 	}
5717acdc34dSpatrick 
5727acdc34dSpatrick 	/* Allocate a CCB for each command slot */
5739f6fb5c7Sderaadt 	ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
5749f6fb5c7Sderaadt 	    M_DEVBUF, M_NOWAIT | M_ZERO);
5757acdc34dSpatrick 	if (ap->ap_ccbs == NULL) {
5767acdc34dSpatrick 		printf("%s: unable to allocate command list for port %d\n",
5777acdc34dSpatrick 		    DEVNAME(sc), port);
5787acdc34dSpatrick 		goto freeport;
5797acdc34dSpatrick 	}
5807acdc34dSpatrick 
5817acdc34dSpatrick 	/* Command List Structures and Command Tables */
5827acdc34dSpatrick 	ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
5837acdc34dSpatrick 	    sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
5847acdc34dSpatrick 	ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
5857acdc34dSpatrick 	    sc->sc_ncmds * sizeof(struct ahci_cmd_table));
5867acdc34dSpatrick 	if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
5877acdc34dSpatrick nomem:
5887acdc34dSpatrick 		printf("%s: unable to allocate DMA memory for port %d\n",
5897acdc34dSpatrick 		    DEVNAME(sc), port);
5907acdc34dSpatrick 		goto freeport;
5917acdc34dSpatrick 	}
5927acdc34dSpatrick 
5937acdc34dSpatrick 	/* Setup command list base address */
5947acdc34dSpatrick 	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
5957acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
5967acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
5977acdc34dSpatrick 
5987acdc34dSpatrick 	/* Split CCB allocation into CCBs and assign to command header/table */
5997acdc34dSpatrick 	hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
6007acdc34dSpatrick 	table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
6017acdc34dSpatrick 	for (i = 0; i < sc->sc_ncmds; i++) {
6027acdc34dSpatrick 		ccb = &ap->ap_ccbs[i];
6037acdc34dSpatrick 
6047acdc34dSpatrick 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
6057acdc34dSpatrick 		    (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
6067acdc34dSpatrick 		    &ccb->ccb_dmamap) != 0) {
6077acdc34dSpatrick 			printf("%s: unable to create dmamap for port %d "
6087acdc34dSpatrick 			    "ccb %d\n", DEVNAME(sc), port, i);
6097acdc34dSpatrick 			goto freeport;
6107acdc34dSpatrick 		}
6117acdc34dSpatrick 
6127acdc34dSpatrick 		ccb->ccb_slot = i;
6137acdc34dSpatrick 		ccb->ccb_port = ap;
6147acdc34dSpatrick 		ccb->ccb_cmd_hdr = &hdr[i];
6157acdc34dSpatrick 		ccb->ccb_cmd_table = &table[i];
6164d3a7877Sdlg 		htolem64(&ccb->ccb_cmd_hdr->ctba,
6174d3a7877Sdlg 		    AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
6184d3a7877Sdlg 		    ccb->ccb_slot * sizeof(struct ahci_cmd_table));
6197acdc34dSpatrick 
6207acdc34dSpatrick 		ccb->ccb_xa.fis =
6217acdc34dSpatrick 		    (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
6227acdc34dSpatrick 		ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
6237acdc34dSpatrick 		ccb->ccb_xa.tag = i;
6247acdc34dSpatrick 
6257acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_COMPLETE;
6267acdc34dSpatrick 		ahci_put_ccb(ccb);
6277acdc34dSpatrick 	}
6287acdc34dSpatrick 
6297acdc34dSpatrick 	/* grab a ccb for use during error recovery */
6307acdc34dSpatrick 	ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
6317acdc34dSpatrick 	TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry);
6327acdc34dSpatrick 	ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE;
6337acdc34dSpatrick 
6347acdc34dSpatrick 	/* Wait for ICC change to complete */
6357acdc34dSpatrick 	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
63696739e86Sjmatthew 	rc = 0;
6377acdc34dSpatrick 
63896739e86Sjmatthew freeport:
63996739e86Sjmatthew 	if (rc != 0)
64096739e86Sjmatthew 		ahci_port_free(sc, port);
64196739e86Sjmatthew reterr:
64296739e86Sjmatthew 	return (rc);
64396739e86Sjmatthew }
6447acdc34dSpatrick 
64596739e86Sjmatthew void
64696739e86Sjmatthew ahci_port_detect(struct ahci_softc *sc, u_int port)
64796739e86Sjmatthew {
64896739e86Sjmatthew 	struct ahci_port		*ap;
64996739e86Sjmatthew 	const char			*speed;
65096739e86Sjmatthew 	int				rc;
65196739e86Sjmatthew 
65296739e86Sjmatthew 	ap = sc->sc_ports[port];
65396739e86Sjmatthew 
65496739e86Sjmatthew 	rc = ahci_port_portreset_finish(ap, 1);
6557acdc34dSpatrick 	switch (rc) {
6567acdc34dSpatrick 	case ENODEV:
6577acdc34dSpatrick 		switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
6587acdc34dSpatrick 		case AHCI_PREG_SSTS_DET_DEV_NE:
6597acdc34dSpatrick 			printf("%s: device not communicating on port %d\n",
6607acdc34dSpatrick 			    DEVNAME(sc), port);
6617acdc34dSpatrick 			break;
6627acdc34dSpatrick 		case AHCI_PREG_SSTS_DET_PHYOFFLINE:
6637acdc34dSpatrick 			printf("%s: PHY offline on port %d\n", DEVNAME(sc),
6647acdc34dSpatrick 			    port);
6657acdc34dSpatrick 			break;
6667acdc34dSpatrick 		default:
6677acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
6687acdc34dSpatrick 			    "on port %d\n", DEVNAME(sc), port);
6697acdc34dSpatrick 			break;
6707acdc34dSpatrick 		}
6717acdc34dSpatrick 		goto freeport;
6727acdc34dSpatrick 
6737acdc34dSpatrick 	case EBUSY:
6747acdc34dSpatrick 		printf("%s: device on port %d didn't come ready, "
6757acdc34dSpatrick 		    "TFD: 0x%b\n", DEVNAME(sc), port,
6767acdc34dSpatrick 		    ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
6777acdc34dSpatrick 
6787acdc34dSpatrick 		/* Try a soft reset to clear busy */
6797acdc34dSpatrick 		rc = ahci_port_softreset(ap);
6807acdc34dSpatrick 		if (rc) {
6817acdc34dSpatrick 			printf("%s: unable to communicate "
6827acdc34dSpatrick 			    "with device on port %d\n", DEVNAME(sc), port);
6837acdc34dSpatrick 			goto freeport;
6847acdc34dSpatrick 		}
6857acdc34dSpatrick 		break;
6867acdc34dSpatrick 
6877acdc34dSpatrick 	default:
6887acdc34dSpatrick 		break;
6897acdc34dSpatrick 	}
6907acdc34dSpatrick 
6917acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
6927acdc34dSpatrick 	    DEVNAME(sc), port, rc);
6937acdc34dSpatrick 
69417fb629fSbrynet 	/* Read current link speed */
69517fb629fSbrynet 	switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) {
69617fb629fSbrynet 	case AHCI_PREG_SSTS_SPD_GEN1:
6979cd47e4cSmpi 		speed = "1.5Gb/s";
69817fb629fSbrynet 		break;
69917fb629fSbrynet 	case AHCI_PREG_SSTS_SPD_GEN2:
7009cd47e4cSmpi 		speed = "3.0Gb/s";
70117fb629fSbrynet 		break;
70217fb629fSbrynet 	case AHCI_PREG_SSTS_SPD_GEN3:
7039cd47e4cSmpi 		speed = "6.0Gb/s";
70417fb629fSbrynet 		break;
70517fb629fSbrynet 	default:
70617fb629fSbrynet 		speed = NULL;
70717fb629fSbrynet 		break;
70817fb629fSbrynet 	}
70917fb629fSbrynet 	if (speed != NULL)
7109cd47e4cSmpi 		printf("%s: port %d: %s\n", PORTNAME(ap), port, speed);
71117fb629fSbrynet 
7127acdc34dSpatrick 	/* Enable command transfers on port */
7137acdc34dSpatrick 	if (ahci_port_start(ap, 0)) {
7147acdc34dSpatrick 		printf("%s: failed to start command DMA on port %d, "
7157acdc34dSpatrick 		    "disabling\n", DEVNAME(sc), port);
7167acdc34dSpatrick 		rc = ENXIO;	/* couldn't start port */
7177acdc34dSpatrick 	}
7187acdc34dSpatrick 
7197acdc34dSpatrick 	/* Flush interrupts for port */
7207acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
7217acdc34dSpatrick 	ahci_write(sc, AHCI_REG_IS, 1 << port);
7227acdc34dSpatrick 
7237acdc34dSpatrick 	ahci_enable_interrupts(ap);
7247acdc34dSpatrick freeport:
7257acdc34dSpatrick 	if (rc != 0)
7267acdc34dSpatrick 		ahci_port_free(sc, port);
7277acdc34dSpatrick }
7287acdc34dSpatrick 
7297acdc34dSpatrick void
7307acdc34dSpatrick ahci_port_free(struct ahci_softc *sc, u_int port)
7317acdc34dSpatrick {
7327acdc34dSpatrick 	struct ahci_port		*ap = sc->sc_ports[port];
7337acdc34dSpatrick 	struct ahci_ccb			*ccb;
7347acdc34dSpatrick 
7357acdc34dSpatrick 	/* Ensure port is disabled and its interrupts are flushed */
7367acdc34dSpatrick 	if (ap->ap_sc) {
7377acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_CMD, 0);
7387acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IE, 0);
7397acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
7407acdc34dSpatrick 		ahci_write(sc, AHCI_REG_IS, 1 << port);
7417acdc34dSpatrick 	}
7427acdc34dSpatrick 
7437acdc34dSpatrick 	if (ap->ap_ccb_err)
7447acdc34dSpatrick 		ahci_put_ccb(ap->ap_ccb_err);
7457acdc34dSpatrick 
7467acdc34dSpatrick 	if (ap->ap_ccbs) {
7477acdc34dSpatrick 		while ((ccb = ahci_get_ccb(ap)) != NULL)
7487acdc34dSpatrick 			bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
7491f559dcaSderaadt 		free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb));
7507acdc34dSpatrick 	}
7517acdc34dSpatrick 
7527acdc34dSpatrick 	if (ap->ap_dmamem_cmd_list)
7537acdc34dSpatrick 		ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
7547acdc34dSpatrick 	if (ap->ap_dmamem_rfis)
7557acdc34dSpatrick 		ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
7567acdc34dSpatrick 	if (ap->ap_dmamem_cmd_table)
7577acdc34dSpatrick 		ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
7587acdc34dSpatrick 	if (ap->ap_err_scratch)
7597acdc34dSpatrick 		dma_free(ap->ap_err_scratch, DEV_BSIZE);
7607acdc34dSpatrick 
7617acdc34dSpatrick 	/* bus_space(9) says we dont free the subregions handle */
7627acdc34dSpatrick 
7631f559dcaSderaadt 	free(ap, M_DEVBUF, sizeof(*ap));
7647acdc34dSpatrick 	sc->sc_ports[port] = NULL;
7657acdc34dSpatrick }
7667acdc34dSpatrick 
7677acdc34dSpatrick int
7687acdc34dSpatrick ahci_port_init(struct ahci_softc *sc, u_int port)
7697acdc34dSpatrick {
7707acdc34dSpatrick 	struct ahci_port		*ap;
7717acdc34dSpatrick 	u_int64_t			dva;
7727acdc34dSpatrick 	u_int32_t			cmd;
7737acdc34dSpatrick 	int				rc = ENOMEM;
7747acdc34dSpatrick 
7757acdc34dSpatrick 	ap = sc->sc_ports[port];
7767acdc34dSpatrick #ifdef AHCI_DEBUG
7777acdc34dSpatrick 	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
7787acdc34dSpatrick 	    DEVNAME(sc), port);
7797acdc34dSpatrick #endif
7807acdc34dSpatrick 
7817acdc34dSpatrick 	/* Disable port interrupts */
7827acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IE, 0);
7837acdc34dSpatrick 
7847acdc34dSpatrick 	/* Sec 10.1.2 - deinitialise port if it is already running */
7857acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD);
7867acdc34dSpatrick 	if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
7877acdc34dSpatrick 	    AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
7887acdc34dSpatrick 	    ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
7897acdc34dSpatrick 		int r;
7907acdc34dSpatrick 
7917acdc34dSpatrick 		r = ahci_port_stop(ap, 1);
7927acdc34dSpatrick 		if (r) {
7937acdc34dSpatrick 			printf("%s: unable to disable %s, ignoring port %d\n",
7947acdc34dSpatrick 			    DEVNAME(sc), r == 2 ? "CR" : "FR", port);
7957acdc34dSpatrick 			rc = ENXIO;
7967acdc34dSpatrick 			goto reterr;
7977acdc34dSpatrick 		}
7987acdc34dSpatrick 
7997acdc34dSpatrick 		/* Write DET to zero */
8007acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
8017acdc34dSpatrick 	}
8027acdc34dSpatrick 
8037acdc34dSpatrick 	/* Setup RFIS base address */
8047acdc34dSpatrick 	ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
8057acdc34dSpatrick 	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
8067acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
8077acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
8087acdc34dSpatrick 
8097acdc34dSpatrick 	/* Enable FIS reception and activate port. */
8107acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
8117acdc34dSpatrick 	cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
8127acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
8137acdc34dSpatrick 
8147acdc34dSpatrick 	/* Check whether port activated.  Skip it if not. */
8157acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
8167acdc34dSpatrick 	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
8177acdc34dSpatrick 		rc = ENXIO;
8187acdc34dSpatrick 		goto reterr;
8197acdc34dSpatrick 	}
8207acdc34dSpatrick 
8217acdc34dSpatrick 	/* Setup command list base address */
8227acdc34dSpatrick 	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
8237acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
8247acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
8257acdc34dSpatrick 
8267acdc34dSpatrick 	/* Wait for ICC change to complete */
8277acdc34dSpatrick 	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
8287acdc34dSpatrick 
8297acdc34dSpatrick 	/* Reset port */
8307acdc34dSpatrick 	rc = ahci_port_portreset(ap, 1);
8317acdc34dSpatrick 	switch (rc) {
8327acdc34dSpatrick 	case ENODEV:
8337acdc34dSpatrick 		switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
8347acdc34dSpatrick 		case AHCI_PREG_SSTS_DET_DEV_NE:
8357acdc34dSpatrick 			printf("%s: device not communicating on port %d\n",
8367acdc34dSpatrick 			    DEVNAME(sc), port);
8377acdc34dSpatrick 			break;
8387acdc34dSpatrick 		case AHCI_PREG_SSTS_DET_PHYOFFLINE:
8397acdc34dSpatrick 			printf("%s: PHY offline on port %d\n", DEVNAME(sc),
8407acdc34dSpatrick 			    port);
8417acdc34dSpatrick 			break;
8427acdc34dSpatrick 		default:
8437acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
8447acdc34dSpatrick 			    "on port %d\n", DEVNAME(sc), port);
8457acdc34dSpatrick 			break;
8467acdc34dSpatrick 		}
8477acdc34dSpatrick 		goto reterr;
8487acdc34dSpatrick 
8497acdc34dSpatrick 	case EBUSY:
8507acdc34dSpatrick 		printf("%s: device on port %d didn't come ready, "
8517acdc34dSpatrick 		    "TFD: 0x%b\n", DEVNAME(sc), port,
8527acdc34dSpatrick 		    ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
8537acdc34dSpatrick 
8547acdc34dSpatrick 		/* Try a soft reset to clear busy */
8557acdc34dSpatrick 		rc = ahci_port_softreset(ap);
8567acdc34dSpatrick 		if (rc) {
8577acdc34dSpatrick 			printf("%s: unable to communicate "
8587acdc34dSpatrick 			    "with device on port %d\n", DEVNAME(sc), port);
8597acdc34dSpatrick 			goto reterr;
8607acdc34dSpatrick 		}
8617acdc34dSpatrick 		break;
8627acdc34dSpatrick 
8637acdc34dSpatrick 	default:
8647acdc34dSpatrick 		break;
8657acdc34dSpatrick 	}
8667acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
8677acdc34dSpatrick 	    DEVNAME(sc), port);
8687acdc34dSpatrick 
8697acdc34dSpatrick 	if (ap->ap_pmp_ports > 0) {
8707acdc34dSpatrick 		int p;
8717acdc34dSpatrick 
8727acdc34dSpatrick 		for (p = 0; p < ap->ap_pmp_ports; p++) {
8737acdc34dSpatrick 			int sig;
8747acdc34dSpatrick 
8757acdc34dSpatrick 			/* might need to do a portreset first here? */
8767acdc34dSpatrick 
8777acdc34dSpatrick 			/* softreset the port */
8787acdc34dSpatrick 			if (ahci_pmp_port_softreset(ap, p)) {
8797acdc34dSpatrick 				printf("%s.%d: unable to probe PMP port due to"
8807acdc34dSpatrick 				    " softreset failure\n", PORTNAME(ap), p);
8817acdc34dSpatrick 				continue;
8827acdc34dSpatrick 			}
8837acdc34dSpatrick 
8847acdc34dSpatrick 			sig = ahci_port_signature(ap);
8857acdc34dSpatrick 			printf("%s.%d: port signature returned %d\n",
8867acdc34dSpatrick 			    PORTNAME(ap), p, sig);
8877acdc34dSpatrick 		}
8887acdc34dSpatrick 	}
8897acdc34dSpatrick 
8907acdc34dSpatrick 	/* Enable command transfers on port */
8917acdc34dSpatrick 	if (ahci_port_start(ap, 0)) {
8927acdc34dSpatrick 		printf("%s: failed to start command DMA on port %d, "
8937acdc34dSpatrick 		    "disabling\n", DEVNAME(sc), port);
8947acdc34dSpatrick 		rc = ENXIO;	/* couldn't start port */
8957acdc34dSpatrick 	}
8967acdc34dSpatrick 
8977acdc34dSpatrick 	/* Flush interrupts for port */
8987acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
8997acdc34dSpatrick 	ahci_write(sc, AHCI_REG_IS, 1 << port);
9007acdc34dSpatrick 
9017acdc34dSpatrick 	ahci_enable_interrupts(ap);
9027acdc34dSpatrick 
9037acdc34dSpatrick reterr:
9047acdc34dSpatrick 	return (rc);
9057acdc34dSpatrick }
9067acdc34dSpatrick 
9077acdc34dSpatrick int
90893c0f120Sjsg ahci_default_port_start(struct ahci_port *ap, int fre_only)
9097acdc34dSpatrick {
9107acdc34dSpatrick 	u_int32_t			r;
9117acdc34dSpatrick 
9127acdc34dSpatrick 	/* Turn on FRE (and ST) */
9137acdc34dSpatrick 	r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
9147acdc34dSpatrick 	r |= AHCI_PREG_CMD_FRE;
9157acdc34dSpatrick 	if (!fre_only)
9167acdc34dSpatrick 		r |= AHCI_PREG_CMD_ST;
9177acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, r);
9187acdc34dSpatrick 
9197acdc34dSpatrick #ifdef AHCI_COALESCE
9207acdc34dSpatrick 	/* (Re-)enable coalescing on the port. */
9217acdc34dSpatrick 	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
9227acdc34dSpatrick 		ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
9237acdc34dSpatrick 		ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
9247acdc34dSpatrick 		    ap->ap_sc->sc_ccc_ports_cur);
9257acdc34dSpatrick 	}
9267acdc34dSpatrick #endif
9277acdc34dSpatrick 
9287acdc34dSpatrick 	return (0);
9297acdc34dSpatrick }
9307acdc34dSpatrick 
9317acdc34dSpatrick int
9327acdc34dSpatrick ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
9337acdc34dSpatrick {
9347acdc34dSpatrick 	u_int32_t			r;
9357acdc34dSpatrick 
9367acdc34dSpatrick #ifdef AHCI_COALESCE
9377acdc34dSpatrick 	/* Disable coalescing on the port while it is stopped. */
9387acdc34dSpatrick 	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
9397acdc34dSpatrick 		ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
9407acdc34dSpatrick 		ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
9417acdc34dSpatrick 		    ap->ap_sc->sc_ccc_ports_cur);
9427acdc34dSpatrick 	}
9437acdc34dSpatrick #endif
9447acdc34dSpatrick 
9457acdc34dSpatrick 	/* Turn off ST (and FRE) */
9467acdc34dSpatrick 	r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
9477acdc34dSpatrick 	r &= ~AHCI_PREG_CMD_ST;
9487acdc34dSpatrick 	if (stop_fis_rx)
9497acdc34dSpatrick 		r &= ~AHCI_PREG_CMD_FRE;
9507acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, r);
9517acdc34dSpatrick 
9527acdc34dSpatrick 	/* Wait for CR to go off */
9537acdc34dSpatrick 	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
9547acdc34dSpatrick 		return (1);
9557acdc34dSpatrick 
9567acdc34dSpatrick 	/* Wait for FR to go off */
9577acdc34dSpatrick 	if (stop_fis_rx &&
9587acdc34dSpatrick 	    ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1))
9597acdc34dSpatrick 		return (2);
9607acdc34dSpatrick 
9617acdc34dSpatrick 	return (0);
9627acdc34dSpatrick }
9637acdc34dSpatrick 
9647acdc34dSpatrick /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
9657acdc34dSpatrick int
9667acdc34dSpatrick ahci_port_clo(struct ahci_port *ap)
9677acdc34dSpatrick {
9687acdc34dSpatrick 	struct ahci_softc		*sc = ap->ap_sc;
9697acdc34dSpatrick 	u_int32_t			cmd;
9707acdc34dSpatrick 
9717acdc34dSpatrick 	/* Only attempt CLO if supported by controller */
9727acdc34dSpatrick 	if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
9737acdc34dSpatrick 		return (1);
9747acdc34dSpatrick 
9757acdc34dSpatrick 	/* Issue CLO */
9767acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
9777acdc34dSpatrick #ifdef DIAGNOSTIC
9787acdc34dSpatrick 	if (ISSET(cmd, AHCI_PREG_CMD_ST))
9797acdc34dSpatrick 		printf("%s: CLO requested while port running\n", PORTNAME(ap));
9807acdc34dSpatrick #endif
9817acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
9827acdc34dSpatrick 
9837acdc34dSpatrick 	/* Wait for completion */
9847acdc34dSpatrick 	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) {
9857acdc34dSpatrick 		printf("%s: CLO did not complete\n", PORTNAME(ap));
9867acdc34dSpatrick 		return (1);
9877acdc34dSpatrick 	}
9887acdc34dSpatrick 
9897acdc34dSpatrick 	return (0);
9907acdc34dSpatrick }
9917acdc34dSpatrick 
9927acdc34dSpatrick /* AHCI soft reset, Section 10.4.1 */
9937acdc34dSpatrick int
9947acdc34dSpatrick ahci_port_softreset(struct ahci_port *ap)
9957acdc34dSpatrick {
9967acdc34dSpatrick 	struct ahci_ccb			*ccb = NULL;
9977acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot;
9987acdc34dSpatrick 	u_int8_t			*fis;
9997acdc34dSpatrick 	int				s, rc = EIO, oldstate;
10007acdc34dSpatrick 	u_int32_t			cmd;
10017acdc34dSpatrick 
10027acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
10037acdc34dSpatrick 
10047acdc34dSpatrick 	s = splbio();
10057acdc34dSpatrick 	oldstate = ap->ap_state;
10067acdc34dSpatrick 	ap->ap_state = AP_S_ERROR_RECOVERY;
10077acdc34dSpatrick 
10087acdc34dSpatrick 	/* Save previous command register state */
10097acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
10107acdc34dSpatrick 
10117acdc34dSpatrick 	/* Idle port */
10127acdc34dSpatrick 	if (ahci_port_stop(ap, 0)) {
10137acdc34dSpatrick 		printf("%s: failed to stop port, cannot softreset\n",
10147acdc34dSpatrick 		    PORTNAME(ap));
10157acdc34dSpatrick 		goto err;
10167acdc34dSpatrick 	}
10177acdc34dSpatrick 
10187acdc34dSpatrick 	/* Request CLO if device appears hung */
10197acdc34dSpatrick 	if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
10207acdc34dSpatrick 	    AHCI_PREG_TFD_STS_DRQ))
10217acdc34dSpatrick 		ahci_port_clo(ap);
10227acdc34dSpatrick 
10237acdc34dSpatrick 	/* Clear port errors to permit TFD transfer */
10247acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
10257acdc34dSpatrick 
10267acdc34dSpatrick 	/* Restart port */
10277acdc34dSpatrick 	if (ahci_port_start(ap, 0)) {
10287acdc34dSpatrick 		printf("%s: failed to start port, cannot softreset\n",
10297acdc34dSpatrick 		    PORTNAME(ap));
10307acdc34dSpatrick 		goto err;
10317acdc34dSpatrick 	}
10327acdc34dSpatrick 
10337acdc34dSpatrick 	/* Check whether CLO worked */
10347acdc34dSpatrick 	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
10357acdc34dSpatrick 	    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
10367acdc34dSpatrick 		printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
10377acdc34dSpatrick 		    ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
10387acdc34dSpatrick 		    ? "failed" : "unsupported");
10397acdc34dSpatrick 		rc = EBUSY;
10407acdc34dSpatrick 		goto err;
10417acdc34dSpatrick 	}
10427acdc34dSpatrick 
10437acdc34dSpatrick 	/* Prep first D2H command with SRST feature & clear busy/reset flags */
10447acdc34dSpatrick 	ccb = ahci_get_err_ccb(ap);
10457acdc34dSpatrick 	cmd_slot = ccb->ccb_cmd_hdr;
1046414fa24dSdlg 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
10477acdc34dSpatrick 
10487acdc34dSpatrick 	fis = ccb->ccb_cmd_table->cfis;
10497acdc34dSpatrick 	fis[0] = ATA_FIS_TYPE_H2D;
10507acdc34dSpatrick 	fis[15] = ATA_FIS_CONTROL_SRST;
10517acdc34dSpatrick 
10527acdc34dSpatrick 	cmd_slot->prdtl = 0;
10533f8c2449Sdlg 	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
10543f8c2449Sdlg 	    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
10553f8c2449Sdlg 	    AHCI_CMD_LIST_FLAG_W);
10567acdc34dSpatrick 
10577acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PENDING;
10587acdc34dSpatrick 	if (ahci_poll(ccb, 1000, NULL) != 0)
10597acdc34dSpatrick 		goto err;
10607acdc34dSpatrick 
10617acdc34dSpatrick 	/* Prep second D2H command to read status and complete reset sequence */
10627acdc34dSpatrick 	fis[0] = ATA_FIS_TYPE_H2D;
10637acdc34dSpatrick 	fis[15] = 0;
10647acdc34dSpatrick 
10657acdc34dSpatrick 	cmd_slot->prdtl = 0;
10663f8c2449Sdlg 	htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W);
10677acdc34dSpatrick 
10687acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PENDING;
10697acdc34dSpatrick 	if (ahci_poll(ccb, 1000, NULL) != 0)
10707acdc34dSpatrick 		goto err;
10717acdc34dSpatrick 
10727acdc34dSpatrick 	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
10737acdc34dSpatrick 	    AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) {
10747acdc34dSpatrick 		printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
10757acdc34dSpatrick 		    PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
10767acdc34dSpatrick 		    AHCI_PFMT_TFD_STS);
10777acdc34dSpatrick 		rc = EBUSY;
10787acdc34dSpatrick 		goto err;
10797acdc34dSpatrick 	}
10807acdc34dSpatrick 
10817acdc34dSpatrick 	rc = 0;
10827acdc34dSpatrick err:
10837acdc34dSpatrick 	if (ccb != NULL) {
10847acdc34dSpatrick 		/* Abort our command, if it failed, by stopping command DMA. */
10857acdc34dSpatrick 		if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
10867acdc34dSpatrick 			printf("%s: stopping the port, softreset slot %d was "
10877acdc34dSpatrick 			    "still active.\n", PORTNAME(ap), ccb->ccb_slot);
10887acdc34dSpatrick 			ahci_port_stop(ap, 0);
10897acdc34dSpatrick 		}
10907acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_ERROR;
10917acdc34dSpatrick 		ahci_put_err_ccb(ccb);
10927acdc34dSpatrick 	}
10937acdc34dSpatrick 
10947acdc34dSpatrick 	/* Restore saved CMD register state */
10957acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
10967acdc34dSpatrick 	ap->ap_state = oldstate;
10977acdc34dSpatrick 
10987acdc34dSpatrick 	splx(s);
10997acdc34dSpatrick 
11007acdc34dSpatrick 	return (rc);
11017acdc34dSpatrick }
11027acdc34dSpatrick 
11037acdc34dSpatrick int
11047acdc34dSpatrick ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
11057acdc34dSpatrick {
11067acdc34dSpatrick 	struct ahci_ccb		*ccb = NULL;
11077acdc34dSpatrick 	u_int32_t		data;
11087acdc34dSpatrick 	int			count;
11097acdc34dSpatrick 	int			rc;
11107acdc34dSpatrick 	int			s;
11117acdc34dSpatrick 	struct ahci_cmd_hdr	*cmd_slot;
11127acdc34dSpatrick 	u_int8_t		*fis;
11137acdc34dSpatrick 
11147acdc34dSpatrick 	s = splbio();
11157acdc34dSpatrick 	/* ignore spurious IFS errors while resetting */
11167acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
11177acdc34dSpatrick 	ap->ap_pmp_ignore_ifs = 1;
11187acdc34dSpatrick 
11197acdc34dSpatrick 	count = 2;
11207acdc34dSpatrick 	rc = 0;
11217acdc34dSpatrick 	do {
11227acdc34dSpatrick 		if (ccb != NULL) {
11237acdc34dSpatrick 			ahci_put_pmp_ccb(ccb);
11247acdc34dSpatrick 			ccb = NULL;
11257acdc34dSpatrick 		}
11267acdc34dSpatrick 
11277acdc34dSpatrick 		if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
11287acdc34dSpatrick 			printf("%s.%d: unable to clear PHY status\n",
11297acdc34dSpatrick 			    PORTNAME(ap), pmp_port);
11307acdc34dSpatrick 		}
11317acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
11327acdc34dSpatrick 		/* maybe don't do this on the first loop: */
11337acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
11347acdc34dSpatrick 		ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
11357acdc34dSpatrick 
11367acdc34dSpatrick 		/* send first softreset FIS */
1137189d7e68Skrw 		ccb = ahci_get_pmp_ccb(ap);	/* Always returns non-NULL. */
11387acdc34dSpatrick 		cmd_slot = ccb->ccb_cmd_hdr;
1139414fa24dSdlg 		memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
11407acdc34dSpatrick 
11417acdc34dSpatrick 		fis = ccb->ccb_cmd_table->cfis;
11427acdc34dSpatrick 		fis[0] = ATA_FIS_TYPE_H2D;
11437acdc34dSpatrick 		fis[1] = pmp_port;
11447acdc34dSpatrick 		fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
11457acdc34dSpatrick 
11467acdc34dSpatrick 		cmd_slot->prdtl = 0;
11473f8c2449Sdlg 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
11483f8c2449Sdlg 		    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
11493f8c2449Sdlg 		    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
11507acdc34dSpatrick 
11517acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_PENDING;
11527acdc34dSpatrick 
11537acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
11547acdc34dSpatrick 		    PORTNAME(ap), pmp_port);
11557acdc34dSpatrick 		if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
11567acdc34dSpatrick 			printf("%s.%d: PMP port softreset cmd failed\n",
11577acdc34dSpatrick 			    PORTNAME(ap), pmp_port);
11587acdc34dSpatrick 			rc = EBUSY;
11597acdc34dSpatrick 			if (count > 0) {
11607acdc34dSpatrick 				/* probably delay a while to allow
11617acdc34dSpatrick 				 * it to settle down?
11627acdc34dSpatrick 				 */
11637acdc34dSpatrick 			}
11647acdc34dSpatrick 			continue;
11657acdc34dSpatrick 		}
11667acdc34dSpatrick 
11677acdc34dSpatrick 		/* send signature FIS */
1168414fa24dSdlg 		memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
11697acdc34dSpatrick 		fis[0] = ATA_FIS_TYPE_H2D;
11707acdc34dSpatrick 		fis[1] = pmp_port;
11717acdc34dSpatrick 		fis[15] = ATA_FIS_CONTROL_4BIT;
11727acdc34dSpatrick 
11737acdc34dSpatrick 		cmd_slot->prdtl = 0;
11743f8c2449Sdlg 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
11753f8c2449Sdlg 		    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
11767acdc34dSpatrick 
11777acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
11787acdc34dSpatrick 		    PORTNAME(ap), pmp_port);
11797acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_PENDING;
11807acdc34dSpatrick 		if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
11817acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
11827acdc34dSpatrick 			    "failed\n", PORTNAME(ap), pmp_port);
11837acdc34dSpatrick 			rc = EBUSY;
11847acdc34dSpatrick 			if (count > 0) {
11857acdc34dSpatrick 				/* sleep a while? */
11867acdc34dSpatrick 			}
11877acdc34dSpatrick 			continue;
11887acdc34dSpatrick 		}
11897acdc34dSpatrick 
11907acdc34dSpatrick 		fis[15] = 0;
11917acdc34dSpatrick 		break;
11927acdc34dSpatrick 	} while (count--);
11937acdc34dSpatrick 
11947acdc34dSpatrick 	if (ccb != NULL) {
11957acdc34dSpatrick 		ahci_put_pmp_ccb(ccb);
11967acdc34dSpatrick 		ccb = NULL;
11977acdc34dSpatrick 	}
11987acdc34dSpatrick 
11997acdc34dSpatrick 	/* clean up a bit */
12007acdc34dSpatrick 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
12017acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SERR, -1);
12027acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
12037acdc34dSpatrick 	ap->ap_pmp_ignore_ifs = 0;
12047acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
12057acdc34dSpatrick 	splx(s);
12067acdc34dSpatrick 
12077acdc34dSpatrick 	return (rc);
12087acdc34dSpatrick }
12097acdc34dSpatrick 
12107acdc34dSpatrick int
12117acdc34dSpatrick ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
12127acdc34dSpatrick {
12137acdc34dSpatrick 	int sig;
12147acdc34dSpatrick 
12157acdc34dSpatrick 	ap->ap_state = AP_S_PMP_PORT_PROBE;
12167acdc34dSpatrick 
12177acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
12187acdc34dSpatrick 	    pmp_port);
12197acdc34dSpatrick 	if (ahci_pmp_port_portreset(ap, pmp_port)) {
12207acdc34dSpatrick 		printf("%s.%d: unable to probe PMP port; portreset failed\n",
12217acdc34dSpatrick 		    PORTNAME(ap), pmp_port);
12227acdc34dSpatrick 		ap->ap_state = AP_S_NORMAL;
12237acdc34dSpatrick 		return (ATA_PORT_T_NONE);
12247acdc34dSpatrick 	}
12257acdc34dSpatrick 
12267acdc34dSpatrick 	if (ahci_pmp_port_softreset(ap, pmp_port)) {
12277acdc34dSpatrick 		printf("%s.%d: unable to probe PMP port due to softreset "
12287acdc34dSpatrick 		    "failure\n", PORTNAME(ap), pmp_port);
12297acdc34dSpatrick 		ap->ap_state = AP_S_NORMAL;
12307acdc34dSpatrick 		return (ATA_PORT_T_NONE);
12317acdc34dSpatrick 	}
12327acdc34dSpatrick 
12337acdc34dSpatrick 	sig = ahci_port_signature(ap);
12347acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
12357acdc34dSpatrick 	    PORTNAME(ap), pmp_port, sig);
12367acdc34dSpatrick 	ap->ap_state = AP_S_NORMAL;
12377acdc34dSpatrick 	return (sig);
12387acdc34dSpatrick }
12397acdc34dSpatrick 
12407acdc34dSpatrick 
12417acdc34dSpatrick void
12427acdc34dSpatrick ahci_flush_tfd(struct ahci_port *ap)
12437acdc34dSpatrick {
12447acdc34dSpatrick 	u_int32_t r;
12457acdc34dSpatrick 
12467acdc34dSpatrick 	r = ahci_pread(ap, AHCI_PREG_SERR);
12477acdc34dSpatrick 	if (r & AHCI_PREG_SERR_DIAG_X)
12487acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
12497acdc34dSpatrick }
12507acdc34dSpatrick 
12517acdc34dSpatrick u_int32_t
12527acdc34dSpatrick ahci_active_mask(struct ahci_port *ap)
12537acdc34dSpatrick {
12547acdc34dSpatrick 	u_int32_t mask;
12557acdc34dSpatrick 
12567acdc34dSpatrick 	mask = ahci_pread(ap, AHCI_PREG_CI);
12577acdc34dSpatrick 	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ)
12587acdc34dSpatrick 		mask |= ahci_pread(ap, AHCI_PREG_SACT);
12597acdc34dSpatrick 	return mask;
12607acdc34dSpatrick }
12617acdc34dSpatrick 
12627acdc34dSpatrick void
12637acdc34dSpatrick ahci_pmp_probe_timeout(void *cookie)
12647acdc34dSpatrick {
12657acdc34dSpatrick 	struct ahci_ccb *ccb = cookie;
12667acdc34dSpatrick 	struct ahci_port *ap = ccb->ccb_port;
12677acdc34dSpatrick 	u_int32_t mask;
12687acdc34dSpatrick 
12697acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
12707acdc34dSpatrick 	switch (ccb->ccb_xa.state) {
12717acdc34dSpatrick 	case ATA_S_PENDING:
12727acdc34dSpatrick 		TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
12737acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_TIMEOUT;
12747acdc34dSpatrick 		break;
12757acdc34dSpatrick 
12767acdc34dSpatrick 	case ATA_S_ONCHIP:
12777acdc34dSpatrick 	case ATA_S_ERROR:  /* currently mostly here for the ATI SBx00 quirk */
12787acdc34dSpatrick 		/* clear the command on-chip */
12797acdc34dSpatrick 		KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
12807acdc34dSpatrick 		    ap->ap_sactive == 0);
12817acdc34dSpatrick 		ahci_port_stop(ap, 0);
12827acdc34dSpatrick 		ahci_port_start(ap, 0);
12837acdc34dSpatrick 
12847acdc34dSpatrick 		if (ahci_active_mask(ap) != 0) {
12857acdc34dSpatrick 			ahci_port_stop(ap, 0);
12867acdc34dSpatrick 			ahci_port_start(ap, 0);
12877acdc34dSpatrick 			mask = ahci_active_mask(ap);
12887acdc34dSpatrick 			if (mask != 0) {
12897acdc34dSpatrick 				printf("%s: ahci_pmp_probe_timeout: failed to "
12907acdc34dSpatrick 				    "clear active cmds: %08x\n", PORTNAME(ap),
12917acdc34dSpatrick 				    mask);
12927acdc34dSpatrick 			}
12937acdc34dSpatrick 		}
12947acdc34dSpatrick 
12957acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_TIMEOUT;
12967acdc34dSpatrick 		ap->ap_active &= ~(1 << ccb->ccb_slot);
12977acdc34dSpatrick 		KASSERT(ap->ap_active_cnt > 0);
12987acdc34dSpatrick 		--ap->ap_active_cnt;
129977db1dd4Skrw 		DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
130077db1dd4Skrw 		    "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
130177db1dd4Skrw 		    ap->ap_active, ap->ap_active_cnt);
13027acdc34dSpatrick 		break;
13037acdc34dSpatrick 
13047acdc34dSpatrick 	default:
13057acdc34dSpatrick 		panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
13067acdc34dSpatrick 			PORTNAME(ap), ccb->ccb_xa.state);
13077acdc34dSpatrick 	}
13087acdc34dSpatrick }
13097acdc34dSpatrick 
13107acdc34dSpatrick int
13117acdc34dSpatrick ahci_port_signature(struct ahci_port *ap)
13127acdc34dSpatrick {
13137acdc34dSpatrick 	u_int32_t sig;
13147acdc34dSpatrick 
13157acdc34dSpatrick 	sig = ahci_pread(ap, AHCI_PREG_SIG);
13167acdc34dSpatrick 	if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
13177acdc34dSpatrick 		return (ATA_PORT_T_ATAPI);
13187acdc34dSpatrick 	else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER &
13197acdc34dSpatrick 	    0xffff0000))
13207acdc34dSpatrick 		return (ATA_PORT_T_PM);
13217acdc34dSpatrick 	else
13227acdc34dSpatrick 		return (ATA_PORT_T_DISK);
13237acdc34dSpatrick }
13247acdc34dSpatrick 
13257acdc34dSpatrick int
13267acdc34dSpatrick ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
13277acdc34dSpatrick {
13287acdc34dSpatrick 	u_int32_t cmd, data;
13297acdc34dSpatrick 	int loop;
13307acdc34dSpatrick 	int rc = 1;
13317acdc34dSpatrick 	int s;
13327acdc34dSpatrick 
13337acdc34dSpatrick 	s = splbio();
13347acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
13357acdc34dSpatrick 	    pmp_port);
13367acdc34dSpatrick 
13377dbd8a00Spatrick 	/* Save previous command register state */
13387acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
13397acdc34dSpatrick 
13407acdc34dSpatrick 	/* turn off power management and disable the PHY */
13417acdc34dSpatrick 	data = AHCI_PREG_SCTL_IPM_DISABLED;
13427acdc34dSpatrick 	/* maybe add AHCI_PREG_SCTL_DET_DISABLE */
13437acdc34dSpatrick 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1))
13447acdc34dSpatrick 		goto err;
13457acdc34dSpatrick 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
13467acdc34dSpatrick 		goto err;
13477acdc34dSpatrick 	delay(10000);
13487acdc34dSpatrick 
13497acdc34dSpatrick 	/* start COMRESET */
13507acdc34dSpatrick 	data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
13517acdc34dSpatrick 	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
13527acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
13537acdc34dSpatrick 		    pmp_port);
13547acdc34dSpatrick 		data |= AHCI_PREG_SCTL_SPD_GEN1;
13557acdc34dSpatrick 	} else
13567acdc34dSpatrick 		data |= AHCI_PREG_SCTL_SPD_ANY;
13577acdc34dSpatrick 
13587acdc34dSpatrick 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
13597acdc34dSpatrick 		goto err;
13607acdc34dSpatrick 
13617acdc34dSpatrick 	/* give it a while to settle down */
13627acdc34dSpatrick 	delay(100000);
13637acdc34dSpatrick 
13647acdc34dSpatrick 	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
13657acdc34dSpatrick 		printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap),
13667acdc34dSpatrick 		    pmp_port);
13677acdc34dSpatrick 	}
13687acdc34dSpatrick 
13697acdc34dSpatrick 	/* start trying to negotiate */
13707acdc34dSpatrick 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
13717acdc34dSpatrick 	data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE;
13727acdc34dSpatrick 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
13737acdc34dSpatrick 		goto err;
13747acdc34dSpatrick 
13757acdc34dSpatrick 	/* give it a while to detect */
13767acdc34dSpatrick 	for (loop = 3; loop; --loop) {
13777acdc34dSpatrick 		if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
13787acdc34dSpatrick 			goto err;
13797acdc34dSpatrick 		if (data & AHCI_PREG_SSTS_DET)
13807acdc34dSpatrick 			break;
13817acdc34dSpatrick 		delay(100000);
13827acdc34dSpatrick 	}
13837acdc34dSpatrick 	if (loop == 0) {
13847acdc34dSpatrick 		printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port);
13857acdc34dSpatrick 		goto err;
13867acdc34dSpatrick 	}
13877acdc34dSpatrick 
13887acdc34dSpatrick 	/* give it even longer to fully negotiate */
13897acdc34dSpatrick 	for (loop = 30; loop; --loop) {
13907acdc34dSpatrick 		if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
13917acdc34dSpatrick 			goto err;
13927acdc34dSpatrick 		if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV)
13937acdc34dSpatrick 			break;
13947acdc34dSpatrick 		delay(100000);
13957acdc34dSpatrick 	}
13967acdc34dSpatrick 
13977acdc34dSpatrick 	if (loop == 0) {
13987acdc34dSpatrick 		printf("%s.%d: device is not negotiating\n", PORTNAME(ap),
13997acdc34dSpatrick 		    pmp_port);
14007acdc34dSpatrick 		goto err;
14017acdc34dSpatrick 	}
14027acdc34dSpatrick 
14037acdc34dSpatrick 	/* device detected */
14047acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
14057acdc34dSpatrick 	    pmp_port);
14067acdc34dSpatrick 
14077acdc34dSpatrick 	/* clean up a bit */
14087acdc34dSpatrick 	delay(100000);
14097acdc34dSpatrick 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
14107acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SERR, -1);
14117acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
14127acdc34dSpatrick 
14137acdc34dSpatrick 	rc = 0;
14147acdc34dSpatrick err:
14157dbd8a00Spatrick 	/* Restore preserved port state */
14167dbd8a00Spatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
14177acdc34dSpatrick 	splx(s);
14187acdc34dSpatrick 	return (rc);
14197acdc34dSpatrick }
14207acdc34dSpatrick 
14217acdc34dSpatrick /* AHCI port reset, Section 10.4.2 */
142296739e86Sjmatthew 
142396739e86Sjmatthew void
142496739e86Sjmatthew ahci_port_comreset(struct ahci_port *ap)
14257acdc34dSpatrick {
142696739e86Sjmatthew 	u_int32_t			r;
14277acdc34dSpatrick 
14287acdc34dSpatrick 	r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
14297acdc34dSpatrick 	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
14307acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
14317acdc34dSpatrick 		r |= AHCI_PREG_SCTL_SPD_GEN1;
14327acdc34dSpatrick 	} else
14337acdc34dSpatrick 		r |= AHCI_PREG_SCTL_SPD_ANY;
14347acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SCTL, r);
14357acdc34dSpatrick 	delay(10000);	/* wait at least 1ms for COMRESET to be sent */
14367acdc34dSpatrick 	r &= ~AHCI_PREG_SCTL_DET_INIT;
14377acdc34dSpatrick 	r |= AHCI_PREG_SCTL_DET_NONE;
14387acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SCTL, r);
14397acdc34dSpatrick 	delay(10000);
144096739e86Sjmatthew }
14417acdc34dSpatrick 
144296739e86Sjmatthew void
144396739e86Sjmatthew ahci_port_portreset_start(struct ahci_port *ap)
144496739e86Sjmatthew {
144596739e86Sjmatthew 	int				s;
144696739e86Sjmatthew 
144796739e86Sjmatthew 	s = splbio();
144896739e86Sjmatthew 	DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
144996739e86Sjmatthew 
145096739e86Sjmatthew 	/* Save previous command register state */
145196739e86Sjmatthew 	ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
145296739e86Sjmatthew 
145396739e86Sjmatthew 	/* Clear ST, ignoring failure */
145496739e86Sjmatthew 	ahci_port_stop(ap, 0);
145596739e86Sjmatthew 
145696739e86Sjmatthew 	/* Perform device detection */
145796739e86Sjmatthew 	ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
145896739e86Sjmatthew 	delay(10000);
145996739e86Sjmatthew 	ahci_port_comreset(ap);
146096739e86Sjmatthew 	splx(s);
146196739e86Sjmatthew }
146296739e86Sjmatthew 
146396739e86Sjmatthew int
146496739e86Sjmatthew ahci_port_portreset_poll(struct ahci_port *ap)
146596739e86Sjmatthew {
146696739e86Sjmatthew 	if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) !=
146796739e86Sjmatthew 	    AHCI_PREG_SSTS_DET_DEV)
146896739e86Sjmatthew 		return (EAGAIN);
146996739e86Sjmatthew 	return (0);
147096739e86Sjmatthew }
147196739e86Sjmatthew 
147296739e86Sjmatthew void
147396739e86Sjmatthew ahci_port_portreset_wait(struct ahci_port *ap)
147496739e86Sjmatthew {
147596739e86Sjmatthew 	int				i;
147696739e86Sjmatthew 
147796739e86Sjmatthew 	for (i = 0; i < 1000; i++) {
147896739e86Sjmatthew 		if (ahci_port_portreset_poll(ap) == 0)
147996739e86Sjmatthew 			break;
148096739e86Sjmatthew 		delay(1000);
148196739e86Sjmatthew 	}
148296739e86Sjmatthew }
148396739e86Sjmatthew 
148496739e86Sjmatthew int
148596739e86Sjmatthew ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
148696739e86Sjmatthew {
148796739e86Sjmatthew 	int				rc, s, retries = 0;
148896739e86Sjmatthew 
148996739e86Sjmatthew 	s = splbio();
149096739e86Sjmatthew retry:
149196739e86Sjmatthew 	if (ahci_port_portreset_poll(ap)) {
14927acdc34dSpatrick 		rc = ENODEV;
14937acdc34dSpatrick 		if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
14947acdc34dSpatrick 			/* this may be a port multiplier with no device
14957acdc34dSpatrick 			 * on port 0, so still do the pmp check if requested.
14967acdc34dSpatrick 			 */
14977acdc34dSpatrick 		} else {
14987acdc34dSpatrick 			goto err;
14997acdc34dSpatrick 		}
15007acdc34dSpatrick 	} else {
15017acdc34dSpatrick 		/* Clear SERR (incl X bit), so TFD can update */
15027acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
15037acdc34dSpatrick 
15047acdc34dSpatrick 		/* Wait for device to become ready */
15057acdc34dSpatrick 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
15067acdc34dSpatrick 		    AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) {
15077acdc34dSpatrick 			/* even if the device doesn't wake up, check if there's
15087acdc34dSpatrick 			 * a port multiplier there
15097acdc34dSpatrick 			 */
1510c1fa4ddeSjmatthew 			if (retries == 0) {
1511c1fa4ddeSjmatthew 				retries = 1;
151296739e86Sjmatthew 				ahci_port_comreset(ap);
151396739e86Sjmatthew 				ahci_port_portreset_wait(ap);
1514c1fa4ddeSjmatthew 				goto retry;
1515c1fa4ddeSjmatthew 			}
15167acdc34dSpatrick 			rc = EBUSY;
15177acdc34dSpatrick 		} else {
15187acdc34dSpatrick 			rc = 0;
15197acdc34dSpatrick 		}
15207acdc34dSpatrick 	}
15217acdc34dSpatrick 
15227acdc34dSpatrick 	if (pmp != 0) {
152396739e86Sjmatthew 		if (ahci_port_detect_pmp(ap)) {
152496739e86Sjmatthew 			/* reset again without pmp support */
152596739e86Sjmatthew 			pmp = 0;
152696739e86Sjmatthew 			retries = 0;
152796739e86Sjmatthew 			ahci_port_comreset(ap);
152896739e86Sjmatthew 			ahci_port_portreset_wait(ap);
152996739e86Sjmatthew 			goto retry;
15307acdc34dSpatrick 		}
15317acdc34dSpatrick 	}
15327acdc34dSpatrick 
15337acdc34dSpatrick err:
15347acdc34dSpatrick 	/* Restore preserved port state */
153596739e86Sjmatthew 	ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd);
153696739e86Sjmatthew 	ap->ap_saved_cmd = 0;
15377acdc34dSpatrick 	splx(s);
15387acdc34dSpatrick 
15397acdc34dSpatrick 	return (rc);
15407acdc34dSpatrick }
15417acdc34dSpatrick 
15427acdc34dSpatrick int
154396739e86Sjmatthew ahci_port_portreset(struct ahci_port *ap, int pmp)
154496739e86Sjmatthew {
154596739e86Sjmatthew 	ahci_port_portreset_start(ap);
154696739e86Sjmatthew 	ahci_port_portreset_wait(ap);
154796739e86Sjmatthew 	return (ahci_port_portreset_finish(ap, pmp));
154896739e86Sjmatthew }
154996739e86Sjmatthew 
155096739e86Sjmatthew int
15517acdc34dSpatrick ahci_port_detect_pmp(struct ahci_port *ap)
15527acdc34dSpatrick {
15537acdc34dSpatrick 	int				 count, pmp_rc, rc;
15547acdc34dSpatrick 	u_int32_t			 r, cmd;
15557acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot;
15567acdc34dSpatrick 	struct ahci_ccb			*ccb = NULL;
15577acdc34dSpatrick 	u_int8_t			*fis = NULL;
15587acdc34dSpatrick 
15597acdc34dSpatrick 	if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) ||
15607acdc34dSpatrick 	    !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) {
15617acdc34dSpatrick 		return 0;
15627acdc34dSpatrick 	}
15637acdc34dSpatrick 
15647acdc34dSpatrick 	rc = 0;
15657acdc34dSpatrick 	pmp_rc = 0;
15667acdc34dSpatrick 	count = 2;
15677acdc34dSpatrick 	do {
15687acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
15697acdc34dSpatrick 		    count);
15707acdc34dSpatrick 		if (ccb != NULL) {
15717acdc34dSpatrick 			ahci_put_pmp_ccb(ccb);
15727acdc34dSpatrick 			ccb = NULL;
15737acdc34dSpatrick 		}
15747acdc34dSpatrick 		ahci_port_stop(ap, 0);
15757acdc34dSpatrick 		ap->ap_state = AP_S_PMP_PROBE;
15767acdc34dSpatrick 
15777acdc34dSpatrick 		/* set PMA in cmd reg */
15787acdc34dSpatrick 		cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
15797acdc34dSpatrick 		if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
15807acdc34dSpatrick 			cmd |= AHCI_PREG_CMD_PMA;
15817acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
15827acdc34dSpatrick 		}
15837acdc34dSpatrick 
15847acdc34dSpatrick 		/* Flush errors and request CLO unconditionally,
15857acdc34dSpatrick 		 * then start the port
15867acdc34dSpatrick 		 */
15877acdc34dSpatrick 		r = ahci_pread(ap, AHCI_PREG_SERR);
15887acdc34dSpatrick 		if (r & AHCI_PREG_SERR_DIAG_X)
15897acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_SERR,
15907acdc34dSpatrick 			    AHCI_PREG_SERR_DIAG_X);
15917acdc34dSpatrick 
15927acdc34dSpatrick 		/* Request CLO */
15937acdc34dSpatrick 		ahci_port_clo(ap);
15947acdc34dSpatrick 
15957acdc34dSpatrick 		/* Clear port errors to permit TFD transfer */
15967acdc34dSpatrick 		r = ahci_pread(ap, AHCI_PREG_SERR);
15977acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, r);
15987acdc34dSpatrick 
15997acdc34dSpatrick 		/* Restart port */
16007acdc34dSpatrick 		if (ahci_port_start(ap, 0)) {
16017acdc34dSpatrick 			rc = EBUSY;
16027acdc34dSpatrick 			printf("%s: failed to start port, cannot probe PMP\n",
16037acdc34dSpatrick 			    PORTNAME(ap));
16047acdc34dSpatrick 			break;
16057acdc34dSpatrick 		}
16067acdc34dSpatrick 
16077acdc34dSpatrick 		/* Check whether CLO worked */
16087acdc34dSpatrick 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
16097acdc34dSpatrick 		    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
16107acdc34dSpatrick 			u_int32_t cap;
16117acdc34dSpatrick 
16127acdc34dSpatrick 			cap = ahci_read(ap->ap_sc, AHCI_REG_CAP);
16137acdc34dSpatrick 			printf("%s: CLO %s, need port reset\n",
16147acdc34dSpatrick 			    PORTNAME(ap),
16157acdc34dSpatrick 			    ISSET(cap, AHCI_REG_CAP_SCLO)
16167acdc34dSpatrick 			    ? "failed" : "unsupported");
16177acdc34dSpatrick 			pmp_rc = EBUSY;
16187acdc34dSpatrick 			break;
16197acdc34dSpatrick 		}
16207acdc34dSpatrick 
16217acdc34dSpatrick 		/* Prep first command with SRST feature &
16227acdc34dSpatrick 		 * clear busy/reset flags
16237acdc34dSpatrick 		 */
1624189d7e68Skrw 		ccb = ahci_get_pmp_ccb(ap);	/* Always returns non-NULL. */
16257acdc34dSpatrick 		cmd_slot = ccb->ccb_cmd_hdr;
1626414fa24dSdlg 		memset(ccb->ccb_cmd_table, 0,
16277acdc34dSpatrick 		    sizeof(struct ahci_cmd_table));
16287acdc34dSpatrick 
16297acdc34dSpatrick 		fis = ccb->ccb_cmd_table->cfis;
16307acdc34dSpatrick 		fis[0] = ATA_FIS_TYPE_H2D;
16317acdc34dSpatrick 		fis[1] = SATA_PMP_CONTROL_PORT;
16327acdc34dSpatrick 		fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
16337acdc34dSpatrick 
16347acdc34dSpatrick 		cmd_slot->prdtl = 0;
16353f8c2449Sdlg 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
16363f8c2449Sdlg 		    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
16373f8c2449Sdlg 		    AHCI_CMD_LIST_FLAG_PMP);
16387acdc34dSpatrick 
16397acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
16407acdc34dSpatrick 		    PORTNAME(ap));
16417acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_PENDING;
16427acdc34dSpatrick 		if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
16437acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
16447acdc34dSpatrick 			    PORTNAME(ap));
16457acdc34dSpatrick 			pmp_rc = EBUSY;
16467acdc34dSpatrick 			continue;
16477acdc34dSpatrick 		}
16487acdc34dSpatrick 
16497acdc34dSpatrick 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
16507acdc34dSpatrick 		    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
16517acdc34dSpatrick 			printf("%s: port busy after first PMP probe FIS\n",
16527acdc34dSpatrick 			    PORTNAME(ap));
16537acdc34dSpatrick 		}
16547acdc34dSpatrick 
16557acdc34dSpatrick 		/* clear errors in case the device
16567acdc34dSpatrick 		 * didn't reset cleanly
16577acdc34dSpatrick 		 */
16587acdc34dSpatrick 		ahci_flush_tfd(ap);
16597acdc34dSpatrick 		r = ahci_pread(ap, AHCI_PREG_SERR);
16607acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, r);
16617acdc34dSpatrick 
16627acdc34dSpatrick 		/* Prep second command to read status and
16637acdc34dSpatrick 		 * complete reset sequence
16647acdc34dSpatrick 		 */
1665414fa24dSdlg 		memset(ccb->ccb_cmd_table, 0,
16667acdc34dSpatrick 		    sizeof(struct ahci_cmd_table));
16677acdc34dSpatrick 		fis[0] = ATA_FIS_TYPE_H2D;
16687acdc34dSpatrick 		fis[1] = SATA_PMP_CONTROL_PORT;
16697acdc34dSpatrick 		fis[15] = ATA_FIS_CONTROL_4BIT;
16707acdc34dSpatrick 
16717acdc34dSpatrick 		cmd_slot->prdtl = 0;
16723f8c2449Sdlg 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
16733f8c2449Sdlg 		    AHCI_CMD_LIST_FLAG_PMP);
16747acdc34dSpatrick 
16757acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
16767acdc34dSpatrick 		    PORTNAME(ap));
16777acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_PENDING;
16787acdc34dSpatrick 		if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
16797acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
16807acdc34dSpatrick 			    "cmd failed\n", PORTNAME(ap));
16817acdc34dSpatrick 			pmp_rc = EBUSY;
16827acdc34dSpatrick 			continue;
16837acdc34dSpatrick 		}
16847acdc34dSpatrick 
16857acdc34dSpatrick 		/* apparently we need to retry at least once
16867acdc34dSpatrick 		 * to get the right signature
16877acdc34dSpatrick 		 */
16887acdc34dSpatrick 		fis[15] = 0;
16897acdc34dSpatrick 		pmp_rc = 0;
16907acdc34dSpatrick 	} while (--count);
16917acdc34dSpatrick 
16927acdc34dSpatrick 	if (ccb != NULL) {
16937acdc34dSpatrick 		ahci_put_pmp_ccb(ccb);
16947acdc34dSpatrick 		ccb = NULL;
16957acdc34dSpatrick 	}
16967acdc34dSpatrick 
16977acdc34dSpatrick 	if (ap->ap_state == AP_S_PMP_PROBE) {
16987acdc34dSpatrick 		ap->ap_state = AP_S_NORMAL;
16997acdc34dSpatrick 	}
17007acdc34dSpatrick 
17017acdc34dSpatrick 	if (pmp_rc == 0) {
17027acdc34dSpatrick 		if (ahci_port_signature(ap) != ATA_PORT_T_PM) {
17037acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
17047acdc34dSpatrick 			    PORTNAME(ap));
17057acdc34dSpatrick 			pmp_rc = EBUSY;
17067acdc34dSpatrick 		} else {
17077acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
17087acdc34dSpatrick 			    PORTNAME(ap));
17097acdc34dSpatrick 		}
17107acdc34dSpatrick 	}
17117acdc34dSpatrick 
17127acdc34dSpatrick 	if (pmp_rc == 0) {
17137acdc34dSpatrick 		if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
17147acdc34dSpatrick 			pmp_rc = EBUSY;
17157acdc34dSpatrick 		} else {
17167acdc34dSpatrick 			rc = 0;
17177acdc34dSpatrick 		}
17187acdc34dSpatrick 	}
17197acdc34dSpatrick 
17207acdc34dSpatrick 	/* if PMP detection failed, so turn off the PMA bit and
17217acdc34dSpatrick 	 * reset the port again
17227acdc34dSpatrick 	 */
17237acdc34dSpatrick 	if (pmp_rc != 0) {
17247acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
17257acdc34dSpatrick 		    "the port\n", PORTNAME(ap));
17267acdc34dSpatrick 		ahci_port_stop(ap, 0);
17277acdc34dSpatrick 		ahci_port_clo(ap);
17287acdc34dSpatrick 		cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
17297acdc34dSpatrick 		cmd &= ~AHCI_PREG_CMD_PMA;
17307acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
17317acdc34dSpatrick 
17327acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IE, 0);
17337acdc34dSpatrick 		ahci_port_stop(ap, 0);
17347acdc34dSpatrick 		if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
17357acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
17367acdc34dSpatrick 		ahci_flush_tfd(ap);
17377acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
17387acdc34dSpatrick 
17397acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, -1);
17407acdc34dSpatrick 
17417acdc34dSpatrick 		ahci_enable_interrupts(ap);
17427acdc34dSpatrick 
174396739e86Sjmatthew 		rc = pmp_rc;
17447acdc34dSpatrick 	}
17457acdc34dSpatrick 
17467acdc34dSpatrick 	return (rc);
17477acdc34dSpatrick }
17487acdc34dSpatrick 
1749b1514dfeSdlg void
1750b1514dfeSdlg ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1751b1514dfeSdlg     u_int32_t flags)
1752b1514dfeSdlg {
1753b1514dfeSdlg 	flags |= len - 1;
1754b1514dfeSdlg 
1755b1514dfeSdlg 	htolem64(&prd->dba, addr);
1756b1514dfeSdlg 	htolem32(&prd->flags, flags);
1757b1514dfeSdlg }
1758b1514dfeSdlg 
17597acdc34dSpatrick int
17607acdc34dSpatrick ahci_load_prdt(struct ahci_ccb *ccb)
17617acdc34dSpatrick {
17627acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
17637acdc34dSpatrick 	struct ahci_softc		*sc = ap->ap_sc;
17647acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
1765b1514dfeSdlg 	struct ahci_prdt		*prdt = ccb->ccb_cmd_table->prdt;
17667acdc34dSpatrick 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
17677acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot = ccb->ccb_cmd_hdr;
17687acdc34dSpatrick 	int				i, error;
17697acdc34dSpatrick 
17707acdc34dSpatrick 	if (xa->datalen == 0) {
17717acdc34dSpatrick 		ccb->ccb_cmd_hdr->prdtl = 0;
17727acdc34dSpatrick 		return (0);
17737acdc34dSpatrick 	}
17747acdc34dSpatrick 
17757acdc34dSpatrick 	error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
17767acdc34dSpatrick 	    (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
17777acdc34dSpatrick 	if (error != 0) {
17787acdc34dSpatrick 		printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
17797acdc34dSpatrick 		return (1);
17807acdc34dSpatrick 	}
17817acdc34dSpatrick 
1782b1514dfeSdlg 	for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1783b1514dfeSdlg 		ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1784b1514dfeSdlg 		    dmap->dm_segs[i].ds_len, 0);
1785b1514dfeSdlg 	}
17867acdc34dSpatrick 
1787b1514dfeSdlg 	ahci_load_prdt_seg(&prdt[i],
1788b1514dfeSdlg 	    dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1789b1514dfeSdlg 	    ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0);
17907acdc34dSpatrick 
1791b1514dfeSdlg 	htolem16(&cmd_slot->prdtl, dmap->dm_nsegs);
17927acdc34dSpatrick 
17937acdc34dSpatrick 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
17947acdc34dSpatrick 	    (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
17957acdc34dSpatrick 	    BUS_DMASYNC_PREWRITE);
17967acdc34dSpatrick 
17977acdc34dSpatrick 	return (0);
17987acdc34dSpatrick }
17997acdc34dSpatrick 
18007acdc34dSpatrick void
18017acdc34dSpatrick ahci_unload_prdt(struct ahci_ccb *ccb)
18027acdc34dSpatrick {
18037acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
18047acdc34dSpatrick 	struct ahci_softc		*sc = ap->ap_sc;
18057acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
18067acdc34dSpatrick 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
18077acdc34dSpatrick 
18087acdc34dSpatrick 	if (xa->datalen != 0) {
18097acdc34dSpatrick 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
18107acdc34dSpatrick 		    (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
18117acdc34dSpatrick 		    BUS_DMASYNC_POSTWRITE);
18127acdc34dSpatrick 
18137acdc34dSpatrick 		bus_dmamap_unload(sc->sc_dmat, dmap);
18147acdc34dSpatrick 
18157acdc34dSpatrick 		if (ccb->ccb_xa.flags & ATA_F_NCQ)
18167acdc34dSpatrick 			xa->resid = 0;
18177acdc34dSpatrick 		else
18187acdc34dSpatrick 			xa->resid = xa->datalen -
1819b1514dfeSdlg 			    lemtoh32(&ccb->ccb_cmd_hdr->prdbc);
18207acdc34dSpatrick 	}
18217acdc34dSpatrick }
18227acdc34dSpatrick 
18237acdc34dSpatrick int
18247acdc34dSpatrick ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
18257acdc34dSpatrick {
18267acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
18277acdc34dSpatrick 	int				s;
18287acdc34dSpatrick 
18297acdc34dSpatrick 	s = splbio();
18307acdc34dSpatrick 	ahci_start(ccb);
18317acdc34dSpatrick 	do {
18327acdc34dSpatrick 		if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
18337acdc34dSpatrick 		    1 << ccb->ccb_slot)) {
18347acdc34dSpatrick 			splx(s);
18357acdc34dSpatrick 			return (0);
18367acdc34dSpatrick 		}
18377acdc34dSpatrick 		if (ccb->ccb_xa.state == ATA_S_ERROR) {
18387acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
18397acdc34dSpatrick 			    PORTNAME(ap), ccb->ccb_slot);
18407acdc34dSpatrick 			/* pretend it timed out? */
18417acdc34dSpatrick 			if (timeout_fn != NULL) {
18427acdc34dSpatrick 				timeout_fn(ccb);
18437acdc34dSpatrick 			}
18447acdc34dSpatrick 			splx(s);
18457acdc34dSpatrick 			return (1);
18467acdc34dSpatrick 		}
18477acdc34dSpatrick 
18487acdc34dSpatrick 		delay(1000);
18497acdc34dSpatrick 	} while (--timeout > 0);
18507acdc34dSpatrick 
18517acdc34dSpatrick 	/* Run timeout while at splbio, otherwise ahci_intr could interfere. */
18527acdc34dSpatrick 	if (timeout_fn != NULL)
18537acdc34dSpatrick 		timeout_fn(ccb);
18547acdc34dSpatrick 
18557acdc34dSpatrick 	splx(s);
18567acdc34dSpatrick 
18577acdc34dSpatrick 	return (1);
18587acdc34dSpatrick }
18597acdc34dSpatrick 
18607acdc34dSpatrick void
18617acdc34dSpatrick ahci_start(struct ahci_ccb *ccb)
18627acdc34dSpatrick {
18637acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
18647acdc34dSpatrick 	struct ahci_softc		*sc = ap->ap_sc;
18657acdc34dSpatrick 
18667acdc34dSpatrick 	/* Zero transferred byte count before transfer */
18677acdc34dSpatrick 	ccb->ccb_cmd_hdr->prdbc = 0;
18687acdc34dSpatrick 
18697acdc34dSpatrick 	/* Sync command list entry and corresponding command table entry */
18707acdc34dSpatrick 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
18717acdc34dSpatrick 	    ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
18727acdc34dSpatrick 	    sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
18737acdc34dSpatrick 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
18747acdc34dSpatrick 	    ccb->ccb_slot * sizeof(struct ahci_cmd_table),
18757acdc34dSpatrick 	    sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
18767acdc34dSpatrick 
18777acdc34dSpatrick 	/* Prepare RFIS area for write by controller */
18787acdc34dSpatrick 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
18797acdc34dSpatrick 	    sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
18807acdc34dSpatrick 
18817acdc34dSpatrick 	if (ccb->ccb_xa.flags & ATA_F_NCQ) {
18827acdc34dSpatrick 		/* Issue NCQ commands only when there are no outstanding
18837acdc34dSpatrick 		 * standard commands. */
18847acdc34dSpatrick 		if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) ||
18857acdc34dSpatrick 		    (ap->ap_sactive != 0 &&
18867acdc34dSpatrick 		     ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
18877acdc34dSpatrick 			TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
18887acdc34dSpatrick 		} else {
18897acdc34dSpatrick 			KASSERT(ap->ap_active_cnt == 0);
18907acdc34dSpatrick 			ap->ap_sactive |= (1 << ccb->ccb_slot);
18917acdc34dSpatrick 			ccb->ccb_xa.state = ATA_S_ONCHIP;
18927acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
18937acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
18947acdc34dSpatrick 			ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
18957acdc34dSpatrick 		}
18967acdc34dSpatrick 	} else {
18977acdc34dSpatrick 		/* Wait for all NCQ commands to finish before issuing standard
18987acdc34dSpatrick 		 * command. */
18997acdc34dSpatrick 		if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
19007acdc34dSpatrick 			TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
19017acdc34dSpatrick 		else if (ap->ap_active_cnt < 2) {
19027acdc34dSpatrick 			ap->ap_active |= 1 << ccb->ccb_slot;
19037acdc34dSpatrick 			ccb->ccb_xa.state = ATA_S_ONCHIP;
19047acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
19057acdc34dSpatrick 			ap->ap_active_cnt++;
19067acdc34dSpatrick 		}
19077acdc34dSpatrick 	}
19087acdc34dSpatrick }
19097acdc34dSpatrick 
19107acdc34dSpatrick void
19117acdc34dSpatrick ahci_issue_pending_ncq_commands(struct ahci_port *ap)
19127acdc34dSpatrick {
19137acdc34dSpatrick 	struct ahci_ccb			*nextccb;
19147acdc34dSpatrick 	u_int32_t			sact_change = 0;
19157acdc34dSpatrick 
19167acdc34dSpatrick 	KASSERT(ap->ap_active_cnt == 0);
19177acdc34dSpatrick 
19187acdc34dSpatrick 	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
19197acdc34dSpatrick 	if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
19207acdc34dSpatrick 		return;
19217acdc34dSpatrick 
19227acdc34dSpatrick 	/* Start all the NCQ commands at the head of the pending list.
19237acdc34dSpatrick 	 * If a port multiplier is attached to the port, we can only
19247acdc34dSpatrick 	 * issue commands for one of its ports at a time.
19257acdc34dSpatrick 	 */
19267acdc34dSpatrick 	if (ap->ap_sactive != 0 &&
19277acdc34dSpatrick 	    ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
19287acdc34dSpatrick 		return;
19297acdc34dSpatrick 	}
19307acdc34dSpatrick 
19317acdc34dSpatrick 	ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
19327acdc34dSpatrick 	do {
19337acdc34dSpatrick 		TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
19347acdc34dSpatrick 		sact_change |= 1 << nextccb->ccb_slot;
19357acdc34dSpatrick 		nextccb->ccb_xa.state = ATA_S_ONCHIP;
19367acdc34dSpatrick 		nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
19377acdc34dSpatrick 	} while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) &&
19387acdc34dSpatrick 	    (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
19397acdc34dSpatrick 
19407acdc34dSpatrick 	ap->ap_sactive |= sact_change;
19417acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
19427acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
19437acdc34dSpatrick }
19447acdc34dSpatrick 
19457acdc34dSpatrick void
19467acdc34dSpatrick ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
19477acdc34dSpatrick {
19487acdc34dSpatrick 	struct ahci_ccb			*nextccb;
19497acdc34dSpatrick 
19507acdc34dSpatrick 	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
19517acdc34dSpatrick 	if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
19527acdc34dSpatrick 		if (last_was_ncq) {
19537acdc34dSpatrick 			KASSERT(nextccb->ccb_xa.pmp_port !=
19547acdc34dSpatrick 			    ap->ap_pmp_ncq_port);
19557acdc34dSpatrick 			/* otherwise it should have been started already */
19567acdc34dSpatrick 		} else {
19577acdc34dSpatrick 			ap->ap_active_cnt--;
19587acdc34dSpatrick 		}
19597acdc34dSpatrick 
19607acdc34dSpatrick 		/* Issue NCQ commands only when there are no outstanding
19617acdc34dSpatrick 		 * standard commands, and previous NCQ commands for other
19627acdc34dSpatrick 		 * PMP ports have finished.
19637acdc34dSpatrick 		 */
19647acdc34dSpatrick 		if (ap->ap_active == 0)
19657acdc34dSpatrick 			ahci_issue_pending_ncq_commands(ap);
19667acdc34dSpatrick 		else
19677acdc34dSpatrick 			KASSERT(ap->ap_active_cnt == 1);
19687acdc34dSpatrick 	} else if (nextccb) {
19697acdc34dSpatrick 		if (ap->ap_sactive != 0 || last_was_ncq)
19707acdc34dSpatrick 			KASSERT(ap->ap_active_cnt == 0);
19717acdc34dSpatrick 
19727acdc34dSpatrick 		/* Wait for all NCQ commands to finish before issuing standard
19737acdc34dSpatrick 		 * command. */
19747acdc34dSpatrick 		if (ap->ap_sactive != 0)
19757acdc34dSpatrick 			return;
19767acdc34dSpatrick 
19777acdc34dSpatrick 		/* Keep up to 2 standard commands on-chip at a time. */
19787acdc34dSpatrick 		do {
19797acdc34dSpatrick 			TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
19807acdc34dSpatrick 			ap->ap_active |= 1 << nextccb->ccb_slot;
19817acdc34dSpatrick 			nextccb->ccb_xa.state = ATA_S_ONCHIP;
19827acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
19837acdc34dSpatrick 			if (last_was_ncq)
19847acdc34dSpatrick 				ap->ap_active_cnt++;
19857acdc34dSpatrick 			if (ap->ap_active_cnt == 2)
19867acdc34dSpatrick 				break;
19877acdc34dSpatrick 			KASSERT(ap->ap_active_cnt == 1);
19887acdc34dSpatrick 			nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
19897acdc34dSpatrick 		} while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
19907acdc34dSpatrick 	} else if (!last_was_ncq) {
19917acdc34dSpatrick 		KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
19927acdc34dSpatrick 
19937acdc34dSpatrick 		/* Standard command finished, none waiting to start. */
19947acdc34dSpatrick 		ap->ap_active_cnt--;
19957acdc34dSpatrick 	} else {
19967acdc34dSpatrick 		KASSERT(ap->ap_active_cnt == 0);
19977acdc34dSpatrick 
19987acdc34dSpatrick 		/* NCQ command finished. */
19997acdc34dSpatrick 	}
20007acdc34dSpatrick }
20017acdc34dSpatrick 
20027acdc34dSpatrick int
20037acdc34dSpatrick ahci_intr(void *arg)
20047acdc34dSpatrick {
20057acdc34dSpatrick 	struct ahci_softc		*sc = arg;
20067acdc34dSpatrick 	u_int32_t			is, ack = 0;
20077acdc34dSpatrick 	int				port;
20087acdc34dSpatrick 
20097acdc34dSpatrick 	/* Read global interrupt status */
20107acdc34dSpatrick 	is = ahci_read(sc, AHCI_REG_IS);
20117acdc34dSpatrick 	if (is == 0 || is == 0xffffffff)
20127acdc34dSpatrick 		return (0);
20137acdc34dSpatrick 	ack = is;
20147acdc34dSpatrick 
20157acdc34dSpatrick #ifdef AHCI_COALESCE
20167acdc34dSpatrick 	/* Check coalescing interrupt first */
20177acdc34dSpatrick 	if (is & sc->sc_ccc_mask) {
20187acdc34dSpatrick 		DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
20197acdc34dSpatrick 		    DEVNAME(sc));
20207acdc34dSpatrick 		is &= ~sc->sc_ccc_mask;
20217acdc34dSpatrick 		is |= sc->sc_ccc_ports_cur;
20227acdc34dSpatrick 	}
20237acdc34dSpatrick #endif
20247acdc34dSpatrick 
20257acdc34dSpatrick 	/* Process interrupts for each port */
20267acdc34dSpatrick 	while (is) {
20277acdc34dSpatrick 		port = ffs(is) - 1;
20287acdc34dSpatrick 		if (sc->sc_ports[port])
20297acdc34dSpatrick 			ahci_port_intr(sc->sc_ports[port],
20307acdc34dSpatrick 			    AHCI_PREG_CI_ALL_SLOTS);
20317acdc34dSpatrick 		is &= ~(1 << port);
20327acdc34dSpatrick 	}
20337acdc34dSpatrick 
20347acdc34dSpatrick 	/* Finally, acknowledge global interrupt */
20357acdc34dSpatrick 	ahci_write(sc, AHCI_REG_IS, ack);
20367acdc34dSpatrick 
20377acdc34dSpatrick 	return (1);
20387acdc34dSpatrick }
20397acdc34dSpatrick 
20407acdc34dSpatrick u_int32_t
20417acdc34dSpatrick ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
20427acdc34dSpatrick {
20437acdc34dSpatrick 	struct ahci_softc		*sc = ap->ap_sc;
20447acdc34dSpatrick 	u_int32_t			is, ci_saved, ci_masked, processed = 0;
20457acdc34dSpatrick 	int				slot, need_restart = 0;
20467acdc34dSpatrick 	int				process_error = 0;
20477acdc34dSpatrick 	struct ahci_ccb			*ccb;
20487acdc34dSpatrick 	volatile u_int32_t		*active;
20497acdc34dSpatrick #ifdef DIAGNOSTIC
20507acdc34dSpatrick 	u_int32_t			tmp;
20517acdc34dSpatrick #endif
20527acdc34dSpatrick 
20537acdc34dSpatrick 	is = ahci_pread(ap, AHCI_PREG_IS);
20547acdc34dSpatrick 
20557acdc34dSpatrick 	/* Ack port interrupt only if checking all command slots. */
20567acdc34dSpatrick 	if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
20577acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, is);
20587acdc34dSpatrick 
20597acdc34dSpatrick 	if (is)
20607acdc34dSpatrick 		DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
20617acdc34dSpatrick 		    is, AHCI_PFMT_IS);
20627acdc34dSpatrick 
20637acdc34dSpatrick 	if (ap->ap_sactive) {
20647acdc34dSpatrick 		/* Active NCQ commands - use SActive instead of CI */
20657acdc34dSpatrick 		KASSERT(ap->ap_active == 0);
20667acdc34dSpatrick 		KASSERT(ap->ap_active_cnt == 0);
20677acdc34dSpatrick 		ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
20687acdc34dSpatrick 		active = &ap->ap_sactive;
20697acdc34dSpatrick 	} else {
20707acdc34dSpatrick 		/* Save CI */
20717acdc34dSpatrick 		ci_saved = ahci_pread(ap, AHCI_PREG_CI);
20727acdc34dSpatrick 		active = &ap->ap_active;
20737acdc34dSpatrick 	}
20747acdc34dSpatrick 
20757acdc34dSpatrick 	if (is & AHCI_PREG_IS_TFES) {
20767acdc34dSpatrick 		process_error = 1;
20777acdc34dSpatrick 	} else if (is & AHCI_PREG_IS_DHRS) {
20787acdc34dSpatrick 		u_int32_t tfd;
20797acdc34dSpatrick 		u_int32_t cmd;
20807acdc34dSpatrick 		u_int32_t serr;
20817acdc34dSpatrick 
20827acdc34dSpatrick 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
20837acdc34dSpatrick 		cmd = ahci_pread(ap, AHCI_PREG_CMD);
20847acdc34dSpatrick 		serr = ahci_pread(ap, AHCI_PREG_SERR);
20857acdc34dSpatrick 		if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
20867acdc34dSpatrick 		    (cmd & AHCI_PREG_CMD_CR) == 0) {
20877acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
20887acdc34dSpatrick 			    " %b, DIAG: %b\n", PORTNAME(ap), tfd,
20897acdc34dSpatrick 			    AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
20907acdc34dSpatrick 			    AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
20917acdc34dSpatrick 			    AHCI_PFMT_SERR_DIAG);
20927acdc34dSpatrick 			process_error = 1;
20937acdc34dSpatrick 		} else {
20947acdc34dSpatrick 			/* rfis copy back is in the normal execution path */
20957acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
20967acdc34dSpatrick 		}
20977acdc34dSpatrick 	}
20987acdc34dSpatrick 
20997acdc34dSpatrick 	/* Command failed.  See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
21007acdc34dSpatrick 	if (process_error) {
21017acdc34dSpatrick 		u_int32_t		tfd, serr;
21027acdc34dSpatrick 		int			err_slot;
21037acdc34dSpatrick 
21047acdc34dSpatrick 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
21057acdc34dSpatrick 		serr = ahci_pread(ap, AHCI_PREG_SERR);
21067acdc34dSpatrick 
21077acdc34dSpatrick 		if (ap->ap_sactive == 0) {
21087acdc34dSpatrick 			/* Errored slot is easy to determine from CMD. */
21097acdc34dSpatrick 			err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
21107acdc34dSpatrick 			    AHCI_PREG_CMD));
21117acdc34dSpatrick 
21127acdc34dSpatrick 			if ((ci_saved & (1 << err_slot)) == 0) {
21137acdc34dSpatrick 				/*
21147acdc34dSpatrick 				 * Hardware doesn't seem to report correct
21157acdc34dSpatrick 				 * slot number. If there's only one
21167acdc34dSpatrick 				 * outstanding command we can cope,
21177acdc34dSpatrick 				 * otherwise fail all active commands.
21187acdc34dSpatrick 				 */
21197acdc34dSpatrick 				if (ap->ap_active_cnt == 1)
21207acdc34dSpatrick 					err_slot = ffs(ap->ap_active) - 1;
21217acdc34dSpatrick 				else
21227acdc34dSpatrick 					goto failall;
21237acdc34dSpatrick 			}
21247acdc34dSpatrick 
21257acdc34dSpatrick 			ccb = &ap->ap_ccbs[err_slot];
21267acdc34dSpatrick 
21277acdc34dSpatrick 			/* Preserve received taskfile data from the RFIS. */
21287acdc34dSpatrick 			memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
21297acdc34dSpatrick 			    sizeof(struct ata_fis_d2h));
21307acdc34dSpatrick 		} else
21317acdc34dSpatrick 			err_slot = -1;	/* Must extract error from log page */
21327acdc34dSpatrick 
21337acdc34dSpatrick 		DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
21347acdc34dSpatrick 		    " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
21357acdc34dSpatrick 		    AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
21367acdc34dSpatrick 		    AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
21377acdc34dSpatrick 		    AHCI_PFMT_SERR_DIAG);
21387acdc34dSpatrick 
21397acdc34dSpatrick 		/* Turn off ST to clear CI and SACT. */
21407acdc34dSpatrick 		ahci_port_stop(ap, 0);
21417acdc34dSpatrick 		need_restart = 1;
21427acdc34dSpatrick 
21437acdc34dSpatrick 		/* Clear SERR to enable capturing new errors. */
21447acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, serr);
21457acdc34dSpatrick 
21467acdc34dSpatrick 		/* Acknowledge the interrupts we can recover from. */
21477acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
21487acdc34dSpatrick 		    AHCI_PREG_IS_IFS);
21497acdc34dSpatrick 		is = ahci_pread(ap, AHCI_PREG_IS);
21507acdc34dSpatrick 
21517acdc34dSpatrick 		/* If device hasn't cleared its busy status, try to idle it. */
21527acdc34dSpatrick 		if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
21537acdc34dSpatrick 
21547acdc34dSpatrick 			if ((ap->ap_state == AP_S_PMP_PORT_PROBE) ||
21557acdc34dSpatrick 			    (ap->ap_state == AP_S_ERROR_RECOVERY)) {
21567acdc34dSpatrick 				/* can't reset the port here, just make sure
21577acdc34dSpatrick 				 * the operation fails and the port still works.
21587acdc34dSpatrick 				 */
21597acdc34dSpatrick 			} else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
21607acdc34dSpatrick 				printf("%s: error on PMP port %d, idling "
21617acdc34dSpatrick 				    "device\n", PORTNAME(ap),
21627acdc34dSpatrick 				    ccb->ccb_xa.pmp_port);
21637acdc34dSpatrick 				if (ahci_pmp_port_softreset(ap,
21647acdc34dSpatrick 				        ccb->ccb_xa.pmp_port) == 0) {
21657acdc34dSpatrick 					printf("%s: unable to softreset port "
21667acdc34dSpatrick 					    "%d\n", PORTNAME(ap),
21677acdc34dSpatrick 					    ccb->ccb_xa.pmp_port);
21687acdc34dSpatrick 					if (ahci_pmp_port_portreset(ap,
21697acdc34dSpatrick 						ccb->ccb_xa.pmp_port)) {
21707acdc34dSpatrick 						printf("%s: failed to port "
21717acdc34dSpatrick 						    " reset %d, giving up on "
21727acdc34dSpatrick 						    "it\n", PORTNAME(ap),
21737acdc34dSpatrick 						    ccb->ccb_xa.pmp_port);
21747acdc34dSpatrick 						goto fatal;
21757acdc34dSpatrick 					}
21767acdc34dSpatrick 				}
21777acdc34dSpatrick 			} else {
21787acdc34dSpatrick 				printf("%s: attempting to idle device\n",
21797acdc34dSpatrick 				    PORTNAME(ap));
21807acdc34dSpatrick 				if (ahci_port_softreset(ap)) {
21817acdc34dSpatrick 					printf("%s: failed to soft reset "
21827acdc34dSpatrick 					    "device\n", PORTNAME(ap));
21837acdc34dSpatrick 					if (ahci_port_portreset(ap, 0)) {
21847acdc34dSpatrick 						printf("%s: failed to port "
21857acdc34dSpatrick 						    "reset device, give up on "
21867acdc34dSpatrick 						    "it\n", PORTNAME(ap));
21877acdc34dSpatrick 						goto fatal;
21887acdc34dSpatrick 					}
21897acdc34dSpatrick 				}
21907acdc34dSpatrick 			}
21917acdc34dSpatrick 
21927acdc34dSpatrick 			/* Had to reset device, can't gather extended info. */
21937acdc34dSpatrick 		} else if (ap->ap_sactive) {
21947acdc34dSpatrick 			/* Recover the NCQ error from log page 10h.
21956f1d62e9Sjmatthew 			 * We can only have queued commands active for one port
21966f1d62e9Sjmatthew 			 * at a time, so we know which device errored.
21977acdc34dSpatrick 			 */
21987acdc34dSpatrick 			ahci_port_read_ncq_error(ap, &err_slot,
21997acdc34dSpatrick 			    ap->ap_pmp_ncq_port);
22007acdc34dSpatrick 			if (err_slot < 0)
22017acdc34dSpatrick 				goto failall;
22027acdc34dSpatrick 
22037acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
22047acdc34dSpatrick 				PORTNAME(ap), err_slot);
22057acdc34dSpatrick 
22067acdc34dSpatrick 			ccb = &ap->ap_ccbs[err_slot];
220792a87063Sjmatthew 			if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
220892a87063Sjmatthew 				printf("%s: NCQ errored slot %d is idle"
220992a87063Sjmatthew 				    " (%08x active)\n", PORTNAME(ap), err_slot,
221092a87063Sjmatthew 				    ci_saved);
221192a87063Sjmatthew 				goto failall;
221292a87063Sjmatthew 			}
22137acdc34dSpatrick 		} else {
22147acdc34dSpatrick 			/* Didn't reset, could gather extended info from log. */
22157acdc34dSpatrick 		}
22167acdc34dSpatrick 
22177acdc34dSpatrick 		/*
22187acdc34dSpatrick 		 * If we couldn't determine the errored slot, reset the port
22197acdc34dSpatrick 		 * and fail all the active slots.
22207acdc34dSpatrick 		 */
22217acdc34dSpatrick 		if (err_slot == -1) {
22227acdc34dSpatrick 			if (ahci_port_softreset(ap) != 0 &&
22237acdc34dSpatrick 			    ahci_port_portreset(ap, 0) != 0) {
22247acdc34dSpatrick 				printf("%s: couldn't reset after NCQ error, "
22257acdc34dSpatrick 				    "disabling device.\n", PORTNAME(ap));
22267acdc34dSpatrick 				goto fatal;
22277acdc34dSpatrick 			}
22287acdc34dSpatrick 			printf("%s: couldn't recover NCQ error, failing "
22297acdc34dSpatrick 			    "all outstanding commands.\n", PORTNAME(ap));
22307acdc34dSpatrick 			goto failall;
22317acdc34dSpatrick 		}
22327acdc34dSpatrick 
22337acdc34dSpatrick 		/* Clear the failed command in saved CI so completion runs. */
22347acdc34dSpatrick 		ci_saved &= ~(1 << err_slot);
22357acdc34dSpatrick 
22367acdc34dSpatrick 		/* Note the error in the ata_xfer. */
22377acdc34dSpatrick 		KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
22387acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_ERROR;
22397acdc34dSpatrick 
22407acdc34dSpatrick #ifdef DIAGNOSTIC
22417acdc34dSpatrick 		/* There may only be one outstanding standard command now. */
22427acdc34dSpatrick 		if (ap->ap_sactive == 0) {
22437acdc34dSpatrick 			tmp = ci_saved;
22447acdc34dSpatrick 			if (tmp) {
22457acdc34dSpatrick 				slot = ffs(tmp) - 1;
22467acdc34dSpatrick 				tmp &= ~(1 << slot);
22477acdc34dSpatrick 				KASSERT(tmp == 0);
22487acdc34dSpatrick 			}
22497acdc34dSpatrick 		}
22507acdc34dSpatrick #endif
22517acdc34dSpatrick 	}
22527acdc34dSpatrick 
22537acdc34dSpatrick 	/* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
22547acdc34dSpatrick 	 * when there's a normal SATA device attached.
22557acdc34dSpatrick 	 */
22567acdc34dSpatrick 	if ((ap->ap_state == AP_S_PMP_PROBE) &&
22577acdc34dSpatrick 	    (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) &&
22587acdc34dSpatrick 	    (is & AHCI_PREG_IS_IPMS)) {
22597acdc34dSpatrick 		slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD));
22607acdc34dSpatrick 		DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
22617acdc34dSpatrick 		    PORTNAME(ap), slot);
22627acdc34dSpatrick 
22637acdc34dSpatrick 		ccb = &ap->ap_ccbs[slot];
22647acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_ERROR;
22657acdc34dSpatrick 
22667acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS);
22677acdc34dSpatrick 		is &= ~AHCI_PREG_IS_IPMS;
22687acdc34dSpatrick 	}
22697acdc34dSpatrick 
22707acdc34dSpatrick 	/* ignore IFS errors while resetting a PMP port */
22717acdc34dSpatrick 	if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) {
22727acdc34dSpatrick 		DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
22737acdc34dSpatrick 		    "port\n", PORTNAME(ap));
22747acdc34dSpatrick 
22757acdc34dSpatrick 		need_restart = 1;
22767acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
22777acdc34dSpatrick 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
22787acdc34dSpatrick 		is &= ~AHCI_PREG_IS_IFS;
22797acdc34dSpatrick 		goto failall;
22807acdc34dSpatrick 	}
22817acdc34dSpatrick 
22827acdc34dSpatrick 	/* Check for remaining errors - they are fatal. */
22837acdc34dSpatrick 	if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
22847acdc34dSpatrick 	    AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
22857acdc34dSpatrick 		printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
22867acdc34dSpatrick 		    PORTNAME(ap), is, AHCI_PFMT_IS);
22877acdc34dSpatrick 
22887acdc34dSpatrick 		/* XXX try recovery first */
22897acdc34dSpatrick 		goto fatal;
22907acdc34dSpatrick 	}
22917acdc34dSpatrick 
22927acdc34dSpatrick 	/* Fail all outstanding commands if we know the port won't recover. */
22937acdc34dSpatrick 	if (ap->ap_state == AP_S_FATAL_ERROR) {
22947acdc34dSpatrick fatal:
22957acdc34dSpatrick 		ap->ap_state = AP_S_FATAL_ERROR;
22967acdc34dSpatrick failall:
22977acdc34dSpatrick 
22987acdc34dSpatrick 		/* Ensure port is shut down. */
22997acdc34dSpatrick 		ahci_port_stop(ap, 1);
23007acdc34dSpatrick 
23017acdc34dSpatrick 		/* Error all the active slots. */
23027acdc34dSpatrick 		ci_masked = ci_saved & *active;
23037acdc34dSpatrick 		while (ci_masked) {
23047acdc34dSpatrick 			slot = ffs(ci_masked) - 1;
23057acdc34dSpatrick 			ccb = &ap->ap_ccbs[slot];
23067acdc34dSpatrick 			ci_masked &= ~(1 << slot);
23077acdc34dSpatrick 			ccb->ccb_xa.state = ATA_S_ERROR;
23087acdc34dSpatrick 		}
23097acdc34dSpatrick 
23107acdc34dSpatrick 		/* Run completion for all active slots. */
23117acdc34dSpatrick 		ci_saved &= ~*active;
23127acdc34dSpatrick 
23137acdc34dSpatrick 		/* Don't restart the port if our problems were deemed fatal. */
23147acdc34dSpatrick 		if (ap->ap_state == AP_S_FATAL_ERROR)
23157acdc34dSpatrick 			need_restart = 0;
23167acdc34dSpatrick 	}
23177acdc34dSpatrick 
23187acdc34dSpatrick 	/*
23197acdc34dSpatrick 	 * CCB completion is detected by noticing its slot's bit in CI has
23207acdc34dSpatrick 	 * changed to zero some time after we activated it.
23217acdc34dSpatrick 	 * If we are polling, we may only be interested in particular slot(s).
23227acdc34dSpatrick 	 */
23237acdc34dSpatrick 	ci_masked = ~ci_saved & *active & ci_mask;
23247acdc34dSpatrick 	while (ci_masked) {
23257acdc34dSpatrick 		slot = ffs(ci_masked) - 1;
23267acdc34dSpatrick 		ccb = &ap->ap_ccbs[slot];
23277acdc34dSpatrick 		ci_masked &= ~(1 << slot);
23287acdc34dSpatrick 
23297acdc34dSpatrick 		DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
23307acdc34dSpatrick 		    PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
23317acdc34dSpatrick 		    " (error)" : "");
23327acdc34dSpatrick 
23337acdc34dSpatrick 		bus_dmamap_sync(sc->sc_dmat,
23347acdc34dSpatrick 		    AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
23357acdc34dSpatrick 		    ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
23367acdc34dSpatrick 		    sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
23377acdc34dSpatrick 
23387acdc34dSpatrick 		bus_dmamap_sync(sc->sc_dmat,
23397acdc34dSpatrick 		    AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
23407acdc34dSpatrick 		    ccb->ccb_slot * sizeof(struct ahci_cmd_table),
23417acdc34dSpatrick 		    sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
23427acdc34dSpatrick 
23437acdc34dSpatrick 		bus_dmamap_sync(sc->sc_dmat,
23447acdc34dSpatrick 		    AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
23457acdc34dSpatrick 		    sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
23467acdc34dSpatrick 
23477acdc34dSpatrick 		*active &= ~(1 << ccb->ccb_slot);
23487acdc34dSpatrick 		/* Copy the rfis into the ccb if we were asked for it */
23497acdc34dSpatrick 		if (ccb->ccb_xa.state == ATA_S_ONCHIP &&
23507acdc34dSpatrick 		    ccb->ccb_xa.flags & ATA_F_GET_RFIS) {
23517acdc34dSpatrick 			memcpy(&ccb->ccb_xa.rfis,
23527acdc34dSpatrick 			       ap->ap_rfis->rfis,
23537acdc34dSpatrick 			       sizeof(struct ata_fis_d2h));
23547acdc34dSpatrick 		}
23557acdc34dSpatrick 
23567acdc34dSpatrick 		processed |= 1 << ccb->ccb_slot;
23576579b5f8Sjsg 
23586579b5f8Sjsg 		ccb->ccb_done(ccb);
23597acdc34dSpatrick 	}
23607acdc34dSpatrick 
23617acdc34dSpatrick 	if (need_restart) {
23627acdc34dSpatrick 		/* Restart command DMA on the port */
23637acdc34dSpatrick 		ahci_port_start(ap, 0);
23647acdc34dSpatrick 
23657acdc34dSpatrick 		/* Re-enable outstanding commands on port. */
23667acdc34dSpatrick 		if (ci_saved) {
23677acdc34dSpatrick #ifdef DIAGNOSTIC
23687acdc34dSpatrick 			tmp = ci_saved;
23697acdc34dSpatrick 			while (tmp) {
23707acdc34dSpatrick 				slot = ffs(tmp) - 1;
23717acdc34dSpatrick 				tmp &= ~(1 << slot);
23727acdc34dSpatrick 				ccb = &ap->ap_ccbs[slot];
23737acdc34dSpatrick 				KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
23747acdc34dSpatrick 				KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
23757acdc34dSpatrick 				    (!!ap->ap_sactive));
23767acdc34dSpatrick 			}
23777acdc34dSpatrick #endif
23787acdc34dSpatrick 			DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
23797acdc34dSpatrick 			    "re-enabling%s slots %08x\n", PORTNAME(ap),
23807acdc34dSpatrick 			    ap->ap_sactive ? " NCQ" : "", ci_saved);
23817acdc34dSpatrick 
23827acdc34dSpatrick 			if (ap->ap_sactive)
23837acdc34dSpatrick 				ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
23847acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
23857acdc34dSpatrick 		}
23867acdc34dSpatrick 	}
23877acdc34dSpatrick 
23887acdc34dSpatrick 	return (processed);
23897acdc34dSpatrick }
23907acdc34dSpatrick 
23917acdc34dSpatrick struct ahci_ccb *
23927acdc34dSpatrick ahci_get_ccb(struct ahci_port *ap)
23937acdc34dSpatrick {
23947acdc34dSpatrick 	struct ahci_ccb			*ccb;
23957acdc34dSpatrick 
23967acdc34dSpatrick 	mtx_enter(&ap->ap_ccb_mtx);
23977acdc34dSpatrick 	ccb = TAILQ_FIRST(&ap->ap_ccb_free);
23987acdc34dSpatrick 	if (ccb != NULL) {
23997acdc34dSpatrick 		KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
24007acdc34dSpatrick 		TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
24017acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_SETUP;
24027acdc34dSpatrick 	}
24037acdc34dSpatrick 	mtx_leave(&ap->ap_ccb_mtx);
24047acdc34dSpatrick 
24057acdc34dSpatrick 	return (ccb);
24067acdc34dSpatrick }
24077acdc34dSpatrick 
24087acdc34dSpatrick void
24097acdc34dSpatrick ahci_put_ccb(struct ahci_ccb *ccb)
24107acdc34dSpatrick {
24117acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
24127acdc34dSpatrick 
24137acdc34dSpatrick #ifdef DIAGNOSTIC
24147acdc34dSpatrick 	if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
24157acdc34dSpatrick 	    ccb->ccb_xa.state != ATA_S_TIMEOUT &&
24167acdc34dSpatrick 	    ccb->ccb_xa.state != ATA_S_ERROR) {
24177acdc34dSpatrick 		printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
24187acdc34dSpatrick 		    "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
24197acdc34dSpatrick 		    ccb->ccb_slot);
24207acdc34dSpatrick 	}
24217acdc34dSpatrick #endif
24227acdc34dSpatrick 
24237acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PUT;
24247acdc34dSpatrick 	mtx_enter(&ap->ap_ccb_mtx);
24257acdc34dSpatrick 	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
24267acdc34dSpatrick 	mtx_leave(&ap->ap_ccb_mtx);
24277acdc34dSpatrick }
24287acdc34dSpatrick 
24297acdc34dSpatrick struct ahci_ccb *
24307acdc34dSpatrick ahci_get_err_ccb(struct ahci_port *ap)
24317acdc34dSpatrick {
24327acdc34dSpatrick 	struct ahci_ccb *err_ccb;
24337acdc34dSpatrick 	u_int32_t sact;
24347acdc34dSpatrick 
24357acdc34dSpatrick 	splassert(IPL_BIO);
24367acdc34dSpatrick 
24377acdc34dSpatrick 	/* No commands may be active on the chip. */
24387acdc34dSpatrick 	sact = ahci_pread(ap, AHCI_PREG_SACT);
24397acdc34dSpatrick 	if (sact != 0)
24407acdc34dSpatrick 		printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
24417acdc34dSpatrick 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
24427acdc34dSpatrick 
24437acdc34dSpatrick #ifdef DIAGNOSTIC
24447acdc34dSpatrick 	KASSERT(ap->ap_err_busy == 0);
24457acdc34dSpatrick 	ap->ap_err_busy = 1;
24467acdc34dSpatrick #endif
24477acdc34dSpatrick 	/* Save outstanding command state. */
24487acdc34dSpatrick 	ap->ap_err_saved_active = ap->ap_active;
24497acdc34dSpatrick 	ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
24507acdc34dSpatrick 	ap->ap_err_saved_sactive = ap->ap_sactive;
24517acdc34dSpatrick 
24527acdc34dSpatrick 	/*
24537acdc34dSpatrick 	 * Pretend we have no commands outstanding, so that completions won't
24547acdc34dSpatrick 	 * run prematurely.
24557acdc34dSpatrick 	 */
24567acdc34dSpatrick 	ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
24577acdc34dSpatrick 
24587acdc34dSpatrick 	/*
24597acdc34dSpatrick 	 * Grab a CCB to use for error recovery.  This should never fail, as
24607acdc34dSpatrick 	 * we ask atascsi to reserve one for us at init time.
24617acdc34dSpatrick 	 */
24627acdc34dSpatrick 	err_ccb = ap->ap_ccb_err;
24637acdc34dSpatrick 	err_ccb->ccb_xa.flags = 0;
24647acdc34dSpatrick 	err_ccb->ccb_xa.state = ATA_S_SETUP;
24657acdc34dSpatrick 	err_ccb->ccb_done = ahci_empty_done;
24667acdc34dSpatrick 
24677acdc34dSpatrick 	return (err_ccb);
24687acdc34dSpatrick }
24697acdc34dSpatrick 
24707acdc34dSpatrick void
24717acdc34dSpatrick ahci_put_err_ccb(struct ahci_ccb *ccb)
24727acdc34dSpatrick {
24737acdc34dSpatrick 	struct ahci_port *ap = ccb->ccb_port;
24747acdc34dSpatrick 	u_int32_t sact;
24757acdc34dSpatrick 
24767acdc34dSpatrick 	splassert(IPL_BIO);
24777acdc34dSpatrick 
24787acdc34dSpatrick #ifdef DIAGNOSTIC
24797acdc34dSpatrick 	KASSERT(ap->ap_err_busy);
24807acdc34dSpatrick #endif
24817acdc34dSpatrick 	/* No commands may be active on the chip */
24827acdc34dSpatrick 	sact = ahci_pread(ap, AHCI_PREG_SACT);
24837acdc34dSpatrick 	if (sact != 0)
248477db1dd4Skrw 		printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
24857acdc34dSpatrick 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
24867acdc34dSpatrick 
24877acdc34dSpatrick 	/* Done with the CCB */
24887acdc34dSpatrick 	KASSERT(ccb == ap->ap_ccb_err);
24897acdc34dSpatrick 
24907acdc34dSpatrick 	/* Restore outstanding command state */
24917acdc34dSpatrick 	ap->ap_sactive = ap->ap_err_saved_sactive;
24927acdc34dSpatrick 	ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
24937acdc34dSpatrick 	ap->ap_active = ap->ap_err_saved_active;
24947acdc34dSpatrick 
24957acdc34dSpatrick #ifdef DIAGNOSTIC
24967acdc34dSpatrick 	ap->ap_err_busy = 0;
24977acdc34dSpatrick #endif
24987acdc34dSpatrick }
24997acdc34dSpatrick 
25007acdc34dSpatrick struct ahci_ccb *
25017acdc34dSpatrick ahci_get_pmp_ccb(struct ahci_port *ap)
25027acdc34dSpatrick {
25037acdc34dSpatrick 	struct ahci_ccb *ccb;
25047acdc34dSpatrick 	u_int32_t sact;
25057acdc34dSpatrick 
25067acdc34dSpatrick 	/* some PMP commands need to be issued on slot 1,
25077acdc34dSpatrick 	 * particularly the command that clears SRST and
25087acdc34dSpatrick 	 * fetches the device signature.
25097acdc34dSpatrick 	 *
25107acdc34dSpatrick 	 * ensure the chip is idle and ccb 1 is available.
25117acdc34dSpatrick 	 */
25127acdc34dSpatrick 	splassert(IPL_BIO);
25137acdc34dSpatrick 
25147acdc34dSpatrick 	sact = ahci_pread(ap, AHCI_PREG_SACT);
25157acdc34dSpatrick 	if (sact != 0)
25167acdc34dSpatrick 		printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
25177acdc34dSpatrick 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
25187acdc34dSpatrick 
25197acdc34dSpatrick 	ccb = &ap->ap_ccbs[1];
25207acdc34dSpatrick 	KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
25217acdc34dSpatrick 	ccb->ccb_xa.flags = 0;
25227acdc34dSpatrick 	ccb->ccb_done = ahci_pmp_cmd_done;
25237acdc34dSpatrick 
25247acdc34dSpatrick 	mtx_enter(&ap->ap_ccb_mtx);
25257acdc34dSpatrick 	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
25267acdc34dSpatrick 	mtx_leave(&ap->ap_ccb_mtx);
25277acdc34dSpatrick 
25287acdc34dSpatrick 	return ccb;
25297acdc34dSpatrick }
25307acdc34dSpatrick 
25317acdc34dSpatrick void
25327acdc34dSpatrick ahci_put_pmp_ccb(struct ahci_ccb *ccb)
25337acdc34dSpatrick {
25347acdc34dSpatrick 	struct ahci_port *ap = ccb->ccb_port;
25357acdc34dSpatrick 	u_int32_t sact;
25367acdc34dSpatrick 
25377acdc34dSpatrick 	/* make sure this is the right ccb */
25387acdc34dSpatrick 	KASSERT(ccb == &ap->ap_ccbs[1]);
25397acdc34dSpatrick 
25407acdc34dSpatrick 	/* No commands may be active on the chip */
25417acdc34dSpatrick 	sact = ahci_pread(ap, AHCI_PREG_SACT);
25427acdc34dSpatrick 	if (sact != 0)
254377db1dd4Skrw 		printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
25447acdc34dSpatrick 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
25457acdc34dSpatrick 
25467acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PUT;
25477acdc34dSpatrick 	mtx_enter(&ap->ap_ccb_mtx);
25487acdc34dSpatrick 	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
25497acdc34dSpatrick 	mtx_leave(&ap->ap_ccb_mtx);
25507acdc34dSpatrick }
25517acdc34dSpatrick 
25527acdc34dSpatrick int
25537acdc34dSpatrick ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
25547acdc34dSpatrick {
25557acdc34dSpatrick 	struct ahci_ccb			*ccb;
25567acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot;
25577acdc34dSpatrick 	u_int32_t			cmd;
25587acdc34dSpatrick 	struct ata_fis_h2d		*fis;
25597acdc34dSpatrick 	int				rc = EIO, oldstate;
25607acdc34dSpatrick 
25617acdc34dSpatrick 	DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
25627acdc34dSpatrick 	oldstate = ap->ap_state;
25637acdc34dSpatrick 	ap->ap_state = AP_S_ERROR_RECOVERY;
25647acdc34dSpatrick 
25657acdc34dSpatrick 	/* Save command register state. */
25667acdc34dSpatrick 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
25677acdc34dSpatrick 
25687acdc34dSpatrick 	/* Port should have been idled already.  Start it. */
25697acdc34dSpatrick 	KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
25707acdc34dSpatrick 	ahci_port_start(ap, 0);
25717acdc34dSpatrick 
25727acdc34dSpatrick 	/* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
25737acdc34dSpatrick 	ccb = ahci_get_err_ccb(ap);
25747acdc34dSpatrick 	ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
25757acdc34dSpatrick 	ccb->ccb_xa.data = ap->ap_err_scratch;
25767acdc34dSpatrick 	ccb->ccb_xa.datalen = 512;
25777acdc34dSpatrick 	cmd_slot = ccb->ccb_cmd_hdr;
2578414fa24dSdlg 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
25797acdc34dSpatrick 
25807acdc34dSpatrick 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
25817acdc34dSpatrick 	fis->type = ATA_FIS_TYPE_H2D;
25827acdc34dSpatrick 	fis->flags = ATA_H2D_FLAGS_CMD | pmp_port;
25837acdc34dSpatrick 	fis->command = ATA_C_READ_LOG_EXT;
25847acdc34dSpatrick 	fis->lba_low = 0x10;		/* queued error log page (10h) */
25857acdc34dSpatrick 	fis->sector_count = 1;		/* number of sectors (1) */
25867acdc34dSpatrick 	fis->sector_count_exp = 0;
25877acdc34dSpatrick 	fis->lba_mid = 0;		/* starting offset */
25887acdc34dSpatrick 	fis->lba_mid_exp = 0;
25897acdc34dSpatrick 	fis->device = 0;
25907acdc34dSpatrick 
25913f8c2449Sdlg 	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
25923f8c2449Sdlg 	    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
25937acdc34dSpatrick 
25947acdc34dSpatrick 	if (ahci_load_prdt(ccb) != 0) {
25957acdc34dSpatrick 		rc = ENOMEM;	/* XXX caller must abort all commands */
25967acdc34dSpatrick 		goto err;
25977acdc34dSpatrick 	}
25987acdc34dSpatrick 
25997acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PENDING;
26007acdc34dSpatrick 	if (ahci_poll(ccb, 1000, NULL) != 0 ||
26017acdc34dSpatrick 	    ccb->ccb_xa.state == ATA_S_ERROR)
26027acdc34dSpatrick 		goto err;
26037acdc34dSpatrick 
26047acdc34dSpatrick 	rc = 0;
26057acdc34dSpatrick err:
26067acdc34dSpatrick 	/* Abort our command, if it failed, by stopping command DMA. */
26077acdc34dSpatrick 	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
26087acdc34dSpatrick 		printf("%s: log page read failed, slot %d was still active.\n",
26097acdc34dSpatrick 		    PORTNAME(ap), ccb->ccb_slot);
26107acdc34dSpatrick 		ahci_port_stop(ap, 0);
26117acdc34dSpatrick 	}
26127acdc34dSpatrick 
26137acdc34dSpatrick 	/* Done with the error CCB now. */
26147acdc34dSpatrick 	ahci_unload_prdt(ccb);
26157acdc34dSpatrick 	ahci_put_err_ccb(ccb);
26167acdc34dSpatrick 
26177acdc34dSpatrick 	/* Extract failed register set and tags from the scratch space. */
26187acdc34dSpatrick 	if (rc == 0) {
26197acdc34dSpatrick 		struct ata_log_page_10h		*log;
26207acdc34dSpatrick 		int				err_slot;
26217acdc34dSpatrick 
26227acdc34dSpatrick 		log = (struct ata_log_page_10h *)ap->ap_err_scratch;
26237acdc34dSpatrick 		if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
26247acdc34dSpatrick 			/* Not queued bit was set - wasn't an NCQ error? */
26257acdc34dSpatrick 			printf("%s: read NCQ error page, but not an NCQ "
26267acdc34dSpatrick 			    "error?\n", PORTNAME(ap));
26277acdc34dSpatrick 			rc = ESRCH;
26287acdc34dSpatrick 		} else {
26297acdc34dSpatrick 			/* Copy back the log record as a D2H register FIS. */
26307acdc34dSpatrick 			*err_slotp = err_slot = log->err_regs.type &
26317acdc34dSpatrick 			    ATA_LOG_10H_TYPE_TAG_MASK;
26327acdc34dSpatrick 
26337acdc34dSpatrick 			ccb = &ap->ap_ccbs[err_slot];
26347acdc34dSpatrick 			memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
26357acdc34dSpatrick 			    sizeof(struct ata_fis_d2h));
26367acdc34dSpatrick 			ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
26377acdc34dSpatrick 			ccb->ccb_xa.rfis.flags = 0;
26387acdc34dSpatrick 		}
26397acdc34dSpatrick 	}
26407acdc34dSpatrick 
26417acdc34dSpatrick 	/* Restore saved CMD register state */
26427acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
26437acdc34dSpatrick 	ap->ap_state = oldstate;
26447acdc34dSpatrick 
26457acdc34dSpatrick 	return (rc);
26467acdc34dSpatrick }
26477acdc34dSpatrick 
26487acdc34dSpatrick struct ahci_dmamem *
26497acdc34dSpatrick ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
26507acdc34dSpatrick {
26517acdc34dSpatrick 	struct ahci_dmamem		*adm;
26527acdc34dSpatrick 	int				nsegs;
26537acdc34dSpatrick 
26547acdc34dSpatrick 	adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO);
26557acdc34dSpatrick 	if (adm == NULL)
26567acdc34dSpatrick 		return (NULL);
26577acdc34dSpatrick 
26587acdc34dSpatrick 	adm->adm_size = size;
26597acdc34dSpatrick 
26607acdc34dSpatrick 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
26617acdc34dSpatrick 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
26627acdc34dSpatrick 		goto admfree;
26637acdc34dSpatrick 
26647acdc34dSpatrick 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
26657acdc34dSpatrick 	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
26667acdc34dSpatrick 		goto destroy;
26677acdc34dSpatrick 
26687acdc34dSpatrick 	if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
2669b70c9e02Spatrick 	    &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0)
26707acdc34dSpatrick 		goto free;
26717acdc34dSpatrick 
26727acdc34dSpatrick 	if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
26737acdc34dSpatrick 	    NULL, BUS_DMA_NOWAIT) != 0)
26747acdc34dSpatrick 		goto unmap;
26757acdc34dSpatrick 
26767acdc34dSpatrick 	return (adm);
26777acdc34dSpatrick 
26787acdc34dSpatrick unmap:
26797acdc34dSpatrick 	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
26807acdc34dSpatrick free:
26817acdc34dSpatrick 	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
26827acdc34dSpatrick destroy:
26837acdc34dSpatrick 	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
26847acdc34dSpatrick admfree:
26851f559dcaSderaadt 	free(adm, M_DEVBUF, sizeof(*adm));
26867acdc34dSpatrick 
26877acdc34dSpatrick 	return (NULL);
26887acdc34dSpatrick }
26897acdc34dSpatrick 
26907acdc34dSpatrick void
26917acdc34dSpatrick ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
26927acdc34dSpatrick {
26937acdc34dSpatrick 	bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
26947acdc34dSpatrick 	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
26957acdc34dSpatrick 	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
26967acdc34dSpatrick 	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
26971f559dcaSderaadt 	free(adm, M_DEVBUF, sizeof(*adm));
26987acdc34dSpatrick }
26997acdc34dSpatrick 
27007acdc34dSpatrick u_int32_t
27017acdc34dSpatrick ahci_read(struct ahci_softc *sc, bus_size_t r)
27027acdc34dSpatrick {
27037acdc34dSpatrick 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
27047acdc34dSpatrick 	    BUS_SPACE_BARRIER_READ);
27057acdc34dSpatrick 	return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
27067acdc34dSpatrick }
27077acdc34dSpatrick 
27087acdc34dSpatrick void
27097acdc34dSpatrick ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
27107acdc34dSpatrick {
27117acdc34dSpatrick 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
27127acdc34dSpatrick 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
27137acdc34dSpatrick 	    BUS_SPACE_BARRIER_WRITE);
27147acdc34dSpatrick }
27157acdc34dSpatrick 
27167acdc34dSpatrick int
27177acdc34dSpatrick ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
27187acdc34dSpatrick     u_int32_t target)
27197acdc34dSpatrick {
27207acdc34dSpatrick 	int				i;
27217acdc34dSpatrick 
27227acdc34dSpatrick 	for (i = 0; i < 1000; i++) {
27237acdc34dSpatrick 		if ((ahci_read(sc, r) & mask) != target)
27247acdc34dSpatrick 			return (0);
27257acdc34dSpatrick 		delay(1000);
27267acdc34dSpatrick 	}
27277acdc34dSpatrick 
27287acdc34dSpatrick 	return (1);
27297acdc34dSpatrick }
27307acdc34dSpatrick 
27317acdc34dSpatrick u_int32_t
27327acdc34dSpatrick ahci_pread(struct ahci_port *ap, bus_size_t r)
27337acdc34dSpatrick {
27347acdc34dSpatrick 	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
27357acdc34dSpatrick 	    BUS_SPACE_BARRIER_READ);
27367acdc34dSpatrick 	return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
27377acdc34dSpatrick }
27387acdc34dSpatrick 
27397acdc34dSpatrick void
27407acdc34dSpatrick ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
27417acdc34dSpatrick {
27427acdc34dSpatrick 	bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
27437acdc34dSpatrick 	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
27447acdc34dSpatrick 	    BUS_SPACE_BARRIER_WRITE);
27457acdc34dSpatrick }
27467acdc34dSpatrick 
27477acdc34dSpatrick int
27487acdc34dSpatrick ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
27497acdc34dSpatrick     u_int32_t target, int n)
27507acdc34dSpatrick {
27517acdc34dSpatrick 	int				i;
27527acdc34dSpatrick 
27537acdc34dSpatrick 	for (i = 0; i < n * 1000; i++) {
27547acdc34dSpatrick 		if ((ahci_pread(ap, r) & mask) == target)
27557acdc34dSpatrick 			return (0);
27567acdc34dSpatrick 		delay(1000);
27577acdc34dSpatrick 	}
27587acdc34dSpatrick 
27597acdc34dSpatrick 	return (1);
27607acdc34dSpatrick }
27617acdc34dSpatrick 
27627acdc34dSpatrick int
27637acdc34dSpatrick ahci_ata_probe(void *xsc, int port, int lun)
27647acdc34dSpatrick {
27657acdc34dSpatrick 	struct ahci_softc		*sc = xsc;
27667acdc34dSpatrick 	struct ahci_port		*ap = sc->sc_ports[port];
27677acdc34dSpatrick 
27687acdc34dSpatrick 	if (ap == NULL)
27697acdc34dSpatrick 		return (ATA_PORT_T_NONE);
27707acdc34dSpatrick 
27717acdc34dSpatrick 	if (lun != 0) {
27727acdc34dSpatrick 		int pmp_port = lun - 1;
27737acdc34dSpatrick 		if (pmp_port >= ap->ap_pmp_ports) {
27747acdc34dSpatrick 			return (ATA_PORT_T_NONE);
27757acdc34dSpatrick 		}
27767acdc34dSpatrick 		return (ahci_pmp_port_probe(ap, pmp_port));
27777acdc34dSpatrick 	} else {
27787acdc34dSpatrick 		return (ahci_port_signature(ap));
27797acdc34dSpatrick 	}
27807acdc34dSpatrick }
27817acdc34dSpatrick 
27827acdc34dSpatrick void
27837acdc34dSpatrick ahci_ata_free(void *xsc, int port, int lun)
27847acdc34dSpatrick {
27857acdc34dSpatrick 
27867acdc34dSpatrick }
27877acdc34dSpatrick 
27887acdc34dSpatrick struct ata_xfer *
27897acdc34dSpatrick ahci_ata_get_xfer(void *aaa_cookie, int port)
27907acdc34dSpatrick {
27917acdc34dSpatrick 	struct ahci_softc		*sc = aaa_cookie;
27927acdc34dSpatrick 	struct ahci_port		*ap = sc->sc_ports[port];
27937acdc34dSpatrick 	struct ahci_ccb			*ccb;
27947acdc34dSpatrick 
27957acdc34dSpatrick 	ccb = ahci_get_ccb(ap);
27967acdc34dSpatrick 	if (ccb == NULL) {
27977acdc34dSpatrick 		DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
27987acdc34dSpatrick 		    PORTNAME(ap));
27997acdc34dSpatrick 		return (NULL);
28007acdc34dSpatrick 	}
28017acdc34dSpatrick 
28027acdc34dSpatrick 	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
28037acdc34dSpatrick 	    PORTNAME(ap), ccb->ccb_slot);
28047acdc34dSpatrick 
28057acdc34dSpatrick 	return ((struct ata_xfer *)ccb);
28067acdc34dSpatrick }
28077acdc34dSpatrick 
28087acdc34dSpatrick void
28097acdc34dSpatrick ahci_ata_put_xfer(struct ata_xfer *xa)
28107acdc34dSpatrick {
28117acdc34dSpatrick 	struct ahci_ccb			*ccb = (struct ahci_ccb *)xa;
28127acdc34dSpatrick 
28137acdc34dSpatrick 	DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
28147acdc34dSpatrick 
28157acdc34dSpatrick 	ahci_put_ccb(ccb);
28167acdc34dSpatrick }
28177acdc34dSpatrick 
28187acdc34dSpatrick void
28197acdc34dSpatrick ahci_ata_cmd(struct ata_xfer *xa)
28207acdc34dSpatrick {
28217acdc34dSpatrick 	struct ahci_ccb			*ccb = (struct ahci_ccb *)xa;
28227acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot;
28237acdc34dSpatrick 	int				s;
28243f8c2449Sdlg 	u_int16_t			flags;
28257acdc34dSpatrick 
28267acdc34dSpatrick 	if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
28277acdc34dSpatrick 		goto failcmd;
28287acdc34dSpatrick 
28297acdc34dSpatrick 	ccb->ccb_done = ahci_ata_cmd_done;
28307acdc34dSpatrick 
28317acdc34dSpatrick 	cmd_slot = ccb->ccb_cmd_hdr;
28323f8c2449Sdlg 	flags = 5 /* FIS length (in DWORDs) */;
28333f8c2449Sdlg 	flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
28347acdc34dSpatrick 
28357acdc34dSpatrick 	if (xa->flags & ATA_F_WRITE)
28363f8c2449Sdlg 		flags |= AHCI_CMD_LIST_FLAG_W;
28377acdc34dSpatrick 
28387acdc34dSpatrick 	if (xa->flags & ATA_F_PACKET)
28393f8c2449Sdlg 		flags |= AHCI_CMD_LIST_FLAG_A;
28403f8c2449Sdlg 
28413f8c2449Sdlg 	htolem16(&cmd_slot->flags, flags);
28427acdc34dSpatrick 
28437acdc34dSpatrick 	if (ahci_load_prdt(ccb) != 0)
28447acdc34dSpatrick 		goto failcmd;
28457acdc34dSpatrick 
28467acdc34dSpatrick 	timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
28477acdc34dSpatrick 
28487acdc34dSpatrick 	xa->state = ATA_S_PENDING;
28497acdc34dSpatrick 
28507acdc34dSpatrick 	if (xa->flags & ATA_F_POLL)
28517acdc34dSpatrick 		ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
28527acdc34dSpatrick 	else {
28537acdc34dSpatrick 		s = splbio();
28547acdc34dSpatrick 		timeout_add_msec(&xa->stimeout, xa->timeout);
28557acdc34dSpatrick 		ahci_start(ccb);
28567acdc34dSpatrick 		splx(s);
28577acdc34dSpatrick 	}
28587acdc34dSpatrick 
28597acdc34dSpatrick 	return;
28607acdc34dSpatrick 
28617acdc34dSpatrick failcmd:
28627acdc34dSpatrick 	s = splbio();
28637acdc34dSpatrick 	xa->state = ATA_S_ERROR;
28647acdc34dSpatrick 	ata_complete(xa);
28657acdc34dSpatrick 	splx(s);
28667acdc34dSpatrick }
28677acdc34dSpatrick 
28687acdc34dSpatrick void
28697acdc34dSpatrick ahci_pmp_cmd_done(struct ahci_ccb *ccb)
28707acdc34dSpatrick {
28717acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
28727acdc34dSpatrick 
28737acdc34dSpatrick 	if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
28747acdc34dSpatrick 		ahci_issue_pending_commands(ccb->ccb_port,
28757acdc34dSpatrick 		    xa->flags & ATA_F_NCQ);
28767acdc34dSpatrick 
28777acdc34dSpatrick 	xa->state = ATA_S_COMPLETE;
28787acdc34dSpatrick }
28797acdc34dSpatrick 
28807acdc34dSpatrick 
28817acdc34dSpatrick void
28827acdc34dSpatrick ahci_ata_cmd_done(struct ahci_ccb *ccb)
28837acdc34dSpatrick {
28847acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
28857acdc34dSpatrick 
28867acdc34dSpatrick 	timeout_del(&xa->stimeout);
28877acdc34dSpatrick 
28887acdc34dSpatrick 	if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
28897acdc34dSpatrick 		ahci_issue_pending_commands(ccb->ccb_port,
28907acdc34dSpatrick 		    xa->flags & ATA_F_NCQ);
28917acdc34dSpatrick 
28927acdc34dSpatrick 	ahci_unload_prdt(ccb);
28937acdc34dSpatrick 
28947acdc34dSpatrick 	if (xa->state == ATA_S_ONCHIP)
28957acdc34dSpatrick 		xa->state = ATA_S_COMPLETE;
28967acdc34dSpatrick #ifdef DIAGNOSTIC
28977acdc34dSpatrick 	else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
28987acdc34dSpatrick 		printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
28997acdc34dSpatrick 		    "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
29007acdc34dSpatrick 		    ccb->ccb_slot);
29017acdc34dSpatrick #endif
29027acdc34dSpatrick 	if (xa->state != ATA_S_TIMEOUT)
29037acdc34dSpatrick 		ata_complete(xa);
29047acdc34dSpatrick }
29057acdc34dSpatrick 
29067acdc34dSpatrick void
29077acdc34dSpatrick ahci_ata_cmd_timeout(void *arg)
29087acdc34dSpatrick {
29097acdc34dSpatrick 	struct ahci_ccb			*ccb = arg;
29107acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
29117acdc34dSpatrick 	struct ahci_port		*ap = ccb->ccb_port;
29127acdc34dSpatrick 	int				s, ccb_was_started, ncq_cmd;
29137acdc34dSpatrick 	volatile u_int32_t		*active;
29147acdc34dSpatrick 
29157acdc34dSpatrick 	s = splbio();
29167acdc34dSpatrick 
29177acdc34dSpatrick 	ncq_cmd = (xa->flags & ATA_F_NCQ);
29187acdc34dSpatrick 	active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
29197acdc34dSpatrick 
29207acdc34dSpatrick 	if (ccb->ccb_xa.state == ATA_S_PENDING) {
29217acdc34dSpatrick 		DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
29227acdc34dSpatrick 		    "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
29237acdc34dSpatrick 		TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
29247acdc34dSpatrick 		ccb_was_started = 0;
29257acdc34dSpatrick 	} else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
29267acdc34dSpatrick 	    1 << ccb->ccb_slot)) {
29277acdc34dSpatrick 		DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
29287acdc34dSpatrick 		    "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
29297acdc34dSpatrick 		goto ret;
29307acdc34dSpatrick 	} else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
29317acdc34dSpatrick 		DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
29327acdc34dSpatrick 		    "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
29337acdc34dSpatrick 		    ISSET(*active, 1 << ccb->ccb_slot) ?
29347acdc34dSpatrick 		    " but slot is still active?" : ".");
29357acdc34dSpatrick 		goto ret;
29367acdc34dSpatrick 	} else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
29377acdc34dSpatrick 	    AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
29387acdc34dSpatrick 	    1 << ccb->ccb_slot)) {
29397acdc34dSpatrick 		DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
29407acdc34dSpatrick 		    "IRQ handler didn't detect it.  Why?\n", PORTNAME(ap),
29417acdc34dSpatrick 		    ccb->ccb_slot);
29427acdc34dSpatrick 		*active &= ~(1 << ccb->ccb_slot);
29437acdc34dSpatrick 		ccb->ccb_done(ccb);
29447acdc34dSpatrick 		goto ret;
29457acdc34dSpatrick 	} else {
29467acdc34dSpatrick 		ccb_was_started = 1;
29477acdc34dSpatrick 	}
29487acdc34dSpatrick 
29497acdc34dSpatrick 	/* Complete the slot with a timeout error. */
29507acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_TIMEOUT;
29517acdc34dSpatrick 	*active &= ~(1 << ccb->ccb_slot);
29527acdc34dSpatrick 	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
29537acdc34dSpatrick 	ccb->ccb_done(ccb);	/* This won't issue pending commands or run the
29547acdc34dSpatrick 				   atascsi completion. */
29557acdc34dSpatrick 
29567acdc34dSpatrick 	/* Reset port to abort running command. */
29577acdc34dSpatrick 	if (ccb_was_started) {
29587acdc34dSpatrick 		DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
29597acdc34dSpatrick 		    "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
29607acdc34dSpatrick 		    ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
29617acdc34dSpatrick 		if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
29627acdc34dSpatrick 		    != 0) {
29637acdc34dSpatrick 			printf("%s: failed to reset port during timeout "
29647acdc34dSpatrick 			    "handling, disabling it\n", PORTNAME(ap));
29657acdc34dSpatrick 			ap->ap_state = AP_S_FATAL_ERROR;
29667acdc34dSpatrick 		}
29677acdc34dSpatrick 
29687acdc34dSpatrick 		/* Restart any other commands that were aborted by the reset. */
29697acdc34dSpatrick 		if (*active) {
29707acdc34dSpatrick 			DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
29717acdc34dSpatrick 			    "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
29727acdc34dSpatrick 			    *active);
29737acdc34dSpatrick 			if (ncq_cmd)
29747acdc34dSpatrick 				ahci_pwrite(ap, AHCI_PREG_SACT, *active);
29757acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_CI, *active);
29767acdc34dSpatrick 		}
29777acdc34dSpatrick 	}
29787acdc34dSpatrick 
29797acdc34dSpatrick 	/* Issue any pending commands now. */
29807acdc34dSpatrick 	DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
29817acdc34dSpatrick 	if (ccb_was_started)
29827acdc34dSpatrick 		ahci_issue_pending_commands(ap, ncq_cmd);
29837acdc34dSpatrick 	else if (ap->ap_active == 0)
29847acdc34dSpatrick 		ahci_issue_pending_ncq_commands(ap);
29857acdc34dSpatrick 
29867acdc34dSpatrick 	/* Complete the timed out ata_xfer I/O (may generate new I/O). */
29877acdc34dSpatrick 	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
29887acdc34dSpatrick 	ata_complete(xa);
29897acdc34dSpatrick 
29907acdc34dSpatrick 	DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
29917acdc34dSpatrick ret:
29927acdc34dSpatrick 	splx(s);
29937acdc34dSpatrick }
29947acdc34dSpatrick 
29957acdc34dSpatrick void
29967acdc34dSpatrick ahci_empty_done(struct ahci_ccb *ccb)
29977acdc34dSpatrick {
29987acdc34dSpatrick 	if (ccb->ccb_xa.state != ATA_S_ERROR)
29997acdc34dSpatrick 		ccb->ccb_xa.state = ATA_S_COMPLETE;
30007acdc34dSpatrick }
30017acdc34dSpatrick 
30027acdc34dSpatrick int
30037acdc34dSpatrick ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
30047acdc34dSpatrick {
30057acdc34dSpatrick 	struct ahci_ccb	*ccb;
30067acdc34dSpatrick 	struct ata_fis_h2d *fis;
30077acdc34dSpatrick 	int error;
30087acdc34dSpatrick 
3009189d7e68Skrw 	ccb = ahci_get_pmp_ccb(ap);	/* Always returns non-NULL. */
30107acdc34dSpatrick 	ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS;
30117acdc34dSpatrick 	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
30127acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PENDING;
30137acdc34dSpatrick 
3014414fa24dSdlg 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
30157acdc34dSpatrick 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
30167acdc34dSpatrick 	fis->type = ATA_FIS_TYPE_H2D;
30177acdc34dSpatrick 	fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
30187acdc34dSpatrick 	fis->command = ATA_C_READ_PM;
30197acdc34dSpatrick 	fis->features = which;
30207acdc34dSpatrick 	fis->device = target | ATA_H2D_DEVICE_LBA;
30217acdc34dSpatrick 	fis->control = ATA_FIS_CONTROL_4BIT;
30227acdc34dSpatrick 
30237acdc34dSpatrick 	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
30247acdc34dSpatrick 		error = 1;
30257acdc34dSpatrick 	} else {
30267acdc34dSpatrick 		*datap = ccb->ccb_xa.rfis.sector_count |
30277acdc34dSpatrick 		    (ccb->ccb_xa.rfis.lba_low << 8) |
30287acdc34dSpatrick 		    (ccb->ccb_xa.rfis.lba_mid << 16) |
30297acdc34dSpatrick 		    (ccb->ccb_xa.rfis.lba_high << 24);
30307acdc34dSpatrick 		error = 0;
30317acdc34dSpatrick 	}
30327acdc34dSpatrick 	ahci_put_pmp_ccb(ccb);
30337acdc34dSpatrick 	return (error);
30347acdc34dSpatrick }
30357acdc34dSpatrick 
30367acdc34dSpatrick int
30377acdc34dSpatrick ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
30387acdc34dSpatrick {
30397acdc34dSpatrick 	struct ahci_ccb	*ccb;
30407acdc34dSpatrick 	struct ata_fis_h2d *fis;
30417acdc34dSpatrick 	int error;
30427acdc34dSpatrick 
3043189d7e68Skrw 	ccb = ahci_get_pmp_ccb(ap);	/* Always returns non-NULL. */
30447acdc34dSpatrick 	ccb->ccb_xa.flags = ATA_F_POLL;
30457acdc34dSpatrick 	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
30467acdc34dSpatrick 	ccb->ccb_xa.state = ATA_S_PENDING;
30477acdc34dSpatrick 
3048414fa24dSdlg 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
30497acdc34dSpatrick 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
30507acdc34dSpatrick 	fis->type = ATA_FIS_TYPE_H2D;
30517acdc34dSpatrick 	fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
30527acdc34dSpatrick 	fis->command = ATA_C_WRITE_PM;
30537acdc34dSpatrick 	fis->features = which;
30547acdc34dSpatrick 	fis->device = target | ATA_H2D_DEVICE_LBA;
30557acdc34dSpatrick 	fis->sector_count = (u_int8_t)data;
30567acdc34dSpatrick 	fis->lba_low = (u_int8_t)(data >> 8);
30577acdc34dSpatrick 	fis->lba_mid = (u_int8_t)(data >> 16);
30587acdc34dSpatrick 	fis->lba_high = (u_int8_t)(data >> 24);
30597acdc34dSpatrick 	fis->control = ATA_FIS_CONTROL_4BIT;
30607acdc34dSpatrick 
30617acdc34dSpatrick 	error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
30627acdc34dSpatrick 	ahci_put_pmp_ccb(ccb);
30637acdc34dSpatrick 	return (error);
30647acdc34dSpatrick }
30657acdc34dSpatrick 
30667acdc34dSpatrick int
30677acdc34dSpatrick ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
30687acdc34dSpatrick {
30697acdc34dSpatrick 	int error;
30707acdc34dSpatrick 
30717acdc34dSpatrick 	error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap);
30727acdc34dSpatrick 	if (error == 0)
30737acdc34dSpatrick 		error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1);
30747acdc34dSpatrick 	if (error)
30757acdc34dSpatrick 		*datap = 0;
30767acdc34dSpatrick 
30777acdc34dSpatrick 	return (error);
30787acdc34dSpatrick }
30797acdc34dSpatrick 
30807acdc34dSpatrick int
30817acdc34dSpatrick ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
30827acdc34dSpatrick {
30837acdc34dSpatrick 	u_int32_t chipid;
30847acdc34dSpatrick 	u_int32_t rev;
30857acdc34dSpatrick 	u_int32_t nports;
30867acdc34dSpatrick 	u_int32_t features;
30877acdc34dSpatrick 	u_int32_t enabled;
30887acdc34dSpatrick 	int s;
30897acdc34dSpatrick 
30907acdc34dSpatrick 	s = splbio();
30917acdc34dSpatrick 
30927acdc34dSpatrick 	if (ahci_pmp_read(ap, 15, 0, &chipid) ||
30937acdc34dSpatrick 	    ahci_pmp_read(ap, 15, 1, &rev) ||
30947acdc34dSpatrick 	    ahci_pmp_read(ap, 15, 2, &nports) ||
30957acdc34dSpatrick 	    ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) ||
30967acdc34dSpatrick 	    ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) {
30977acdc34dSpatrick 		printf("%s: port multiplier identification failed\n",
30987acdc34dSpatrick 		    PORTNAME(ap));
30997acdc34dSpatrick 		splx(s);
31007acdc34dSpatrick 		return (1);
31017acdc34dSpatrick 	}
31027acdc34dSpatrick 	splx(s);
31037acdc34dSpatrick 
31047acdc34dSpatrick 	nports &= 0x0F;
31057acdc34dSpatrick 
31067acdc34dSpatrick 	/* ignore SEMB port on SiI3726 port multiplier chips */
31077acdc34dSpatrick 	if (chipid == 0x37261095) {
31087acdc34dSpatrick 		nports--;
31097acdc34dSpatrick 	}
31107acdc34dSpatrick 
31117acdc34dSpatrick 	printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
31127acdc34dSpatrick 	    "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev,
31137acdc34dSpatrick 	    SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled,
31147acdc34dSpatrick 	    SATA_PFMT_PM_FEA);
31157acdc34dSpatrick 
31167acdc34dSpatrick 	*ret_nports = nports;
31177acdc34dSpatrick 	return (0);
31187acdc34dSpatrick }
31197acdc34dSpatrick 
31207acdc34dSpatrick 
31217acdc34dSpatrick #ifdef HIBERNATE
31227acdc34dSpatrick void
31237acdc34dSpatrick ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
31247acdc34dSpatrick {
31257acdc34dSpatrick 	ccb->ccb_cmd_hdr->prdbc = 0;
31267acdc34dSpatrick 	ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
31277acdc34dSpatrick }
31287acdc34dSpatrick 
31297acdc34dSpatrick int
31307acdc34dSpatrick ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
31317acdc34dSpatrick {
31327acdc34dSpatrick 	u_int32_t			is, ci_saved;
31337acdc34dSpatrick 	int				process_error = 0;
31347acdc34dSpatrick 
31357acdc34dSpatrick 	is = ahci_pread(ap, AHCI_PREG_IS);
31367acdc34dSpatrick 
31377acdc34dSpatrick 	ci_saved = ahci_pread(ap, AHCI_PREG_CI);
31387acdc34dSpatrick 
31397acdc34dSpatrick 	if (is & AHCI_PREG_IS_DHRS) {
31407acdc34dSpatrick 		u_int32_t tfd;
31417acdc34dSpatrick 		u_int32_t cmd;
31427acdc34dSpatrick 
31437acdc34dSpatrick 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
31447acdc34dSpatrick 		cmd = ahci_pread(ap, AHCI_PREG_CMD);
31457acdc34dSpatrick 		if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
31467acdc34dSpatrick 		    (cmd & AHCI_PREG_CMD_CR) == 0) {
31477acdc34dSpatrick 			process_error = 1;
31487acdc34dSpatrick 		} else {
31497acdc34dSpatrick 			ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
31507acdc34dSpatrick 		}
31517acdc34dSpatrick 	} else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
31527acdc34dSpatrick 	    AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
31537acdc34dSpatrick 		process_error = 1;
31547acdc34dSpatrick 	}
31557acdc34dSpatrick 
31567acdc34dSpatrick 	/* Command failed.  See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
31577acdc34dSpatrick 	if (process_error) {
31587acdc34dSpatrick 
31597acdc34dSpatrick 		/* Turn off ST to clear CI and SACT. */
31607acdc34dSpatrick 		ahci_port_stop(ap, 0);
31617acdc34dSpatrick 
31627acdc34dSpatrick 		/* just return an error indicator?  we can't meaningfully
31637acdc34dSpatrick 		 * recover, and on the way back out we'll DVACT_RESUME which
31647acdc34dSpatrick 		 * resets and reinits the port.
31657acdc34dSpatrick 		 */
31667acdc34dSpatrick 		return (EIO);
31677acdc34dSpatrick 	}
31687acdc34dSpatrick 
31697acdc34dSpatrick 	/* command is finished when the bit in CI for the slot goes to 0 */
31707acdc34dSpatrick 	if (ci_saved & (1 << ccb->ccb_slot)) {
31717acdc34dSpatrick 		return (EAGAIN);
31727acdc34dSpatrick 	}
31737acdc34dSpatrick 
31747acdc34dSpatrick 	return (0);
31757acdc34dSpatrick }
31767acdc34dSpatrick 
31777acdc34dSpatrick void
31787acdc34dSpatrick ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
31797acdc34dSpatrick {
31807acdc34dSpatrick 	struct ata_xfer			*xa = &ccb->ccb_xa;
3181014fa59dSmlarkin 	struct ahci_prdt		*prdt = ccb->ccb_cmd_table->prdt;
31827acdc34dSpatrick 	struct ahci_cmd_hdr		*cmd_slot = ccb->ccb_cmd_hdr;
31837acdc34dSpatrick 	int				i;
31847acdc34dSpatrick 	paddr_t				data_phys;
31857acdc34dSpatrick 	u_int64_t			data_bus_phys;
31867acdc34dSpatrick 	vaddr_t				data_addr;
31877acdc34dSpatrick 	size_t				seglen;
31887acdc34dSpatrick 	size_t				buflen;
31897acdc34dSpatrick 
31907acdc34dSpatrick 	if (xa->datalen == 0) {
31917acdc34dSpatrick 		ccb->ccb_cmd_hdr->prdtl = 0;
31927acdc34dSpatrick 		return;
31937acdc34dSpatrick 	}
31947acdc34dSpatrick 
31957acdc34dSpatrick 	/* derived from i386/amd64 _bus_dma_load_buffer;
31967acdc34dSpatrick 	 * for amd64 the buffer will always be dma safe.
31977acdc34dSpatrick 	 */
31987acdc34dSpatrick 
31997acdc34dSpatrick 	buflen = xa->datalen;
32007acdc34dSpatrick 	data_addr = (vaddr_t)xa->data;
32017acdc34dSpatrick 	for (i = 0; buflen > 0; i++) {
32027acdc34dSpatrick 		pmap_extract(pmap_kernel(), data_addr, &data_phys);
32037acdc34dSpatrick 		data_bus_phys = data_phys;
32047acdc34dSpatrick 
32057acdc34dSpatrick 		seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET);
32067acdc34dSpatrick 		if (buflen < seglen)
32077acdc34dSpatrick 			seglen = buflen;
32087acdc34dSpatrick 
3209b1514dfeSdlg 		ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3210b1514dfeSdlg 
32117acdc34dSpatrick 		data_addr += seglen;
32127acdc34dSpatrick 		buflen -= seglen;
32137acdc34dSpatrick 	}
32147acdc34dSpatrick 
32153f8c2449Sdlg 	htolem16(&cmd_slot->prdtl, i);
32167acdc34dSpatrick }
32177acdc34dSpatrick 
32187acdc34dSpatrick int
32197acdc34dSpatrick ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
32207acdc34dSpatrick     int op, void *page)
32217acdc34dSpatrick {
32227acdc34dSpatrick 	/* we use the 'real' ahci_port and ahci_softc here, but
32237acdc34dSpatrick 	 * never write to them
32247acdc34dSpatrick 	 */
32257acdc34dSpatrick 	struct {
32267acdc34dSpatrick 		struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
32277acdc34dSpatrick 		struct ahci_rfis rfis;		 /* 1k aligned, 256 bytes */
32287acdc34dSpatrick 		/* cmd table isn't actually used because of mysteries */
32297acdc34dSpatrick 		struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
32307acdc34dSpatrick 		struct ahci_port *ap;
32317acdc34dSpatrick 		struct ahci_ccb ccb_buf;
32327acdc34dSpatrick 		struct ahci_ccb *ccb;
32337acdc34dSpatrick 		struct ahci_cmd_hdr *hdr_buf;
32347acdc34dSpatrick 		int pmp_port;
3235a1645687Sderaadt 		daddr_t poffset;
3236a1645687Sderaadt 		size_t psize;
32377acdc34dSpatrick 	} *my = page;
32387acdc34dSpatrick 	struct ata_fis_h2d *fis;
32397acdc34dSpatrick 	u_int32_t sector_count;
32407acdc34dSpatrick 	struct ahci_cmd_hdr *cmd_slot;
32417acdc34dSpatrick 	int rc;
32427acdc34dSpatrick 	int timeout;
32433f8c2449Sdlg 	u_int16_t flags;
32447acdc34dSpatrick 
32457acdc34dSpatrick 	if (op == HIB_INIT) {
32467acdc34dSpatrick 		struct device *disk;
32477acdc34dSpatrick 		struct device *scsibus;
32487acdc34dSpatrick 		struct ahci_softc *sc;
32497acdc34dSpatrick 		extern struct cfdriver sd_cd;
32507acdc34dSpatrick 		struct scsi_link *link;
32517acdc34dSpatrick 		struct scsibus_softc *bus_sc;
32527acdc34dSpatrick 		int port;
32537acdc34dSpatrick 		paddr_t page_phys;
32547acdc34dSpatrick 		u_int64_t item_phys;
32557acdc34dSpatrick 		u_int32_t cmd;
32567acdc34dSpatrick 
3257a1645687Sderaadt 		my->poffset = blkno;
3258a1645687Sderaadt 		my->psize = size;
3259a1645687Sderaadt 
32607acdc34dSpatrick 		/* map dev to an ahci port */
32617acdc34dSpatrick 		disk = disk_lookup(&sd_cd, DISKUNIT(dev));
32627acdc34dSpatrick 		scsibus = disk->dv_parent;
32637acdc34dSpatrick 		sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
32647acdc34dSpatrick 
32657acdc34dSpatrick 		/* find the scsi_link for the device, which has the port */
32667acdc34dSpatrick 		port = -1;
32677acdc34dSpatrick 		bus_sc = (struct scsibus_softc *)scsibus;
326820f90945Skrw 		SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) {
32697acdc34dSpatrick 			if (link->device_softc == disk) {
32707acdc34dSpatrick 				port = link->target;
32717acdc34dSpatrick 				if (link->lun > 0)
32727acdc34dSpatrick 					my->pmp_port = link->lun - 1;
32737acdc34dSpatrick 				else
32747acdc34dSpatrick 					my->pmp_port = 0;
32757acdc34dSpatrick 
32767acdc34dSpatrick 				break;
32777acdc34dSpatrick 			}
32787acdc34dSpatrick 		}
32797acdc34dSpatrick 		if (port == -1) {
32807acdc34dSpatrick 			/* don't know where the disk is */
32817acdc34dSpatrick 			return (EIO);
32827acdc34dSpatrick 		}
32837acdc34dSpatrick 
32847acdc34dSpatrick 		my->ap = sc->sc_ports[port];
32857acdc34dSpatrick 
32867acdc34dSpatrick 		/* we're going to use the first command slot,
32877acdc34dSpatrick 		 * so ensure it's not already in use
32887acdc34dSpatrick 		 */
32897acdc34dSpatrick 		if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) {
32907acdc34dSpatrick 			/* this shouldn't happen, we should be idle */
32917acdc34dSpatrick 			return (EIO);
32927acdc34dSpatrick 		}
32937acdc34dSpatrick 
32947acdc34dSpatrick 		/* stop the port so we can relocate to the hibernate page */
32957acdc34dSpatrick 		if (ahci_port_stop(my->ap, 1)) {
32967acdc34dSpatrick 			return (EIO);
32977acdc34dSpatrick 		}
32987acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0);
32997acdc34dSpatrick 
33007acdc34dSpatrick 		pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
33017acdc34dSpatrick 
33027acdc34dSpatrick 		/* Setup RFIS base address */
33037acdc34dSpatrick 		item_phys = page_phys + ((void *)&my->rfis - page);
33047acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_FBU,
33057acdc34dSpatrick 		    (u_int32_t)(item_phys >> 32));
33067acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys);
33077acdc34dSpatrick 
33087acdc34dSpatrick 		/* Enable FIS reception and activate port. */
33097acdc34dSpatrick 		cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
33107acdc34dSpatrick 		cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD |
33117acdc34dSpatrick 		    AHCI_PREG_CMD_SUD;
33127acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd |
33137acdc34dSpatrick 		    AHCI_PREG_CMD_ICC_ACTIVE);
33147acdc34dSpatrick 
33157acdc34dSpatrick 		/* Check whether port activated.  */
33167acdc34dSpatrick 		cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
33177acdc34dSpatrick 		if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
33187acdc34dSpatrick 			return (EIO);
33197acdc34dSpatrick 		}
33207acdc34dSpatrick 
33217acdc34dSpatrick 		/* Set up the single CCB */
33227acdc34dSpatrick 		my->ccb = &my->ccb_buf;
33237acdc34dSpatrick 		my->ccb->ccb_slot = 0;
33247acdc34dSpatrick 		my->ccb->ccb_port = my->ap;
33257acdc34dSpatrick 
33267acdc34dSpatrick 		/* Setup command list base address */
33277acdc34dSpatrick 		item_phys = page_phys + ((void *)&my->cmd_hdr - page);
33287acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_CLBU,
33297acdc34dSpatrick 		    (u_int32_t)(item_phys >> 32));
33307acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys);
33317acdc34dSpatrick 
33327acdc34dSpatrick 		my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
33337acdc34dSpatrick 
333477db1dd4Skrw 		/* use existing cmd table - moving to a new one fails */
33357acdc34dSpatrick 		my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
33367acdc34dSpatrick 		pmap_extract(pmap_kernel(),
33377acdc34dSpatrick 		    (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table),
33387acdc34dSpatrick 		    &page_phys);
33397acdc34dSpatrick 		item_phys = page_phys;
33407acdc34dSpatrick #if 0
33417acdc34dSpatrick 		/* use cmd table in hibernate page (doesn't work) */
33427acdc34dSpatrick 		my->ccb->ccb_cmd_table = &my->cmd_table;
33437acdc34dSpatrick 		item_phys = page_phys + ((void *)&my->cmd_table - page);
33447acdc34dSpatrick #endif
33454d3a7877Sdlg 		htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys);
33467acdc34dSpatrick 
33477acdc34dSpatrick 		my->ccb->ccb_xa.fis =
33487acdc34dSpatrick 		    (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
33497acdc34dSpatrick 		my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
33507acdc34dSpatrick 		my->ccb->ccb_xa.tag = 0;
33517acdc34dSpatrick 
33527acdc34dSpatrick 		/* Wait for ICC change to complete */
33537acdc34dSpatrick 		ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
33547acdc34dSpatrick 
33557acdc34dSpatrick 		if (ahci_port_start(my->ap, 0)) {
33567acdc34dSpatrick 			return (EIO);
33577acdc34dSpatrick 		}
33587acdc34dSpatrick 
33597acdc34dSpatrick 		/* Flush interrupts for port */
33607acdc34dSpatrick 		ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap,
33617acdc34dSpatrick 		    AHCI_PREG_IS));
33627acdc34dSpatrick 		ahci_write(sc, AHCI_REG_IS, 1 << port);
33637acdc34dSpatrick 
33647acdc34dSpatrick 		ahci_enable_interrupts(my->ap);
33657acdc34dSpatrick 		return (0);
33667acdc34dSpatrick 	} else if (op == HIB_DONE) {
33677acdc34dSpatrick 		ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME);
33687acdc34dSpatrick 		return (0);
33697acdc34dSpatrick 	}
33707acdc34dSpatrick 
3371a1645687Sderaadt 	if (blkno > my->psize)
3372a1645687Sderaadt 		return (E2BIG);
3373a1645687Sderaadt 	blkno += my->poffset;
3374a1645687Sderaadt 
33757acdc34dSpatrick 	/* build fis */
33767acdc34dSpatrick 	sector_count = size / 512;	/* dlg promises this is okay */
33777acdc34dSpatrick 	my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ;
33787acdc34dSpatrick 	fis = my->ccb->ccb_xa.fis;
33797acdc34dSpatrick 	fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port;
33807acdc34dSpatrick 	fis->lba_low = blkno & 0xff;
33817acdc34dSpatrick 	fis->lba_mid = (blkno >> 8) & 0xff;
33827acdc34dSpatrick 	fis->lba_high = (blkno >> 16) & 0xff;
33837acdc34dSpatrick 
33847acdc34dSpatrick 	if (sector_count > 0x100 || blkno > 0xfffffff) {
33857acdc34dSpatrick 		/* Use LBA48 */
33867acdc34dSpatrick 		fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT :
33877acdc34dSpatrick 		    ATA_C_READDMA_EXT;
33887acdc34dSpatrick 		fis->device = ATA_H2D_DEVICE_LBA;
33897acdc34dSpatrick 		fis->lba_low_exp = (blkno >> 24) & 0xff;
33907acdc34dSpatrick 		fis->lba_mid_exp = (blkno >> 32) & 0xff;
33917acdc34dSpatrick 		fis->lba_high_exp = (blkno >> 40) & 0xff;
33927acdc34dSpatrick 		fis->sector_count = sector_count & 0xff;
33937acdc34dSpatrick 		fis->sector_count_exp = (sector_count >> 8) & 0xff;
33947acdc34dSpatrick 	} else {
33957acdc34dSpatrick 		/* Use LBA */
33967acdc34dSpatrick 		fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA;
33977acdc34dSpatrick 		fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f);
33987acdc34dSpatrick 		fis->sector_count = sector_count & 0xff;
33997acdc34dSpatrick 	}
34007acdc34dSpatrick 
34017acdc34dSpatrick 	my->ccb->ccb_xa.data = (void *)addr;
34027acdc34dSpatrick 	my->ccb->ccb_xa.datalen = size;
34037acdc34dSpatrick 	my->ccb->ccb_xa.pmp_port = my->pmp_port;
34047acdc34dSpatrick 	my->ccb->ccb_xa.flags |= ATA_F_POLL;
34057acdc34dSpatrick 
34067acdc34dSpatrick 	cmd_slot = my->ccb->ccb_cmd_hdr;
34073f8c2449Sdlg 	flags = 5; /* FIS length (in DWORDs) */
34083f8c2449Sdlg 	flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
34097acdc34dSpatrick 
34107acdc34dSpatrick 	if (op == HIB_W)
34113f8c2449Sdlg 		flags |= AHCI_CMD_LIST_FLAG_W;
34123f8c2449Sdlg 
34133f8c2449Sdlg 	htolem16(&cmd_slot->flags, flags);
34147acdc34dSpatrick 
34157acdc34dSpatrick 	ahci_hibernate_load_prdt(my->ccb);
34167acdc34dSpatrick 
34177acdc34dSpatrick 	ahci_hibernate_io_start(my->ap, my->ccb);
34180b4b247eSmlarkin 	timeout = 1000000;
34197acdc34dSpatrick 	while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) {
34200b4b247eSmlarkin 		delay(1);
34217acdc34dSpatrick 		timeout--;
34227acdc34dSpatrick 		if (timeout == 0) {
34237acdc34dSpatrick 			return (EIO);
34247acdc34dSpatrick 		}
34257acdc34dSpatrick 	}
34267acdc34dSpatrick 
34277acdc34dSpatrick 	return (0);
34287acdc34dSpatrick }
34297acdc34dSpatrick 
34307acdc34dSpatrick #endif
3431