1*1cfc6e0bSjan /* $OpenBSD: ahci.c,v 1.43 2024/11/22 09:29:41 jan Exp $ */ 27acdc34dSpatrick 37acdc34dSpatrick /* 47acdc34dSpatrick * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 57acdc34dSpatrick * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 67acdc34dSpatrick * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 77acdc34dSpatrick * 87acdc34dSpatrick * Permission to use, copy, modify, and distribute this software for any 97acdc34dSpatrick * purpose with or without fee is hereby granted, provided that the above 107acdc34dSpatrick * copyright notice and this permission notice appear in all copies. 117acdc34dSpatrick * 127acdc34dSpatrick * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 137acdc34dSpatrick * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 147acdc34dSpatrick * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 157acdc34dSpatrick * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 167acdc34dSpatrick * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 177acdc34dSpatrick * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 187acdc34dSpatrick * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 197acdc34dSpatrick */ 207acdc34dSpatrick 217acdc34dSpatrick #include <sys/param.h> 227acdc34dSpatrick #include <sys/systm.h> 237acdc34dSpatrick #include <sys/buf.h> 247acdc34dSpatrick #include <sys/kernel.h> 257acdc34dSpatrick #include <sys/malloc.h> 267acdc34dSpatrick #include <sys/device.h> 277acdc34dSpatrick #include <sys/queue.h> 287acdc34dSpatrick #include <sys/mutex.h> 297acdc34dSpatrick #include <sys/pool.h> 307acdc34dSpatrick 317acdc34dSpatrick #include <machine/bus.h> 327acdc34dSpatrick 337acdc34dSpatrick #include <dev/ic/ahcireg.h> 3406133b5aSdlg #include <dev/ic/ahcivar.h> 357acdc34dSpatrick 361b1d835dSjmatthew #ifdef AHCI_DEBUG 371b1d835dSjmatthew #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 381b1d835dSjmatthew while (0) 391b1d835dSjmatthew #define AHCI_D_TIMEOUT 0x00 401b1d835dSjmatthew #define AHCI_D_VERBOSE 0x01 411b1d835dSjmatthew #define AHCI_D_INTR 0x02 421b1d835dSjmatthew #define AHCI_D_XFER 0x08 431b1d835dSjmatthew int ahcidebug = AHCI_D_VERBOSE; 441b1d835dSjmatthew #else 451b1d835dSjmatthew #define DPRINTF(m, f...) 461b1d835dSjmatthew #endif 471b1d835dSjmatthew 4876d3d104Sderaadt #ifdef HIBERNATE 494bb7a4f6Smpi #include <uvm/uvm_extern.h> 5076d3d104Sderaadt #include <sys/hibernate.h> 5176d3d104Sderaadt #include <sys/disk.h> 5276d3d104Sderaadt #include <sys/disklabel.h> 5376d3d104Sderaadt 5476d3d104Sderaadt #include <scsi/scsi_all.h> 5576d3d104Sderaadt #include <scsi/scsiconf.h> 561b1d835dSjmatthew 577acdc34dSpatrick void ahci_hibernate_io_start(struct ahci_port *, 587acdc34dSpatrick struct ahci_ccb *); 597acdc34dSpatrick int ahci_hibernate_io_poll(struct ahci_port *, 607acdc34dSpatrick struct ahci_ccb *); 617acdc34dSpatrick void ahci_hibernate_load_prdt(struct ahci_ccb *); 627acdc34dSpatrick 637acdc34dSpatrick int ahci_hibernate_io(dev_t dev, daddr_t blkno, 647acdc34dSpatrick vaddr_t addr, size_t size, int wr, void *page); 657acdc34dSpatrick #endif 667acdc34dSpatrick 677acdc34dSpatrick struct cfdriver ahci_cd = { 687acdc34dSpatrick NULL, "ahci", DV_DULL 697acdc34dSpatrick }; 707acdc34dSpatrick 717acdc34dSpatrick void ahci_enable_interrupts(struct ahci_port *); 727acdc34dSpatrick 737acdc34dSpatrick int ahci_init(struct ahci_softc *); 747acdc34dSpatrick int ahci_port_alloc(struct ahci_softc *, u_int); 7596739e86Sjmatthew void ahci_port_detect(struct ahci_softc *, u_int); 767acdc34dSpatrick void ahci_port_free(struct ahci_softc *, u_int); 777acdc34dSpatrick int ahci_port_init(struct ahci_softc *, u_int); 787acdc34dSpatrick 7993c0f120Sjsg int ahci_default_port_start(struct ahci_port *, int); 807acdc34dSpatrick int ahci_port_stop(struct ahci_port *, int); 817acdc34dSpatrick int ahci_port_clo(struct ahci_port *); 827acdc34dSpatrick int ahci_port_softreset(struct ahci_port *); 8396739e86Sjmatthew void ahci_port_comreset(struct ahci_port *); 847acdc34dSpatrick int ahci_port_portreset(struct ahci_port *, int); 8596739e86Sjmatthew void ahci_port_portreset_start(struct ahci_port *); 8696739e86Sjmatthew int ahci_port_portreset_poll(struct ahci_port *); 8796739e86Sjmatthew void ahci_port_portreset_wait(struct ahci_port *); 8896739e86Sjmatthew int ahci_port_portreset_finish(struct ahci_port *, int); 897acdc34dSpatrick int ahci_port_signature(struct ahci_port *); 907acdc34dSpatrick int ahci_pmp_port_softreset(struct ahci_port *, int); 917acdc34dSpatrick int ahci_pmp_port_portreset(struct ahci_port *, int); 927acdc34dSpatrick int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 937acdc34dSpatrick 947acdc34dSpatrick int ahci_load_prdt(struct ahci_ccb *); 95b1514dfeSdlg void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 96b1514dfeSdlg u_int32_t, u_int32_t); 977acdc34dSpatrick void ahci_unload_prdt(struct ahci_ccb *); 987acdc34dSpatrick 997acdc34dSpatrick int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 1007acdc34dSpatrick void ahci_start(struct ahci_ccb *); 1017acdc34dSpatrick 1027acdc34dSpatrick void ahci_issue_pending_ncq_commands(struct ahci_port *); 1037acdc34dSpatrick void ahci_issue_pending_commands(struct ahci_port *, int); 1047acdc34dSpatrick 1057acdc34dSpatrick int ahci_intr(void *); 1067acdc34dSpatrick u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 1077acdc34dSpatrick 1087acdc34dSpatrick struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 1097acdc34dSpatrick void ahci_put_ccb(struct ahci_ccb *); 1107acdc34dSpatrick 1117acdc34dSpatrick struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 1127acdc34dSpatrick void ahci_put_err_ccb(struct ahci_ccb *); 1137acdc34dSpatrick 1147acdc34dSpatrick struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 1157acdc34dSpatrick void ahci_put_pmp_ccb(struct ahci_ccb *); 1167acdc34dSpatrick 1177acdc34dSpatrick int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 1187acdc34dSpatrick 1197acdc34dSpatrick struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 1207acdc34dSpatrick void ahci_dmamem_free(struct ahci_softc *, 1217acdc34dSpatrick struct ahci_dmamem *); 1227acdc34dSpatrick 1237acdc34dSpatrick u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 1247acdc34dSpatrick void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 1257acdc34dSpatrick int ahci_wait_ne(struct ahci_softc *, bus_size_t, 1267acdc34dSpatrick u_int32_t, u_int32_t); 1277acdc34dSpatrick 1287acdc34dSpatrick u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 1297acdc34dSpatrick void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 1307acdc34dSpatrick int ahci_pwait_eq(struct ahci_port *, bus_size_t, 1317acdc34dSpatrick u_int32_t, u_int32_t, int); 1327acdc34dSpatrick void ahci_flush_tfd(struct ahci_port *ap); 1337acdc34dSpatrick u_int32_t ahci_active_mask(struct ahci_port *); 1347acdc34dSpatrick int ahci_port_detect_pmp(struct ahci_port *); 1357acdc34dSpatrick void ahci_pmp_probe_timeout(void *); 1367acdc34dSpatrick 1377acdc34dSpatrick /* pmp operations */ 1387acdc34dSpatrick int ahci_pmp_read(struct ahci_port *, int, int, 1397acdc34dSpatrick u_int32_t *); 1407acdc34dSpatrick int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 1417acdc34dSpatrick int ahci_pmp_phy_status(struct ahci_port *, int, 1427acdc34dSpatrick u_int32_t *); 1437acdc34dSpatrick int ahci_pmp_identify(struct ahci_port *, int *); 1447acdc34dSpatrick 1457acdc34dSpatrick 1467acdc34dSpatrick /* Wait for all bits in _b to be cleared */ 1477acdc34dSpatrick #define ahci_pwait_clr(_ap, _r, _b, _n) \ 1487acdc34dSpatrick ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 1497acdc34dSpatrick 1507acdc34dSpatrick /* Wait for all bits in _b to be set */ 1517acdc34dSpatrick #define ahci_pwait_set(_ap, _r, _b, _n) \ 1527acdc34dSpatrick ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 1537acdc34dSpatrick 1547acdc34dSpatrick 1557acdc34dSpatrick 1567acdc34dSpatrick /* provide methods for atascsi to call */ 1577acdc34dSpatrick int ahci_ata_probe(void *, int, int); 1587acdc34dSpatrick void ahci_ata_free(void *, int, int); 1597acdc34dSpatrick struct ata_xfer * ahci_ata_get_xfer(void *, int); 1607acdc34dSpatrick void ahci_ata_put_xfer(struct ata_xfer *); 1617acdc34dSpatrick void ahci_ata_cmd(struct ata_xfer *); 1627acdc34dSpatrick 16351d3b00dSnaddy const struct atascsi_methods ahci_atascsi_methods = { 1647acdc34dSpatrick ahci_ata_probe, 1657acdc34dSpatrick ahci_ata_free, 1667acdc34dSpatrick ahci_ata_get_xfer, 1677acdc34dSpatrick ahci_ata_put_xfer, 1687acdc34dSpatrick ahci_ata_cmd 1697acdc34dSpatrick }; 1707acdc34dSpatrick 1717acdc34dSpatrick /* ccb completions */ 1727acdc34dSpatrick void ahci_ata_cmd_done(struct ahci_ccb *); 1737acdc34dSpatrick void ahci_pmp_cmd_done(struct ahci_ccb *); 1747acdc34dSpatrick void ahci_ata_cmd_timeout(void *); 1757acdc34dSpatrick void ahci_empty_done(struct ahci_ccb *); 1767acdc34dSpatrick 1777acdc34dSpatrick int 1787acdc34dSpatrick ahci_attach(struct ahci_softc *sc) 1797acdc34dSpatrick { 1807acdc34dSpatrick struct atascsi_attach_args aaa; 1817acdc34dSpatrick u_int32_t pi; 18296739e86Sjmatthew int i, j, done; 1837acdc34dSpatrick 18493c0f120Sjsg if (sc->sc_port_start == NULL) 18593c0f120Sjsg sc->sc_port_start = ahci_default_port_start; 18693c0f120Sjsg 1877acdc34dSpatrick if (ahci_init(sc) != 0) { 1887acdc34dSpatrick /* error already printed by ahci_init */ 1897acdc34dSpatrick goto unmap; 1907acdc34dSpatrick } 1917acdc34dSpatrick 1927acdc34dSpatrick printf("\n"); 1937acdc34dSpatrick 1947acdc34dSpatrick sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 1957acdc34dSpatrick sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 1967acdc34dSpatrick #ifdef AHCI_DEBUG 1977acdc34dSpatrick if (ahcidebug & AHCI_D_VERBOSE) { 1987acdc34dSpatrick const char *gen; 1997acdc34dSpatrick 2007acdc34dSpatrick switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 2017acdc34dSpatrick case AHCI_REG_CAP_ISS_G1: 2027acdc34dSpatrick gen = "1 (1.5Gbps)"; 2037acdc34dSpatrick break; 20471e33dbaSpelikan case AHCI_REG_CAP_ISS_G2: 2059cd47e4cSmpi gen = "2 (3.0Gb/s)"; 206b049ffa5Spelikan break; 20771e33dbaSpelikan case AHCI_REG_CAP_ISS_G3: 2089cd47e4cSmpi gen = "3 (6.0Gb/s)"; 2097acdc34dSpatrick break; 2107acdc34dSpatrick default: 2117acdc34dSpatrick gen = "unknown"; 2127acdc34dSpatrick break; 2137acdc34dSpatrick } 2147acdc34dSpatrick 2157acdc34dSpatrick printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 2167acdc34dSpatrick DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 2177acdc34dSpatrick AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 2181ec8db95Sjmatthew printf("%s: extended capabilities 0x%b\n", DEVNAME(sc), 2191ec8db95Sjmatthew ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2); 2207acdc34dSpatrick } 2217acdc34dSpatrick #endif 2227acdc34dSpatrick 2237acdc34dSpatrick pi = ahci_read(sc, AHCI_REG_PI); 2247acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 2257acdc34dSpatrick DEVNAME(sc), pi); 2267acdc34dSpatrick 2277acdc34dSpatrick #ifdef AHCI_COALESCE 2287acdc34dSpatrick /* Naive coalescing support - enable for all ports. */ 2297acdc34dSpatrick if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 2307acdc34dSpatrick u_int16_t ccc_timeout = 20; 2317acdc34dSpatrick u_int8_t ccc_numcomplete = 12; 2327acdc34dSpatrick u_int32_t ccc_ctl; 2337acdc34dSpatrick 2347acdc34dSpatrick /* disable coalescing during reconfiguration. */ 2357acdc34dSpatrick ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 2367acdc34dSpatrick ccc_ctl &= ~0x00000001; 2377acdc34dSpatrick ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 2387acdc34dSpatrick 2397acdc34dSpatrick sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 2407acdc34dSpatrick if (pi & sc->sc_ccc_mask) { 2417acdc34dSpatrick /* A conflict with the implemented port list? */ 2427acdc34dSpatrick printf("%s: coalescing interrupt/implemented port list " 2437acdc34dSpatrick "conflict, PI: %08x, ccc_mask: %08x\n", 2447acdc34dSpatrick DEVNAME(sc), pi, sc->sc_ccc_mask); 2457acdc34dSpatrick sc->sc_ccc_mask = 0; 2467acdc34dSpatrick goto noccc; 2477acdc34dSpatrick } 2487acdc34dSpatrick 2497acdc34dSpatrick /* ahci_port_start will enable each port when it starts. */ 2507acdc34dSpatrick sc->sc_ccc_ports = pi; 2517acdc34dSpatrick sc->sc_ccc_ports_cur = 0; 2527acdc34dSpatrick 2537acdc34dSpatrick /* program thresholds and enable overall coalescing. */ 2547acdc34dSpatrick ccc_ctl &= ~0xffffff00; 2557acdc34dSpatrick ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 2567acdc34dSpatrick ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 2577acdc34dSpatrick ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 2587acdc34dSpatrick ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 2597acdc34dSpatrick } 2607acdc34dSpatrick noccc: 2617acdc34dSpatrick #endif 2629ae3568aSpatrick /* 2639ae3568aSpatrick * Given that ahci_port_alloc() will grab one CCB for error recovery 2649ae3568aSpatrick * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds 2659ae3568aSpatrick * pretend at least 2 command slots for devices without NCQ support. 2669ae3568aSpatrick * That way, also at least 1 slot is made available for atascsi(4). 2679ae3568aSpatrick */ 2689ae3568aSpatrick sc->sc_ncmds = max(2, sc->sc_ncmds); 2697acdc34dSpatrick for (i = 0; i < AHCI_MAX_PORTS; i++) { 2706b06da21Sjsg if (!ISSET(pi, 1U << i)) { 2719593dc34Smglocker /* don't allocate stuff if the port isn't implemented */ 2727acdc34dSpatrick continue; 2737acdc34dSpatrick } 2747acdc34dSpatrick 2757acdc34dSpatrick if (ahci_port_alloc(sc, i) == ENOMEM) 2767acdc34dSpatrick goto freeports; 27796739e86Sjmatthew 27896739e86Sjmatthew if (sc->sc_ports[i] != NULL) 27996739e86Sjmatthew ahci_port_portreset_start(sc->sc_ports[i]); 28096739e86Sjmatthew } 28196739e86Sjmatthew 28296739e86Sjmatthew /* 28396739e86Sjmatthew * Poll for device detection until all ports report a device, or one 28496739e86Sjmatthew * second has elapsed. 28596739e86Sjmatthew */ 28696739e86Sjmatthew for (i = 0; i < 1000; i++) { 28796739e86Sjmatthew done = 1; 28896739e86Sjmatthew for (j = 0; j < AHCI_MAX_PORTS; j++) { 28996739e86Sjmatthew if (sc->sc_ports[j] == NULL) 29096739e86Sjmatthew continue; 29196739e86Sjmatthew 29296739e86Sjmatthew if (ahci_port_portreset_poll(sc->sc_ports[j])) 29396739e86Sjmatthew done = 0; 29496739e86Sjmatthew } 29596739e86Sjmatthew 29696739e86Sjmatthew if (done) 29796739e86Sjmatthew break; 29896739e86Sjmatthew 29996739e86Sjmatthew delay(1000); 30096739e86Sjmatthew } 30196739e86Sjmatthew 30296739e86Sjmatthew /* 30396739e86Sjmatthew * Finish device detection on all ports that initialized. 30496739e86Sjmatthew */ 30596739e86Sjmatthew for (i = 0; i < AHCI_MAX_PORTS; i++) { 30696739e86Sjmatthew if (sc->sc_ports[i] != NULL) 30796739e86Sjmatthew ahci_port_detect(sc, i); 3087acdc34dSpatrick } 3097acdc34dSpatrick 310414fa24dSdlg memset(&aaa, 0, sizeof(aaa)); 3117acdc34dSpatrick aaa.aaa_cookie = sc; 3127acdc34dSpatrick aaa.aaa_methods = &ahci_atascsi_methods; 3137acdc34dSpatrick aaa.aaa_minphys = NULL; 3147acdc34dSpatrick aaa.aaa_nports = AHCI_MAX_PORTS; 3157acdc34dSpatrick aaa.aaa_ncmds = sc->sc_ncmds - 1; 3167acdc34dSpatrick if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 317fd58cd83Spatrick sc->sc_ncmds > 2 && 3187acdc34dSpatrick (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 3197acdc34dSpatrick aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 3207acdc34dSpatrick } 3217acdc34dSpatrick 3227acdc34dSpatrick sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 3237acdc34dSpatrick 324*1cfc6e0bSjan /* Flush all residual bits of the interrupt status register */ 325*1cfc6e0bSjan ahci_write(sc, AHCI_REG_IS, ahci_read(sc, AHCI_REG_IS)); 326*1cfc6e0bSjan 3277acdc34dSpatrick /* Enable interrupts */ 3287acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 3297acdc34dSpatrick 3307acdc34dSpatrick return 0; 3317acdc34dSpatrick 3327acdc34dSpatrick freeports: 3337acdc34dSpatrick for (i = 0; i < AHCI_MAX_PORTS; i++) { 3347acdc34dSpatrick if (sc->sc_ports[i] != NULL) 3357acdc34dSpatrick ahci_port_free(sc, i); 3367acdc34dSpatrick } 3377acdc34dSpatrick unmap: 3387acdc34dSpatrick /* Disable controller */ 3397acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, 0); 3407acdc34dSpatrick return 1; 3417acdc34dSpatrick } 3427acdc34dSpatrick 3437acdc34dSpatrick int 3447acdc34dSpatrick ahci_detach(struct ahci_softc *sc, int flags) 3457acdc34dSpatrick { 3467acdc34dSpatrick int rv, i; 3477acdc34dSpatrick 3487acdc34dSpatrick if (sc->sc_atascsi != NULL) { 3497acdc34dSpatrick rv = atascsi_detach(sc->sc_atascsi, flags); 3507acdc34dSpatrick if (rv != 0) 3517acdc34dSpatrick return (rv); 3527acdc34dSpatrick } 3537acdc34dSpatrick 3547acdc34dSpatrick for (i = 0; i < AHCI_MAX_PORTS; i++) { 3557acdc34dSpatrick if (sc->sc_ports[i] != NULL) 3567acdc34dSpatrick ahci_port_free(sc, i); 3577acdc34dSpatrick } 3587acdc34dSpatrick 3597acdc34dSpatrick return (0); 3607acdc34dSpatrick } 3617acdc34dSpatrick 3627acdc34dSpatrick int 3637acdc34dSpatrick ahci_activate(struct device *self, int act) 3647acdc34dSpatrick { 3657acdc34dSpatrick struct ahci_softc *sc = (struct ahci_softc *)self; 3667acdc34dSpatrick int i, rv = 0; 3677acdc34dSpatrick 3687acdc34dSpatrick switch (act) { 3697acdc34dSpatrick case DVACT_RESUME: 3707acdc34dSpatrick /* enable ahci (global interrupts disabled) */ 3717acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 3727acdc34dSpatrick 3737acdc34dSpatrick /* restore BIOS initialised parameters */ 3747acdc34dSpatrick ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 3757acdc34dSpatrick 3767acdc34dSpatrick for (i = 0; i < AHCI_MAX_PORTS; i++) { 3777acdc34dSpatrick if (sc->sc_ports[i] != NULL) 3787acdc34dSpatrick ahci_port_init(sc, i); 3797acdc34dSpatrick } 3807acdc34dSpatrick 3817acdc34dSpatrick /* Enable interrupts */ 3827acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 3837acdc34dSpatrick 3847acdc34dSpatrick rv = config_activate_children(self, act); 3857acdc34dSpatrick break; 38637ecb596Sderaadt case DVACT_POWERDOWN: 38737ecb596Sderaadt rv = config_activate_children(self, act); 38837ecb596Sderaadt for (i = 0; i < AHCI_MAX_PORTS; i++) { 38937ecb596Sderaadt if (sc->sc_ports[i] != NULL) 39037ecb596Sderaadt ahci_port_stop(sc->sc_ports[i], 1); 39137ecb596Sderaadt } 39237ecb596Sderaadt break; 39337ecb596Sderaadt default: 39437ecb596Sderaadt rv = config_activate_children(self, act); 39537ecb596Sderaadt break; 3967acdc34dSpatrick } 3977acdc34dSpatrick return (rv); 3987acdc34dSpatrick } 3997acdc34dSpatrick 4007acdc34dSpatrick int 4017acdc34dSpatrick ahci_init(struct ahci_softc *sc) 4027acdc34dSpatrick { 4037acdc34dSpatrick u_int32_t reg, cap, pi; 4047acdc34dSpatrick const char *revision; 4057acdc34dSpatrick 4067acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 4077acdc34dSpatrick AHCI_FMT_GHC); 4087acdc34dSpatrick 4097acdc34dSpatrick /* save BIOS initialised parameters, enable staggered spin up */ 4107acdc34dSpatrick cap = ahci_read(sc, AHCI_REG_CAP); 4117acdc34dSpatrick cap &= AHCI_REG_CAP_SMPS; 4127acdc34dSpatrick cap |= AHCI_REG_CAP_SSS; 4137acdc34dSpatrick pi = ahci_read(sc, AHCI_REG_PI); 4147acdc34dSpatrick 4157acdc34dSpatrick if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 4167acdc34dSpatrick /* reset the controller */ 4177acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 4187acdc34dSpatrick if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 4197acdc34dSpatrick AHCI_REG_GHC_HR) != 0) { 4207acdc34dSpatrick printf(" unable to reset controller\n"); 4217acdc34dSpatrick return (1); 4227acdc34dSpatrick } 4237acdc34dSpatrick } 4247acdc34dSpatrick 4257acdc34dSpatrick /* enable ahci (global interrupts disabled) */ 4267acdc34dSpatrick ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 4277acdc34dSpatrick 4287acdc34dSpatrick /* restore parameters */ 4297acdc34dSpatrick ahci_write(sc, AHCI_REG_CAP, cap); 4307acdc34dSpatrick ahci_write(sc, AHCI_REG_PI, pi); 4317acdc34dSpatrick 4327acdc34dSpatrick /* check the revision */ 4337acdc34dSpatrick reg = ahci_read(sc, AHCI_REG_VS); 4347acdc34dSpatrick switch (reg) { 4357acdc34dSpatrick case AHCI_REG_VS_0_95: 4367acdc34dSpatrick revision = "0.95"; 4377acdc34dSpatrick break; 4387acdc34dSpatrick case AHCI_REG_VS_1_0: 4397acdc34dSpatrick revision = "1.0"; 4407acdc34dSpatrick break; 4417acdc34dSpatrick case AHCI_REG_VS_1_1: 4427acdc34dSpatrick revision = "1.1"; 4437acdc34dSpatrick break; 4447acdc34dSpatrick case AHCI_REG_VS_1_2: 4457acdc34dSpatrick revision = "1.2"; 4467acdc34dSpatrick break; 4477acdc34dSpatrick case AHCI_REG_VS_1_3: 4487acdc34dSpatrick revision = "1.3"; 4497acdc34dSpatrick break; 450a4da9f86Sjmatthew case AHCI_REG_VS_1_3_1: 451a4da9f86Sjmatthew revision = "1.3.1"; 452a4da9f86Sjmatthew break; 4537acdc34dSpatrick 4547acdc34dSpatrick default: 4557acdc34dSpatrick printf(" unsupported AHCI revision 0x%08x\n", reg); 4567acdc34dSpatrick return (1); 4577acdc34dSpatrick } 4587acdc34dSpatrick 4597acdc34dSpatrick printf(" AHCI %s", revision); 4607acdc34dSpatrick 4617acdc34dSpatrick return (0); 4627acdc34dSpatrick } 4637acdc34dSpatrick 4647acdc34dSpatrick void 4657acdc34dSpatrick ahci_enable_interrupts(struct ahci_port *ap) 4667acdc34dSpatrick { 4677acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 4687acdc34dSpatrick AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 4697acdc34dSpatrick AHCI_PREG_IE_UFE | 4707acdc34dSpatrick ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 4717acdc34dSpatrick #ifdef AHCI_COALESCE 4727acdc34dSpatrick ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 4737acdc34dSpatrick (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 4747acdc34dSpatrick #else 4757acdc34dSpatrick AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 4767acdc34dSpatrick #endif 4777acdc34dSpatrick ); 4787acdc34dSpatrick } 4797acdc34dSpatrick 4807acdc34dSpatrick int 4817acdc34dSpatrick ahci_port_alloc(struct ahci_softc *sc, u_int port) 4827acdc34dSpatrick { 4837acdc34dSpatrick struct ahci_port *ap; 4847acdc34dSpatrick struct ahci_ccb *ccb; 4857acdc34dSpatrick u_int64_t dva; 4867acdc34dSpatrick u_int32_t cmd; 4877acdc34dSpatrick struct ahci_cmd_hdr *hdr; 4887acdc34dSpatrick struct ahci_cmd_table *table; 4897acdc34dSpatrick int i, rc = ENOMEM; 4907acdc34dSpatrick 4917acdc34dSpatrick ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 4927acdc34dSpatrick if (ap == NULL) { 4937acdc34dSpatrick printf("%s: unable to allocate memory for port %d\n", 4947acdc34dSpatrick DEVNAME(sc), port); 4957acdc34dSpatrick goto reterr; 4967acdc34dSpatrick } 4977acdc34dSpatrick ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 4987acdc34dSpatrick if (ap->ap_err_scratch == NULL) { 4997acdc34dSpatrick printf("%s: unable to allocate DMA scratch buf for port %d\n", 5007acdc34dSpatrick DEVNAME(sc), port); 5014ba7bf3bSjmatthew free(ap, M_DEVBUF, sizeof(*ap)); 5024ba7bf3bSjmatthew goto reterr; 5037acdc34dSpatrick } 5047acdc34dSpatrick 5057acdc34dSpatrick #ifdef AHCI_DEBUG 5067acdc34dSpatrick snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 5077acdc34dSpatrick DEVNAME(sc), port); 5087acdc34dSpatrick #endif 5097acdc34dSpatrick ap->ap_port = port; 5107acdc34dSpatrick sc->sc_ports[port] = ap; 5117acdc34dSpatrick 5127acdc34dSpatrick if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 5137acdc34dSpatrick AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 5147acdc34dSpatrick printf("%s: unable to create register window for port %d\n", 5157acdc34dSpatrick DEVNAME(sc), port); 5167acdc34dSpatrick goto freeport; 5177acdc34dSpatrick } 5187acdc34dSpatrick 5197acdc34dSpatrick ap->ap_sc = sc; 5207acdc34dSpatrick #ifdef AHCI_COALESCE 5217acdc34dSpatrick ap->ap_num = port; 5227acdc34dSpatrick #endif 5237acdc34dSpatrick TAILQ_INIT(&ap->ap_ccb_free); 5247acdc34dSpatrick TAILQ_INIT(&ap->ap_ccb_pending); 5257acdc34dSpatrick mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 5267acdc34dSpatrick 5277acdc34dSpatrick /* Disable port interrupts */ 5287acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IE, 0); 5297acdc34dSpatrick 5307acdc34dSpatrick /* Sec 10.1.2 - deinitialise port if it is already running */ 5317acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD); 5327acdc34dSpatrick if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 5337acdc34dSpatrick AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 5347acdc34dSpatrick ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 5357acdc34dSpatrick int r; 5367acdc34dSpatrick 5377acdc34dSpatrick r = ahci_port_stop(ap, 1); 5387acdc34dSpatrick if (r) { 5397acdc34dSpatrick printf("%s: unable to disable %s, ignoring port %d\n", 5407acdc34dSpatrick DEVNAME(sc), r == 2 ? "CR" : "FR", port); 5417acdc34dSpatrick rc = ENXIO; 5427acdc34dSpatrick goto freeport; 5437acdc34dSpatrick } 5447acdc34dSpatrick 5457acdc34dSpatrick /* Write DET to zero */ 5467acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 5477acdc34dSpatrick } 5487acdc34dSpatrick 5497acdc34dSpatrick /* Allocate RFIS */ 5507acdc34dSpatrick ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 5517acdc34dSpatrick if (ap->ap_dmamem_rfis == NULL) 5527acdc34dSpatrick goto nomem; 5537acdc34dSpatrick 5547acdc34dSpatrick /* Setup RFIS base address */ 5557acdc34dSpatrick ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 5567acdc34dSpatrick dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 5577acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 5587acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 5597acdc34dSpatrick 5607acdc34dSpatrick /* Enable FIS reception and activate port. */ 5617acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 5627acdc34dSpatrick cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 5637acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 5647acdc34dSpatrick 5657acdc34dSpatrick /* Check whether port activated. Skip it if not. */ 5667acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 5677acdc34dSpatrick if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 5687acdc34dSpatrick rc = ENXIO; 5697acdc34dSpatrick goto freeport; 5707acdc34dSpatrick } 5717acdc34dSpatrick 5727acdc34dSpatrick /* Allocate a CCB for each command slot */ 5739f6fb5c7Sderaadt ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 5749f6fb5c7Sderaadt M_DEVBUF, M_NOWAIT | M_ZERO); 5757acdc34dSpatrick if (ap->ap_ccbs == NULL) { 5767acdc34dSpatrick printf("%s: unable to allocate command list for port %d\n", 5777acdc34dSpatrick DEVNAME(sc), port); 5787acdc34dSpatrick goto freeport; 5797acdc34dSpatrick } 5807acdc34dSpatrick 5817acdc34dSpatrick /* Command List Structures and Command Tables */ 5827acdc34dSpatrick ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 5837acdc34dSpatrick sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 5847acdc34dSpatrick ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 5857acdc34dSpatrick sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 5867acdc34dSpatrick if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 5877acdc34dSpatrick nomem: 5887acdc34dSpatrick printf("%s: unable to allocate DMA memory for port %d\n", 5897acdc34dSpatrick DEVNAME(sc), port); 5907acdc34dSpatrick goto freeport; 5917acdc34dSpatrick } 5927acdc34dSpatrick 5937acdc34dSpatrick /* Setup command list base address */ 5947acdc34dSpatrick dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 5957acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 5967acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 5977acdc34dSpatrick 5987acdc34dSpatrick /* Split CCB allocation into CCBs and assign to command header/table */ 5997acdc34dSpatrick hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 6007acdc34dSpatrick table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 6017acdc34dSpatrick for (i = 0; i < sc->sc_ncmds; i++) { 6027acdc34dSpatrick ccb = &ap->ap_ccbs[i]; 6037acdc34dSpatrick 6047acdc34dSpatrick if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 6057acdc34dSpatrick (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 6067acdc34dSpatrick &ccb->ccb_dmamap) != 0) { 6077acdc34dSpatrick printf("%s: unable to create dmamap for port %d " 6087acdc34dSpatrick "ccb %d\n", DEVNAME(sc), port, i); 6097acdc34dSpatrick goto freeport; 6107acdc34dSpatrick } 6117acdc34dSpatrick 6127acdc34dSpatrick ccb->ccb_slot = i; 6137acdc34dSpatrick ccb->ccb_port = ap; 6147acdc34dSpatrick ccb->ccb_cmd_hdr = &hdr[i]; 6157acdc34dSpatrick ccb->ccb_cmd_table = &table[i]; 6164d3a7877Sdlg htolem64(&ccb->ccb_cmd_hdr->ctba, 6174d3a7877Sdlg AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 6184d3a7877Sdlg ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 6197acdc34dSpatrick 6207acdc34dSpatrick ccb->ccb_xa.fis = 6217acdc34dSpatrick (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 6227acdc34dSpatrick ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 6237acdc34dSpatrick ccb->ccb_xa.tag = i; 6247acdc34dSpatrick 6257acdc34dSpatrick ccb->ccb_xa.state = ATA_S_COMPLETE; 6267acdc34dSpatrick ahci_put_ccb(ccb); 6277acdc34dSpatrick } 6287acdc34dSpatrick 6297acdc34dSpatrick /* grab a ccb for use during error recovery */ 6307acdc34dSpatrick ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 6317acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 6327acdc34dSpatrick ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 6337acdc34dSpatrick 6347acdc34dSpatrick /* Wait for ICC change to complete */ 6357acdc34dSpatrick ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 63696739e86Sjmatthew rc = 0; 6377acdc34dSpatrick 63896739e86Sjmatthew freeport: 63996739e86Sjmatthew if (rc != 0) 64096739e86Sjmatthew ahci_port_free(sc, port); 64196739e86Sjmatthew reterr: 64296739e86Sjmatthew return (rc); 64396739e86Sjmatthew } 6447acdc34dSpatrick 64596739e86Sjmatthew void 64696739e86Sjmatthew ahci_port_detect(struct ahci_softc *sc, u_int port) 64796739e86Sjmatthew { 64896739e86Sjmatthew struct ahci_port *ap; 64996739e86Sjmatthew const char *speed; 65096739e86Sjmatthew int rc; 65196739e86Sjmatthew 65296739e86Sjmatthew ap = sc->sc_ports[port]; 65396739e86Sjmatthew 65496739e86Sjmatthew rc = ahci_port_portreset_finish(ap, 1); 6557acdc34dSpatrick switch (rc) { 6567acdc34dSpatrick case ENODEV: 6577acdc34dSpatrick switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 6587acdc34dSpatrick case AHCI_PREG_SSTS_DET_DEV_NE: 6597acdc34dSpatrick printf("%s: device not communicating on port %d\n", 6607acdc34dSpatrick DEVNAME(sc), port); 6617acdc34dSpatrick break; 6627acdc34dSpatrick case AHCI_PREG_SSTS_DET_PHYOFFLINE: 6637acdc34dSpatrick printf("%s: PHY offline on port %d\n", DEVNAME(sc), 6647acdc34dSpatrick port); 6657acdc34dSpatrick break; 6667acdc34dSpatrick default: 6677acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 6687acdc34dSpatrick "on port %d\n", DEVNAME(sc), port); 6697acdc34dSpatrick break; 6707acdc34dSpatrick } 6717acdc34dSpatrick goto freeport; 6727acdc34dSpatrick 6737acdc34dSpatrick case EBUSY: 6747acdc34dSpatrick printf("%s: device on port %d didn't come ready, " 6757acdc34dSpatrick "TFD: 0x%b\n", DEVNAME(sc), port, 6767acdc34dSpatrick ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 6777acdc34dSpatrick 6787acdc34dSpatrick /* Try a soft reset to clear busy */ 6797acdc34dSpatrick rc = ahci_port_softreset(ap); 6807acdc34dSpatrick if (rc) { 6817acdc34dSpatrick printf("%s: unable to communicate " 6827acdc34dSpatrick "with device on port %d\n", DEVNAME(sc), port); 6837acdc34dSpatrick goto freeport; 6847acdc34dSpatrick } 6857acdc34dSpatrick break; 6867acdc34dSpatrick 6877acdc34dSpatrick default: 6887acdc34dSpatrick break; 6897acdc34dSpatrick } 6907acdc34dSpatrick 6917acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 6927acdc34dSpatrick DEVNAME(sc), port, rc); 6937acdc34dSpatrick 69417fb629fSbrynet /* Read current link speed */ 69517fb629fSbrynet switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { 69617fb629fSbrynet case AHCI_PREG_SSTS_SPD_GEN1: 6979cd47e4cSmpi speed = "1.5Gb/s"; 69817fb629fSbrynet break; 69917fb629fSbrynet case AHCI_PREG_SSTS_SPD_GEN2: 7009cd47e4cSmpi speed = "3.0Gb/s"; 70117fb629fSbrynet break; 70217fb629fSbrynet case AHCI_PREG_SSTS_SPD_GEN3: 7039cd47e4cSmpi speed = "6.0Gb/s"; 70417fb629fSbrynet break; 70517fb629fSbrynet default: 70617fb629fSbrynet speed = NULL; 70717fb629fSbrynet break; 70817fb629fSbrynet } 70917fb629fSbrynet if (speed != NULL) 7109cd47e4cSmpi printf("%s: port %d: %s\n", PORTNAME(ap), port, speed); 71117fb629fSbrynet 7127acdc34dSpatrick /* Enable command transfers on port */ 7137acdc34dSpatrick if (ahci_port_start(ap, 0)) { 7147acdc34dSpatrick printf("%s: failed to start command DMA on port %d, " 7157acdc34dSpatrick "disabling\n", DEVNAME(sc), port); 7167acdc34dSpatrick rc = ENXIO; /* couldn't start port */ 7177acdc34dSpatrick } 7187acdc34dSpatrick 7197acdc34dSpatrick /* Flush interrupts for port */ 7207acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 7217acdc34dSpatrick ahci_write(sc, AHCI_REG_IS, 1 << port); 7227acdc34dSpatrick 7237acdc34dSpatrick ahci_enable_interrupts(ap); 7247acdc34dSpatrick freeport: 7257acdc34dSpatrick if (rc != 0) 7267acdc34dSpatrick ahci_port_free(sc, port); 7277acdc34dSpatrick } 7287acdc34dSpatrick 7297acdc34dSpatrick void 7307acdc34dSpatrick ahci_port_free(struct ahci_softc *sc, u_int port) 7317acdc34dSpatrick { 7327acdc34dSpatrick struct ahci_port *ap = sc->sc_ports[port]; 7337acdc34dSpatrick struct ahci_ccb *ccb; 7347acdc34dSpatrick 7357acdc34dSpatrick /* Ensure port is disabled and its interrupts are flushed */ 7367acdc34dSpatrick if (ap->ap_sc) { 7377acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, 0); 7387acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IE, 0); 7397acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 7407acdc34dSpatrick ahci_write(sc, AHCI_REG_IS, 1 << port); 7417acdc34dSpatrick } 7427acdc34dSpatrick 7437acdc34dSpatrick if (ap->ap_ccb_err) 7447acdc34dSpatrick ahci_put_ccb(ap->ap_ccb_err); 7457acdc34dSpatrick 7467acdc34dSpatrick if (ap->ap_ccbs) { 7477acdc34dSpatrick while ((ccb = ahci_get_ccb(ap)) != NULL) 7487acdc34dSpatrick bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 7491f559dcaSderaadt free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb)); 7507acdc34dSpatrick } 7517acdc34dSpatrick 7527acdc34dSpatrick if (ap->ap_dmamem_cmd_list) 7537acdc34dSpatrick ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 7547acdc34dSpatrick if (ap->ap_dmamem_rfis) 7557acdc34dSpatrick ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 7567acdc34dSpatrick if (ap->ap_dmamem_cmd_table) 7577acdc34dSpatrick ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 7587acdc34dSpatrick if (ap->ap_err_scratch) 7597acdc34dSpatrick dma_free(ap->ap_err_scratch, DEV_BSIZE); 7607acdc34dSpatrick 7617acdc34dSpatrick /* bus_space(9) says we dont free the subregions handle */ 7627acdc34dSpatrick 7631f559dcaSderaadt free(ap, M_DEVBUF, sizeof(*ap)); 7647acdc34dSpatrick sc->sc_ports[port] = NULL; 7657acdc34dSpatrick } 7667acdc34dSpatrick 7677acdc34dSpatrick int 7687acdc34dSpatrick ahci_port_init(struct ahci_softc *sc, u_int port) 7697acdc34dSpatrick { 7707acdc34dSpatrick struct ahci_port *ap; 7717acdc34dSpatrick u_int64_t dva; 7727acdc34dSpatrick u_int32_t cmd; 7737acdc34dSpatrick int rc = ENOMEM; 7747acdc34dSpatrick 7757acdc34dSpatrick ap = sc->sc_ports[port]; 7767acdc34dSpatrick #ifdef AHCI_DEBUG 7777acdc34dSpatrick snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 7787acdc34dSpatrick DEVNAME(sc), port); 7797acdc34dSpatrick #endif 7807acdc34dSpatrick 7817acdc34dSpatrick /* Disable port interrupts */ 7827acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IE, 0); 7837acdc34dSpatrick 7847acdc34dSpatrick /* Sec 10.1.2 - deinitialise port if it is already running */ 7857acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD); 7867acdc34dSpatrick if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 7877acdc34dSpatrick AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 7887acdc34dSpatrick ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 7897acdc34dSpatrick int r; 7907acdc34dSpatrick 7917acdc34dSpatrick r = ahci_port_stop(ap, 1); 7927acdc34dSpatrick if (r) { 7937acdc34dSpatrick printf("%s: unable to disable %s, ignoring port %d\n", 7947acdc34dSpatrick DEVNAME(sc), r == 2 ? "CR" : "FR", port); 7957acdc34dSpatrick rc = ENXIO; 7967acdc34dSpatrick goto reterr; 7977acdc34dSpatrick } 7987acdc34dSpatrick 7997acdc34dSpatrick /* Write DET to zero */ 8007acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 8017acdc34dSpatrick } 8027acdc34dSpatrick 8037acdc34dSpatrick /* Setup RFIS base address */ 8047acdc34dSpatrick ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 8057acdc34dSpatrick dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 8067acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 8077acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 8087acdc34dSpatrick 8097acdc34dSpatrick /* Enable FIS reception and activate port. */ 8107acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 8117acdc34dSpatrick cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 8127acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 8137acdc34dSpatrick 8147acdc34dSpatrick /* Check whether port activated. Skip it if not. */ 8157acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 8167acdc34dSpatrick if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 8177acdc34dSpatrick rc = ENXIO; 8187acdc34dSpatrick goto reterr; 8197acdc34dSpatrick } 8207acdc34dSpatrick 8217acdc34dSpatrick /* Setup command list base address */ 8227acdc34dSpatrick dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 8237acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 8247acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 8257acdc34dSpatrick 8267acdc34dSpatrick /* Wait for ICC change to complete */ 8277acdc34dSpatrick ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 8287acdc34dSpatrick 8297acdc34dSpatrick /* Reset port */ 8307acdc34dSpatrick rc = ahci_port_portreset(ap, 1); 8317acdc34dSpatrick switch (rc) { 8327acdc34dSpatrick case ENODEV: 8337acdc34dSpatrick switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 8347acdc34dSpatrick case AHCI_PREG_SSTS_DET_DEV_NE: 8357acdc34dSpatrick printf("%s: device not communicating on port %d\n", 8367acdc34dSpatrick DEVNAME(sc), port); 8377acdc34dSpatrick break; 8387acdc34dSpatrick case AHCI_PREG_SSTS_DET_PHYOFFLINE: 8397acdc34dSpatrick printf("%s: PHY offline on port %d\n", DEVNAME(sc), 8407acdc34dSpatrick port); 8417acdc34dSpatrick break; 8427acdc34dSpatrick default: 8437acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 8447acdc34dSpatrick "on port %d\n", DEVNAME(sc), port); 8457acdc34dSpatrick break; 8467acdc34dSpatrick } 8477acdc34dSpatrick goto reterr; 8487acdc34dSpatrick 8497acdc34dSpatrick case EBUSY: 8507acdc34dSpatrick printf("%s: device on port %d didn't come ready, " 8517acdc34dSpatrick "TFD: 0x%b\n", DEVNAME(sc), port, 8527acdc34dSpatrick ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 8537acdc34dSpatrick 8547acdc34dSpatrick /* Try a soft reset to clear busy */ 8557acdc34dSpatrick rc = ahci_port_softreset(ap); 8567acdc34dSpatrick if (rc) { 8577acdc34dSpatrick printf("%s: unable to communicate " 8587acdc34dSpatrick "with device on port %d\n", DEVNAME(sc), port); 8597acdc34dSpatrick goto reterr; 8607acdc34dSpatrick } 8617acdc34dSpatrick break; 8627acdc34dSpatrick 8637acdc34dSpatrick default: 8647acdc34dSpatrick break; 8657acdc34dSpatrick } 8667acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 8677acdc34dSpatrick DEVNAME(sc), port); 8687acdc34dSpatrick 8697acdc34dSpatrick if (ap->ap_pmp_ports > 0) { 8707acdc34dSpatrick int p; 8717acdc34dSpatrick 8727acdc34dSpatrick for (p = 0; p < ap->ap_pmp_ports; p++) { 8737acdc34dSpatrick int sig; 8747acdc34dSpatrick 8757acdc34dSpatrick /* might need to do a portreset first here? */ 8767acdc34dSpatrick 8777acdc34dSpatrick /* softreset the port */ 8787acdc34dSpatrick if (ahci_pmp_port_softreset(ap, p)) { 8797acdc34dSpatrick printf("%s.%d: unable to probe PMP port due to" 8807acdc34dSpatrick " softreset failure\n", PORTNAME(ap), p); 8817acdc34dSpatrick continue; 8827acdc34dSpatrick } 8837acdc34dSpatrick 8847acdc34dSpatrick sig = ahci_port_signature(ap); 8857acdc34dSpatrick printf("%s.%d: port signature returned %d\n", 8867acdc34dSpatrick PORTNAME(ap), p, sig); 8877acdc34dSpatrick } 8887acdc34dSpatrick } 8897acdc34dSpatrick 8907acdc34dSpatrick /* Enable command transfers on port */ 8917acdc34dSpatrick if (ahci_port_start(ap, 0)) { 8927acdc34dSpatrick printf("%s: failed to start command DMA on port %d, " 8937acdc34dSpatrick "disabling\n", DEVNAME(sc), port); 8947acdc34dSpatrick rc = ENXIO; /* couldn't start port */ 8957acdc34dSpatrick } 8967acdc34dSpatrick 8977acdc34dSpatrick /* Flush interrupts for port */ 8987acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 8997acdc34dSpatrick ahci_write(sc, AHCI_REG_IS, 1 << port); 9007acdc34dSpatrick 9017acdc34dSpatrick ahci_enable_interrupts(ap); 9027acdc34dSpatrick 9037acdc34dSpatrick reterr: 9047acdc34dSpatrick return (rc); 9057acdc34dSpatrick } 9067acdc34dSpatrick 9077acdc34dSpatrick int 90893c0f120Sjsg ahci_default_port_start(struct ahci_port *ap, int fre_only) 9097acdc34dSpatrick { 9107acdc34dSpatrick u_int32_t r; 9117acdc34dSpatrick 9127acdc34dSpatrick /* Turn on FRE (and ST) */ 9137acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 9147acdc34dSpatrick r |= AHCI_PREG_CMD_FRE; 9157acdc34dSpatrick if (!fre_only) 9167acdc34dSpatrick r |= AHCI_PREG_CMD_ST; 9177acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, r); 9187acdc34dSpatrick 9197acdc34dSpatrick #ifdef AHCI_COALESCE 9207acdc34dSpatrick /* (Re-)enable coalescing on the port. */ 9217acdc34dSpatrick if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 9227acdc34dSpatrick ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 9237acdc34dSpatrick ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 9247acdc34dSpatrick ap->ap_sc->sc_ccc_ports_cur); 9257acdc34dSpatrick } 9267acdc34dSpatrick #endif 9277acdc34dSpatrick 9287acdc34dSpatrick return (0); 9297acdc34dSpatrick } 9307acdc34dSpatrick 9317acdc34dSpatrick int 9327acdc34dSpatrick ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 9337acdc34dSpatrick { 9347acdc34dSpatrick u_int32_t r; 9357acdc34dSpatrick 9367acdc34dSpatrick #ifdef AHCI_COALESCE 9377acdc34dSpatrick /* Disable coalescing on the port while it is stopped. */ 9387acdc34dSpatrick if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 9397acdc34dSpatrick ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 9407acdc34dSpatrick ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 9417acdc34dSpatrick ap->ap_sc->sc_ccc_ports_cur); 9427acdc34dSpatrick } 9437acdc34dSpatrick #endif 9447acdc34dSpatrick 9457acdc34dSpatrick /* Turn off ST (and FRE) */ 9467acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 9477acdc34dSpatrick r &= ~AHCI_PREG_CMD_ST; 9487acdc34dSpatrick if (stop_fis_rx) 9497acdc34dSpatrick r &= ~AHCI_PREG_CMD_FRE; 9507acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, r); 9517acdc34dSpatrick 9527acdc34dSpatrick /* Wait for CR to go off */ 9537acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 9547acdc34dSpatrick return (1); 9557acdc34dSpatrick 9567acdc34dSpatrick /* Wait for FR to go off */ 9577acdc34dSpatrick if (stop_fis_rx && 9587acdc34dSpatrick ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 9597acdc34dSpatrick return (2); 9607acdc34dSpatrick 9617acdc34dSpatrick return (0); 9627acdc34dSpatrick } 9637acdc34dSpatrick 9647acdc34dSpatrick /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 9657acdc34dSpatrick int 9667acdc34dSpatrick ahci_port_clo(struct ahci_port *ap) 9677acdc34dSpatrick { 9687acdc34dSpatrick struct ahci_softc *sc = ap->ap_sc; 9697acdc34dSpatrick u_int32_t cmd; 9707acdc34dSpatrick 9717acdc34dSpatrick /* Only attempt CLO if supported by controller */ 9727acdc34dSpatrick if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 9737acdc34dSpatrick return (1); 9747acdc34dSpatrick 9757acdc34dSpatrick /* Issue CLO */ 9767acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 9777acdc34dSpatrick #ifdef DIAGNOSTIC 9787acdc34dSpatrick if (ISSET(cmd, AHCI_PREG_CMD_ST)) 9797acdc34dSpatrick printf("%s: CLO requested while port running\n", PORTNAME(ap)); 9807acdc34dSpatrick #endif 9817acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 9827acdc34dSpatrick 9837acdc34dSpatrick /* Wait for completion */ 9847acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 9857acdc34dSpatrick printf("%s: CLO did not complete\n", PORTNAME(ap)); 9867acdc34dSpatrick return (1); 9877acdc34dSpatrick } 9887acdc34dSpatrick 9897acdc34dSpatrick return (0); 9907acdc34dSpatrick } 9917acdc34dSpatrick 9927acdc34dSpatrick /* AHCI soft reset, Section 10.4.1 */ 9937acdc34dSpatrick int 9947acdc34dSpatrick ahci_port_softreset(struct ahci_port *ap) 9957acdc34dSpatrick { 9967acdc34dSpatrick struct ahci_ccb *ccb = NULL; 9977acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 9987acdc34dSpatrick u_int8_t *fis; 9997acdc34dSpatrick int s, rc = EIO, oldstate; 10007acdc34dSpatrick u_int32_t cmd; 10017acdc34dSpatrick 10027acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 10037acdc34dSpatrick 10047acdc34dSpatrick s = splbio(); 10057acdc34dSpatrick oldstate = ap->ap_state; 10067acdc34dSpatrick ap->ap_state = AP_S_ERROR_RECOVERY; 10077acdc34dSpatrick 10087acdc34dSpatrick /* Save previous command register state */ 10097acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 10107acdc34dSpatrick 10117acdc34dSpatrick /* Idle port */ 10127acdc34dSpatrick if (ahci_port_stop(ap, 0)) { 10137acdc34dSpatrick printf("%s: failed to stop port, cannot softreset\n", 10147acdc34dSpatrick PORTNAME(ap)); 10157acdc34dSpatrick goto err; 10167acdc34dSpatrick } 10177acdc34dSpatrick 10187acdc34dSpatrick /* Request CLO if device appears hung */ 10197acdc34dSpatrick if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 10207acdc34dSpatrick AHCI_PREG_TFD_STS_DRQ)) 10217acdc34dSpatrick ahci_port_clo(ap); 10227acdc34dSpatrick 10237acdc34dSpatrick /* Clear port errors to permit TFD transfer */ 10247acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 10257acdc34dSpatrick 10267acdc34dSpatrick /* Restart port */ 10277acdc34dSpatrick if (ahci_port_start(ap, 0)) { 10287acdc34dSpatrick printf("%s: failed to start port, cannot softreset\n", 10297acdc34dSpatrick PORTNAME(ap)); 10307acdc34dSpatrick goto err; 10317acdc34dSpatrick } 10327acdc34dSpatrick 10337acdc34dSpatrick /* Check whether CLO worked */ 10347acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 10357acdc34dSpatrick AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 10367acdc34dSpatrick printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 10377acdc34dSpatrick ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 10387acdc34dSpatrick ? "failed" : "unsupported"); 10397acdc34dSpatrick rc = EBUSY; 10407acdc34dSpatrick goto err; 10417acdc34dSpatrick } 10427acdc34dSpatrick 10437acdc34dSpatrick /* Prep first D2H command with SRST feature & clear busy/reset flags */ 10447acdc34dSpatrick ccb = ahci_get_err_ccb(ap); 10457acdc34dSpatrick cmd_slot = ccb->ccb_cmd_hdr; 1046414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 10477acdc34dSpatrick 10487acdc34dSpatrick fis = ccb->ccb_cmd_table->cfis; 10497acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 10507acdc34dSpatrick fis[15] = ATA_FIS_CONTROL_SRST; 10517acdc34dSpatrick 10527acdc34dSpatrick cmd_slot->prdtl = 0; 10533f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 10543f8c2449Sdlg AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 10553f8c2449Sdlg AHCI_CMD_LIST_FLAG_W); 10567acdc34dSpatrick 10577acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 10587acdc34dSpatrick if (ahci_poll(ccb, 1000, NULL) != 0) 10597acdc34dSpatrick goto err; 10607acdc34dSpatrick 10617acdc34dSpatrick /* Prep second D2H command to read status and complete reset sequence */ 10627acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 10637acdc34dSpatrick fis[15] = 0; 10647acdc34dSpatrick 10657acdc34dSpatrick cmd_slot->prdtl = 0; 10663f8c2449Sdlg htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 10677acdc34dSpatrick 10687acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 10697acdc34dSpatrick if (ahci_poll(ccb, 1000, NULL) != 0) 10707acdc34dSpatrick goto err; 10717acdc34dSpatrick 10727acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 10737acdc34dSpatrick AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 10747acdc34dSpatrick printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 10757acdc34dSpatrick PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 10767acdc34dSpatrick AHCI_PFMT_TFD_STS); 10777acdc34dSpatrick rc = EBUSY; 10787acdc34dSpatrick goto err; 10797acdc34dSpatrick } 10807acdc34dSpatrick 10817acdc34dSpatrick rc = 0; 10827acdc34dSpatrick err: 10837acdc34dSpatrick if (ccb != NULL) { 10847acdc34dSpatrick /* Abort our command, if it failed, by stopping command DMA. */ 10857acdc34dSpatrick if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 10867acdc34dSpatrick printf("%s: stopping the port, softreset slot %d was " 10877acdc34dSpatrick "still active.\n", PORTNAME(ap), ccb->ccb_slot); 10887acdc34dSpatrick ahci_port_stop(ap, 0); 10897acdc34dSpatrick } 10907acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ERROR; 10917acdc34dSpatrick ahci_put_err_ccb(ccb); 10927acdc34dSpatrick } 10937acdc34dSpatrick 10947acdc34dSpatrick /* Restore saved CMD register state */ 10957acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 10967acdc34dSpatrick ap->ap_state = oldstate; 10977acdc34dSpatrick 10987acdc34dSpatrick splx(s); 10997acdc34dSpatrick 11007acdc34dSpatrick return (rc); 11017acdc34dSpatrick } 11027acdc34dSpatrick 11037acdc34dSpatrick int 11047acdc34dSpatrick ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 11057acdc34dSpatrick { 11067acdc34dSpatrick struct ahci_ccb *ccb = NULL; 11077acdc34dSpatrick u_int32_t data; 11087acdc34dSpatrick int count; 11097acdc34dSpatrick int rc; 11107acdc34dSpatrick int s; 11117acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 11127acdc34dSpatrick u_int8_t *fis; 11137acdc34dSpatrick 11147acdc34dSpatrick s = splbio(); 11157acdc34dSpatrick /* ignore spurious IFS errors while resetting */ 11167acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 11177acdc34dSpatrick ap->ap_pmp_ignore_ifs = 1; 11187acdc34dSpatrick 11197acdc34dSpatrick count = 2; 11207acdc34dSpatrick rc = 0; 11217acdc34dSpatrick do { 11227acdc34dSpatrick if (ccb != NULL) { 11237acdc34dSpatrick ahci_put_pmp_ccb(ccb); 11247acdc34dSpatrick ccb = NULL; 11257acdc34dSpatrick } 11267acdc34dSpatrick 11277acdc34dSpatrick if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 11287acdc34dSpatrick printf("%s.%d: unable to clear PHY status\n", 11297acdc34dSpatrick PORTNAME(ap), pmp_port); 11307acdc34dSpatrick } 11317acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, -1); 11327acdc34dSpatrick /* maybe don't do this on the first loop: */ 11337acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 11347acdc34dSpatrick ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 11357acdc34dSpatrick 11367acdc34dSpatrick /* send first softreset FIS */ 1137189d7e68Skrw ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 11387acdc34dSpatrick cmd_slot = ccb->ccb_cmd_hdr; 1139414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 11407acdc34dSpatrick 11417acdc34dSpatrick fis = ccb->ccb_cmd_table->cfis; 11427acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 11437acdc34dSpatrick fis[1] = pmp_port; 11447acdc34dSpatrick fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 11457acdc34dSpatrick 11467acdc34dSpatrick cmd_slot->prdtl = 0; 11473f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 11483f8c2449Sdlg AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 11493f8c2449Sdlg (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 11507acdc34dSpatrick 11517acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 11527acdc34dSpatrick 11537acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 11547acdc34dSpatrick PORTNAME(ap), pmp_port); 11557acdc34dSpatrick if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 11567acdc34dSpatrick printf("%s.%d: PMP port softreset cmd failed\n", 11577acdc34dSpatrick PORTNAME(ap), pmp_port); 11587acdc34dSpatrick rc = EBUSY; 11597acdc34dSpatrick if (count > 0) { 11607acdc34dSpatrick /* probably delay a while to allow 11617acdc34dSpatrick * it to settle down? 11627acdc34dSpatrick */ 11637acdc34dSpatrick } 11647acdc34dSpatrick continue; 11657acdc34dSpatrick } 11667acdc34dSpatrick 11677acdc34dSpatrick /* send signature FIS */ 1168414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 11697acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 11707acdc34dSpatrick fis[1] = pmp_port; 11717acdc34dSpatrick fis[15] = ATA_FIS_CONTROL_4BIT; 11727acdc34dSpatrick 11737acdc34dSpatrick cmd_slot->prdtl = 0; 11743f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 11753f8c2449Sdlg (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 11767acdc34dSpatrick 11777acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 11787acdc34dSpatrick PORTNAME(ap), pmp_port); 11797acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 11807acdc34dSpatrick if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 11817acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 11827acdc34dSpatrick "failed\n", PORTNAME(ap), pmp_port); 11837acdc34dSpatrick rc = EBUSY; 11847acdc34dSpatrick if (count > 0) { 11857acdc34dSpatrick /* sleep a while? */ 11867acdc34dSpatrick } 11877acdc34dSpatrick continue; 11887acdc34dSpatrick } 11897acdc34dSpatrick 11907acdc34dSpatrick fis[15] = 0; 11917acdc34dSpatrick break; 11927acdc34dSpatrick } while (count--); 11937acdc34dSpatrick 11947acdc34dSpatrick if (ccb != NULL) { 11957acdc34dSpatrick ahci_put_pmp_ccb(ccb); 11967acdc34dSpatrick ccb = NULL; 11977acdc34dSpatrick } 11987acdc34dSpatrick 11997acdc34dSpatrick /* clean up a bit */ 12007acdc34dSpatrick ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 12017acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, -1); 12027acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 12037acdc34dSpatrick ap->ap_pmp_ignore_ifs = 0; 12047acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 12057acdc34dSpatrick splx(s); 12067acdc34dSpatrick 12077acdc34dSpatrick return (rc); 12087acdc34dSpatrick } 12097acdc34dSpatrick 12107acdc34dSpatrick int 12117acdc34dSpatrick ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 12127acdc34dSpatrick { 12137acdc34dSpatrick int sig; 12147acdc34dSpatrick 12157acdc34dSpatrick ap->ap_state = AP_S_PMP_PORT_PROBE; 12167acdc34dSpatrick 12177acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 12187acdc34dSpatrick pmp_port); 12197acdc34dSpatrick if (ahci_pmp_port_portreset(ap, pmp_port)) { 12207acdc34dSpatrick printf("%s.%d: unable to probe PMP port; portreset failed\n", 12217acdc34dSpatrick PORTNAME(ap), pmp_port); 12227acdc34dSpatrick ap->ap_state = AP_S_NORMAL; 12237acdc34dSpatrick return (ATA_PORT_T_NONE); 12247acdc34dSpatrick } 12257acdc34dSpatrick 12267acdc34dSpatrick if (ahci_pmp_port_softreset(ap, pmp_port)) { 12277acdc34dSpatrick printf("%s.%d: unable to probe PMP port due to softreset " 12287acdc34dSpatrick "failure\n", PORTNAME(ap), pmp_port); 12297acdc34dSpatrick ap->ap_state = AP_S_NORMAL; 12307acdc34dSpatrick return (ATA_PORT_T_NONE); 12317acdc34dSpatrick } 12327acdc34dSpatrick 12337acdc34dSpatrick sig = ahci_port_signature(ap); 12347acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 12357acdc34dSpatrick PORTNAME(ap), pmp_port, sig); 12367acdc34dSpatrick ap->ap_state = AP_S_NORMAL; 12377acdc34dSpatrick return (sig); 12387acdc34dSpatrick } 12397acdc34dSpatrick 12407acdc34dSpatrick 12417acdc34dSpatrick void 12427acdc34dSpatrick ahci_flush_tfd(struct ahci_port *ap) 12437acdc34dSpatrick { 12447acdc34dSpatrick u_int32_t r; 12457acdc34dSpatrick 12467acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_SERR); 12477acdc34dSpatrick if (r & AHCI_PREG_SERR_DIAG_X) 12487acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 12497acdc34dSpatrick } 12507acdc34dSpatrick 12517acdc34dSpatrick u_int32_t 12527acdc34dSpatrick ahci_active_mask(struct ahci_port *ap) 12537acdc34dSpatrick { 12547acdc34dSpatrick u_int32_t mask; 12557acdc34dSpatrick 12567acdc34dSpatrick mask = ahci_pread(ap, AHCI_PREG_CI); 12577acdc34dSpatrick if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 12587acdc34dSpatrick mask |= ahci_pread(ap, AHCI_PREG_SACT); 12597acdc34dSpatrick return mask; 12607acdc34dSpatrick } 12617acdc34dSpatrick 12627acdc34dSpatrick void 12637acdc34dSpatrick ahci_pmp_probe_timeout(void *cookie) 12647acdc34dSpatrick { 12657acdc34dSpatrick struct ahci_ccb *ccb = cookie; 12667acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 12677acdc34dSpatrick u_int32_t mask; 12687acdc34dSpatrick 12697acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 12707acdc34dSpatrick switch (ccb->ccb_xa.state) { 12717acdc34dSpatrick case ATA_S_PENDING: 12727acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 12737acdc34dSpatrick ccb->ccb_xa.state = ATA_S_TIMEOUT; 12747acdc34dSpatrick break; 12757acdc34dSpatrick 12767acdc34dSpatrick case ATA_S_ONCHIP: 12777acdc34dSpatrick case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 12787acdc34dSpatrick /* clear the command on-chip */ 12797acdc34dSpatrick KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 12807acdc34dSpatrick ap->ap_sactive == 0); 12817acdc34dSpatrick ahci_port_stop(ap, 0); 12827acdc34dSpatrick ahci_port_start(ap, 0); 12837acdc34dSpatrick 12847acdc34dSpatrick if (ahci_active_mask(ap) != 0) { 12857acdc34dSpatrick ahci_port_stop(ap, 0); 12867acdc34dSpatrick ahci_port_start(ap, 0); 12877acdc34dSpatrick mask = ahci_active_mask(ap); 12887acdc34dSpatrick if (mask != 0) { 12897acdc34dSpatrick printf("%s: ahci_pmp_probe_timeout: failed to " 12907acdc34dSpatrick "clear active cmds: %08x\n", PORTNAME(ap), 12917acdc34dSpatrick mask); 12927acdc34dSpatrick } 12937acdc34dSpatrick } 12947acdc34dSpatrick 12957acdc34dSpatrick ccb->ccb_xa.state = ATA_S_TIMEOUT; 12967acdc34dSpatrick ap->ap_active &= ~(1 << ccb->ccb_slot); 12977acdc34dSpatrick KASSERT(ap->ap_active_cnt > 0); 12987acdc34dSpatrick --ap->ap_active_cnt; 129977db1dd4Skrw DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, " 130077db1dd4Skrw "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot, 130177db1dd4Skrw ap->ap_active, ap->ap_active_cnt); 13027acdc34dSpatrick break; 13037acdc34dSpatrick 13047acdc34dSpatrick default: 13057acdc34dSpatrick panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 13067acdc34dSpatrick PORTNAME(ap), ccb->ccb_xa.state); 13077acdc34dSpatrick } 13087acdc34dSpatrick } 13097acdc34dSpatrick 13107acdc34dSpatrick int 13117acdc34dSpatrick ahci_port_signature(struct ahci_port *ap) 13127acdc34dSpatrick { 13137acdc34dSpatrick u_int32_t sig; 13147acdc34dSpatrick 13157acdc34dSpatrick sig = ahci_pread(ap, AHCI_PREG_SIG); 13167acdc34dSpatrick if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 13177acdc34dSpatrick return (ATA_PORT_T_ATAPI); 13187acdc34dSpatrick else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 13197acdc34dSpatrick 0xffff0000)) 13207acdc34dSpatrick return (ATA_PORT_T_PM); 13217acdc34dSpatrick else 13227acdc34dSpatrick return (ATA_PORT_T_DISK); 13237acdc34dSpatrick } 13247acdc34dSpatrick 13257acdc34dSpatrick int 13267acdc34dSpatrick ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 13277acdc34dSpatrick { 13287acdc34dSpatrick u_int32_t cmd, data; 13297acdc34dSpatrick int loop; 13307acdc34dSpatrick int rc = 1; 13317acdc34dSpatrick int s; 13327acdc34dSpatrick 13337acdc34dSpatrick s = splbio(); 13347acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 13357acdc34dSpatrick pmp_port); 13367acdc34dSpatrick 13377dbd8a00Spatrick /* Save previous command register state */ 13387acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 13397acdc34dSpatrick 13407acdc34dSpatrick /* turn off power management and disable the PHY */ 13417acdc34dSpatrick data = AHCI_PREG_SCTL_IPM_DISABLED; 13427acdc34dSpatrick /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 13437acdc34dSpatrick if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 13447acdc34dSpatrick goto err; 13457acdc34dSpatrick if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 13467acdc34dSpatrick goto err; 13477acdc34dSpatrick delay(10000); 13487acdc34dSpatrick 13497acdc34dSpatrick /* start COMRESET */ 13507acdc34dSpatrick data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 13517acdc34dSpatrick if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 13527acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 13537acdc34dSpatrick pmp_port); 13547acdc34dSpatrick data |= AHCI_PREG_SCTL_SPD_GEN1; 13557acdc34dSpatrick } else 13567acdc34dSpatrick data |= AHCI_PREG_SCTL_SPD_ANY; 13577acdc34dSpatrick 13587acdc34dSpatrick if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 13597acdc34dSpatrick goto err; 13607acdc34dSpatrick 13617acdc34dSpatrick /* give it a while to settle down */ 13627acdc34dSpatrick delay(100000); 13637acdc34dSpatrick 13647acdc34dSpatrick if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 13657acdc34dSpatrick printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 13667acdc34dSpatrick pmp_port); 13677acdc34dSpatrick } 13687acdc34dSpatrick 13697acdc34dSpatrick /* start trying to negotiate */ 13707acdc34dSpatrick ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 13717acdc34dSpatrick data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 13727acdc34dSpatrick if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 13737acdc34dSpatrick goto err; 13747acdc34dSpatrick 13757acdc34dSpatrick /* give it a while to detect */ 13767acdc34dSpatrick for (loop = 3; loop; --loop) { 13777acdc34dSpatrick if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 13787acdc34dSpatrick goto err; 13797acdc34dSpatrick if (data & AHCI_PREG_SSTS_DET) 13807acdc34dSpatrick break; 13817acdc34dSpatrick delay(100000); 13827acdc34dSpatrick } 13837acdc34dSpatrick if (loop == 0) { 13847acdc34dSpatrick printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 13857acdc34dSpatrick goto err; 13867acdc34dSpatrick } 13877acdc34dSpatrick 13887acdc34dSpatrick /* give it even longer to fully negotiate */ 13897acdc34dSpatrick for (loop = 30; loop; --loop) { 13907acdc34dSpatrick if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 13917acdc34dSpatrick goto err; 13927acdc34dSpatrick if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 13937acdc34dSpatrick break; 13947acdc34dSpatrick delay(100000); 13957acdc34dSpatrick } 13967acdc34dSpatrick 13977acdc34dSpatrick if (loop == 0) { 13987acdc34dSpatrick printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 13997acdc34dSpatrick pmp_port); 14007acdc34dSpatrick goto err; 14017acdc34dSpatrick } 14027acdc34dSpatrick 14037acdc34dSpatrick /* device detected */ 14047acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 14057acdc34dSpatrick pmp_port); 14067acdc34dSpatrick 14077acdc34dSpatrick /* clean up a bit */ 14087acdc34dSpatrick delay(100000); 14097acdc34dSpatrick ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 14107acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, -1); 14117acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 14127acdc34dSpatrick 14137acdc34dSpatrick rc = 0; 14147acdc34dSpatrick err: 14157dbd8a00Spatrick /* Restore preserved port state */ 14167dbd8a00Spatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 14177acdc34dSpatrick splx(s); 14187acdc34dSpatrick return (rc); 14197acdc34dSpatrick } 14207acdc34dSpatrick 14217acdc34dSpatrick /* AHCI port reset, Section 10.4.2 */ 142296739e86Sjmatthew 142396739e86Sjmatthew void 142496739e86Sjmatthew ahci_port_comreset(struct ahci_port *ap) 14257acdc34dSpatrick { 142696739e86Sjmatthew u_int32_t r; 14277acdc34dSpatrick 14287acdc34dSpatrick r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 14297acdc34dSpatrick if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 14307acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 14317acdc34dSpatrick r |= AHCI_PREG_SCTL_SPD_GEN1; 14327acdc34dSpatrick } else 14337acdc34dSpatrick r |= AHCI_PREG_SCTL_SPD_ANY; 14347acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SCTL, r); 14357acdc34dSpatrick delay(10000); /* wait at least 1ms for COMRESET to be sent */ 14367acdc34dSpatrick r &= ~AHCI_PREG_SCTL_DET_INIT; 14377acdc34dSpatrick r |= AHCI_PREG_SCTL_DET_NONE; 14387acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SCTL, r); 14397acdc34dSpatrick delay(10000); 144096739e86Sjmatthew } 14417acdc34dSpatrick 144296739e86Sjmatthew void 144396739e86Sjmatthew ahci_port_portreset_start(struct ahci_port *ap) 144496739e86Sjmatthew { 144596739e86Sjmatthew int s; 144696739e86Sjmatthew 144796739e86Sjmatthew s = splbio(); 144896739e86Sjmatthew DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 144996739e86Sjmatthew 145096739e86Sjmatthew /* Save previous command register state */ 145196739e86Sjmatthew ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 145296739e86Sjmatthew 145396739e86Sjmatthew /* Clear ST, ignoring failure */ 145496739e86Sjmatthew ahci_port_stop(ap, 0); 145596739e86Sjmatthew 145696739e86Sjmatthew /* Perform device detection */ 145796739e86Sjmatthew ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 145896739e86Sjmatthew delay(10000); 145996739e86Sjmatthew ahci_port_comreset(ap); 146096739e86Sjmatthew splx(s); 146196739e86Sjmatthew } 146296739e86Sjmatthew 146396739e86Sjmatthew int 146496739e86Sjmatthew ahci_port_portreset_poll(struct ahci_port *ap) 146596739e86Sjmatthew { 146696739e86Sjmatthew if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) != 146796739e86Sjmatthew AHCI_PREG_SSTS_DET_DEV) 146896739e86Sjmatthew return (EAGAIN); 146996739e86Sjmatthew return (0); 147096739e86Sjmatthew } 147196739e86Sjmatthew 147296739e86Sjmatthew void 147396739e86Sjmatthew ahci_port_portreset_wait(struct ahci_port *ap) 147496739e86Sjmatthew { 147596739e86Sjmatthew int i; 147696739e86Sjmatthew 147796739e86Sjmatthew for (i = 0; i < 1000; i++) { 147896739e86Sjmatthew if (ahci_port_portreset_poll(ap) == 0) 147996739e86Sjmatthew break; 148096739e86Sjmatthew delay(1000); 148196739e86Sjmatthew } 148296739e86Sjmatthew } 148396739e86Sjmatthew 148496739e86Sjmatthew int 148596739e86Sjmatthew ahci_port_portreset_finish(struct ahci_port *ap, int pmp) 148696739e86Sjmatthew { 148796739e86Sjmatthew int rc, s, retries = 0; 148896739e86Sjmatthew 148996739e86Sjmatthew s = splbio(); 149096739e86Sjmatthew retry: 149196739e86Sjmatthew if (ahci_port_portreset_poll(ap)) { 14927acdc34dSpatrick rc = ENODEV; 14937acdc34dSpatrick if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 14947acdc34dSpatrick /* this may be a port multiplier with no device 14957acdc34dSpatrick * on port 0, so still do the pmp check if requested. 14967acdc34dSpatrick */ 14977acdc34dSpatrick } else { 14987acdc34dSpatrick goto err; 14997acdc34dSpatrick } 15007acdc34dSpatrick } else { 15017acdc34dSpatrick /* Clear SERR (incl X bit), so TFD can update */ 15027acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 15037acdc34dSpatrick 15047acdc34dSpatrick /* Wait for device to become ready */ 15057acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 15067acdc34dSpatrick AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 15077acdc34dSpatrick /* even if the device doesn't wake up, check if there's 15087acdc34dSpatrick * a port multiplier there 15097acdc34dSpatrick */ 1510c1fa4ddeSjmatthew if (retries == 0) { 1511c1fa4ddeSjmatthew retries = 1; 151296739e86Sjmatthew ahci_port_comreset(ap); 151396739e86Sjmatthew ahci_port_portreset_wait(ap); 1514c1fa4ddeSjmatthew goto retry; 1515c1fa4ddeSjmatthew } 15167acdc34dSpatrick rc = EBUSY; 15177acdc34dSpatrick } else { 15187acdc34dSpatrick rc = 0; 15197acdc34dSpatrick } 15207acdc34dSpatrick } 15217acdc34dSpatrick 15227acdc34dSpatrick if (pmp != 0) { 152396739e86Sjmatthew if (ahci_port_detect_pmp(ap)) { 152496739e86Sjmatthew /* reset again without pmp support */ 152596739e86Sjmatthew pmp = 0; 152696739e86Sjmatthew retries = 0; 152796739e86Sjmatthew ahci_port_comreset(ap); 152896739e86Sjmatthew ahci_port_portreset_wait(ap); 152996739e86Sjmatthew goto retry; 15307acdc34dSpatrick } 15317acdc34dSpatrick } 15327acdc34dSpatrick 15337acdc34dSpatrick err: 15347acdc34dSpatrick /* Restore preserved port state */ 153596739e86Sjmatthew ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd); 153696739e86Sjmatthew ap->ap_saved_cmd = 0; 15377acdc34dSpatrick splx(s); 15387acdc34dSpatrick 15397acdc34dSpatrick return (rc); 15407acdc34dSpatrick } 15417acdc34dSpatrick 15427acdc34dSpatrick int 154396739e86Sjmatthew ahci_port_portreset(struct ahci_port *ap, int pmp) 154496739e86Sjmatthew { 154596739e86Sjmatthew ahci_port_portreset_start(ap); 154696739e86Sjmatthew ahci_port_portreset_wait(ap); 154796739e86Sjmatthew return (ahci_port_portreset_finish(ap, pmp)); 154896739e86Sjmatthew } 154996739e86Sjmatthew 155096739e86Sjmatthew int 15517acdc34dSpatrick ahci_port_detect_pmp(struct ahci_port *ap) 15527acdc34dSpatrick { 15537acdc34dSpatrick int count, pmp_rc, rc; 15547acdc34dSpatrick u_int32_t r, cmd; 15557acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 15567acdc34dSpatrick struct ahci_ccb *ccb = NULL; 15577acdc34dSpatrick u_int8_t *fis = NULL; 15587acdc34dSpatrick 15597acdc34dSpatrick if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 15607acdc34dSpatrick !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 15617acdc34dSpatrick return 0; 15627acdc34dSpatrick } 15637acdc34dSpatrick 15647acdc34dSpatrick rc = 0; 15657acdc34dSpatrick pmp_rc = 0; 15667acdc34dSpatrick count = 2; 15677acdc34dSpatrick do { 15687acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 15697acdc34dSpatrick count); 15707acdc34dSpatrick if (ccb != NULL) { 15717acdc34dSpatrick ahci_put_pmp_ccb(ccb); 15727acdc34dSpatrick ccb = NULL; 15737acdc34dSpatrick } 15747acdc34dSpatrick ahci_port_stop(ap, 0); 15757acdc34dSpatrick ap->ap_state = AP_S_PMP_PROBE; 15767acdc34dSpatrick 15777acdc34dSpatrick /* set PMA in cmd reg */ 15787acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 15797acdc34dSpatrick if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 15807acdc34dSpatrick cmd |= AHCI_PREG_CMD_PMA; 15817acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 15827acdc34dSpatrick } 15837acdc34dSpatrick 15847acdc34dSpatrick /* Flush errors and request CLO unconditionally, 15857acdc34dSpatrick * then start the port 15867acdc34dSpatrick */ 15877acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_SERR); 15887acdc34dSpatrick if (r & AHCI_PREG_SERR_DIAG_X) 15897acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, 15907acdc34dSpatrick AHCI_PREG_SERR_DIAG_X); 15917acdc34dSpatrick 15927acdc34dSpatrick /* Request CLO */ 15937acdc34dSpatrick ahci_port_clo(ap); 15947acdc34dSpatrick 15957acdc34dSpatrick /* Clear port errors to permit TFD transfer */ 15967acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_SERR); 15977acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, r); 15987acdc34dSpatrick 15997acdc34dSpatrick /* Restart port */ 16007acdc34dSpatrick if (ahci_port_start(ap, 0)) { 16017acdc34dSpatrick rc = EBUSY; 16027acdc34dSpatrick printf("%s: failed to start port, cannot probe PMP\n", 16037acdc34dSpatrick PORTNAME(ap)); 16047acdc34dSpatrick break; 16057acdc34dSpatrick } 16067acdc34dSpatrick 16077acdc34dSpatrick /* Check whether CLO worked */ 16087acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 16097acdc34dSpatrick AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 16107acdc34dSpatrick u_int32_t cap; 16117acdc34dSpatrick 16127acdc34dSpatrick cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 16137acdc34dSpatrick printf("%s: CLO %s, need port reset\n", 16147acdc34dSpatrick PORTNAME(ap), 16157acdc34dSpatrick ISSET(cap, AHCI_REG_CAP_SCLO) 16167acdc34dSpatrick ? "failed" : "unsupported"); 16177acdc34dSpatrick pmp_rc = EBUSY; 16187acdc34dSpatrick break; 16197acdc34dSpatrick } 16207acdc34dSpatrick 16217acdc34dSpatrick /* Prep first command with SRST feature & 16227acdc34dSpatrick * clear busy/reset flags 16237acdc34dSpatrick */ 1624189d7e68Skrw ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 16257acdc34dSpatrick cmd_slot = ccb->ccb_cmd_hdr; 1626414fa24dSdlg memset(ccb->ccb_cmd_table, 0, 16277acdc34dSpatrick sizeof(struct ahci_cmd_table)); 16287acdc34dSpatrick 16297acdc34dSpatrick fis = ccb->ccb_cmd_table->cfis; 16307acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 16317acdc34dSpatrick fis[1] = SATA_PMP_CONTROL_PORT; 16327acdc34dSpatrick fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 16337acdc34dSpatrick 16347acdc34dSpatrick cmd_slot->prdtl = 0; 16353f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 16363f8c2449Sdlg AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 16373f8c2449Sdlg AHCI_CMD_LIST_FLAG_PMP); 16387acdc34dSpatrick 16397acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 16407acdc34dSpatrick PORTNAME(ap)); 16417acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 16427acdc34dSpatrick if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 16437acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 16447acdc34dSpatrick PORTNAME(ap)); 16457acdc34dSpatrick pmp_rc = EBUSY; 16467acdc34dSpatrick continue; 16477acdc34dSpatrick } 16487acdc34dSpatrick 16497acdc34dSpatrick if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 16507acdc34dSpatrick AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 16517acdc34dSpatrick printf("%s: port busy after first PMP probe FIS\n", 16527acdc34dSpatrick PORTNAME(ap)); 16537acdc34dSpatrick } 16547acdc34dSpatrick 16557acdc34dSpatrick /* clear errors in case the device 16567acdc34dSpatrick * didn't reset cleanly 16577acdc34dSpatrick */ 16587acdc34dSpatrick ahci_flush_tfd(ap); 16597acdc34dSpatrick r = ahci_pread(ap, AHCI_PREG_SERR); 16607acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, r); 16617acdc34dSpatrick 16627acdc34dSpatrick /* Prep second command to read status and 16637acdc34dSpatrick * complete reset sequence 16647acdc34dSpatrick */ 1665414fa24dSdlg memset(ccb->ccb_cmd_table, 0, 16667acdc34dSpatrick sizeof(struct ahci_cmd_table)); 16677acdc34dSpatrick fis[0] = ATA_FIS_TYPE_H2D; 16687acdc34dSpatrick fis[1] = SATA_PMP_CONTROL_PORT; 16697acdc34dSpatrick fis[15] = ATA_FIS_CONTROL_4BIT; 16707acdc34dSpatrick 16717acdc34dSpatrick cmd_slot->prdtl = 0; 16723f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 16733f8c2449Sdlg AHCI_CMD_LIST_FLAG_PMP); 16747acdc34dSpatrick 16757acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 16767acdc34dSpatrick PORTNAME(ap)); 16777acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 16787acdc34dSpatrick if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 16797acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 16807acdc34dSpatrick "cmd failed\n", PORTNAME(ap)); 16817acdc34dSpatrick pmp_rc = EBUSY; 16827acdc34dSpatrick continue; 16837acdc34dSpatrick } 16847acdc34dSpatrick 16857acdc34dSpatrick /* apparently we need to retry at least once 16867acdc34dSpatrick * to get the right signature 16877acdc34dSpatrick */ 16887acdc34dSpatrick fis[15] = 0; 16897acdc34dSpatrick pmp_rc = 0; 16907acdc34dSpatrick } while (--count); 16917acdc34dSpatrick 16927acdc34dSpatrick if (ccb != NULL) { 16937acdc34dSpatrick ahci_put_pmp_ccb(ccb); 16947acdc34dSpatrick ccb = NULL; 16957acdc34dSpatrick } 16967acdc34dSpatrick 16977acdc34dSpatrick if (ap->ap_state == AP_S_PMP_PROBE) { 16987acdc34dSpatrick ap->ap_state = AP_S_NORMAL; 16997acdc34dSpatrick } 17007acdc34dSpatrick 17017acdc34dSpatrick if (pmp_rc == 0) { 17027acdc34dSpatrick if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 17037acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 17047acdc34dSpatrick PORTNAME(ap)); 17057acdc34dSpatrick pmp_rc = EBUSY; 17067acdc34dSpatrick } else { 17077acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 17087acdc34dSpatrick PORTNAME(ap)); 17097acdc34dSpatrick } 17107acdc34dSpatrick } 17117acdc34dSpatrick 17127acdc34dSpatrick if (pmp_rc == 0) { 17137acdc34dSpatrick if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 17147acdc34dSpatrick pmp_rc = EBUSY; 17157acdc34dSpatrick } else { 17167acdc34dSpatrick rc = 0; 17177acdc34dSpatrick } 17187acdc34dSpatrick } 17197acdc34dSpatrick 17207acdc34dSpatrick /* if PMP detection failed, so turn off the PMA bit and 17217acdc34dSpatrick * reset the port again 17227acdc34dSpatrick */ 17237acdc34dSpatrick if (pmp_rc != 0) { 17247acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 17257acdc34dSpatrick "the port\n", PORTNAME(ap)); 17267acdc34dSpatrick ahci_port_stop(ap, 0); 17277acdc34dSpatrick ahci_port_clo(ap); 17287acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 17297acdc34dSpatrick cmd &= ~AHCI_PREG_CMD_PMA; 17307acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 17317acdc34dSpatrick 17327acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IE, 0); 17337acdc34dSpatrick ahci_port_stop(ap, 0); 17347acdc34dSpatrick if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 17357acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 17367acdc34dSpatrick ahci_flush_tfd(ap); 17377acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, -1); 17387acdc34dSpatrick 17397acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, -1); 17407acdc34dSpatrick 17417acdc34dSpatrick ahci_enable_interrupts(ap); 17427acdc34dSpatrick 174396739e86Sjmatthew rc = pmp_rc; 17447acdc34dSpatrick } 17457acdc34dSpatrick 17467acdc34dSpatrick return (rc); 17477acdc34dSpatrick } 17487acdc34dSpatrick 1749b1514dfeSdlg void 1750b1514dfeSdlg ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1751b1514dfeSdlg u_int32_t flags) 1752b1514dfeSdlg { 1753b1514dfeSdlg flags |= len - 1; 1754b1514dfeSdlg 1755b1514dfeSdlg htolem64(&prd->dba, addr); 1756b1514dfeSdlg htolem32(&prd->flags, flags); 1757b1514dfeSdlg } 1758b1514dfeSdlg 17597acdc34dSpatrick int 17607acdc34dSpatrick ahci_load_prdt(struct ahci_ccb *ccb) 17617acdc34dSpatrick { 17627acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 17637acdc34dSpatrick struct ahci_softc *sc = ap->ap_sc; 17647acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 1765b1514dfeSdlg struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 17667acdc34dSpatrick bus_dmamap_t dmap = ccb->ccb_dmamap; 17677acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 17687acdc34dSpatrick int i, error; 17697acdc34dSpatrick 17707acdc34dSpatrick if (xa->datalen == 0) { 17717acdc34dSpatrick ccb->ccb_cmd_hdr->prdtl = 0; 17727acdc34dSpatrick return (0); 17737acdc34dSpatrick } 17747acdc34dSpatrick 17757acdc34dSpatrick error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 17767acdc34dSpatrick (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 17777acdc34dSpatrick if (error != 0) { 17787acdc34dSpatrick printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 17797acdc34dSpatrick return (1); 17807acdc34dSpatrick } 17817acdc34dSpatrick 1782b1514dfeSdlg for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1783b1514dfeSdlg ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1784b1514dfeSdlg dmap->dm_segs[i].ds_len, 0); 1785b1514dfeSdlg } 17867acdc34dSpatrick 1787b1514dfeSdlg ahci_load_prdt_seg(&prdt[i], 1788b1514dfeSdlg dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1789b1514dfeSdlg ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 17907acdc34dSpatrick 1791b1514dfeSdlg htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 17927acdc34dSpatrick 17937acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 17947acdc34dSpatrick (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 17957acdc34dSpatrick BUS_DMASYNC_PREWRITE); 17967acdc34dSpatrick 17977acdc34dSpatrick return (0); 17987acdc34dSpatrick } 17997acdc34dSpatrick 18007acdc34dSpatrick void 18017acdc34dSpatrick ahci_unload_prdt(struct ahci_ccb *ccb) 18027acdc34dSpatrick { 18037acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 18047acdc34dSpatrick struct ahci_softc *sc = ap->ap_sc; 18057acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 18067acdc34dSpatrick bus_dmamap_t dmap = ccb->ccb_dmamap; 18077acdc34dSpatrick 18087acdc34dSpatrick if (xa->datalen != 0) { 18097acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 18107acdc34dSpatrick (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 18117acdc34dSpatrick BUS_DMASYNC_POSTWRITE); 18127acdc34dSpatrick 18137acdc34dSpatrick bus_dmamap_unload(sc->sc_dmat, dmap); 18147acdc34dSpatrick 18157acdc34dSpatrick if (ccb->ccb_xa.flags & ATA_F_NCQ) 18167acdc34dSpatrick xa->resid = 0; 18177acdc34dSpatrick else 18187acdc34dSpatrick xa->resid = xa->datalen - 1819b1514dfeSdlg lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 18207acdc34dSpatrick } 18217acdc34dSpatrick } 18227acdc34dSpatrick 18237acdc34dSpatrick int 18247acdc34dSpatrick ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 18257acdc34dSpatrick { 18267acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 18277acdc34dSpatrick int s; 18287acdc34dSpatrick 18297acdc34dSpatrick s = splbio(); 18307acdc34dSpatrick ahci_start(ccb); 18317acdc34dSpatrick do { 18327acdc34dSpatrick if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 18337acdc34dSpatrick 1 << ccb->ccb_slot)) { 18347acdc34dSpatrick splx(s); 18357acdc34dSpatrick return (0); 18367acdc34dSpatrick } 18377acdc34dSpatrick if (ccb->ccb_xa.state == ATA_S_ERROR) { 18387acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 18397acdc34dSpatrick PORTNAME(ap), ccb->ccb_slot); 18407acdc34dSpatrick /* pretend it timed out? */ 18417acdc34dSpatrick if (timeout_fn != NULL) { 18427acdc34dSpatrick timeout_fn(ccb); 18437acdc34dSpatrick } 18447acdc34dSpatrick splx(s); 18457acdc34dSpatrick return (1); 18467acdc34dSpatrick } 18477acdc34dSpatrick 18487acdc34dSpatrick delay(1000); 18497acdc34dSpatrick } while (--timeout > 0); 18507acdc34dSpatrick 18517acdc34dSpatrick /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 18527acdc34dSpatrick if (timeout_fn != NULL) 18537acdc34dSpatrick timeout_fn(ccb); 18547acdc34dSpatrick 18557acdc34dSpatrick splx(s); 18567acdc34dSpatrick 18577acdc34dSpatrick return (1); 18587acdc34dSpatrick } 18597acdc34dSpatrick 18607acdc34dSpatrick void 18617acdc34dSpatrick ahci_start(struct ahci_ccb *ccb) 18627acdc34dSpatrick { 18637acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 18647acdc34dSpatrick struct ahci_softc *sc = ap->ap_sc; 18657acdc34dSpatrick 18667acdc34dSpatrick /* Zero transferred byte count before transfer */ 18677acdc34dSpatrick ccb->ccb_cmd_hdr->prdbc = 0; 18687acdc34dSpatrick 18697acdc34dSpatrick /* Sync command list entry and corresponding command table entry */ 18707acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 18717acdc34dSpatrick ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 18727acdc34dSpatrick sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 18737acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 18747acdc34dSpatrick ccb->ccb_slot * sizeof(struct ahci_cmd_table), 18757acdc34dSpatrick sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 18767acdc34dSpatrick 18777acdc34dSpatrick /* Prepare RFIS area for write by controller */ 18787acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 18797acdc34dSpatrick sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 18807acdc34dSpatrick 18817acdc34dSpatrick if (ccb->ccb_xa.flags & ATA_F_NCQ) { 18827acdc34dSpatrick /* Issue NCQ commands only when there are no outstanding 18837acdc34dSpatrick * standard commands. */ 18847acdc34dSpatrick if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 18857acdc34dSpatrick (ap->ap_sactive != 0 && 18867acdc34dSpatrick ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 18877acdc34dSpatrick TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 18887acdc34dSpatrick } else { 18897acdc34dSpatrick KASSERT(ap->ap_active_cnt == 0); 18907acdc34dSpatrick ap->ap_sactive |= (1 << ccb->ccb_slot); 18917acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ONCHIP; 18927acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 18937acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 18947acdc34dSpatrick ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 18957acdc34dSpatrick } 18967acdc34dSpatrick } else { 18977acdc34dSpatrick /* Wait for all NCQ commands to finish before issuing standard 18987acdc34dSpatrick * command. */ 18997acdc34dSpatrick if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 19007acdc34dSpatrick TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 19017acdc34dSpatrick else if (ap->ap_active_cnt < 2) { 19027acdc34dSpatrick ap->ap_active |= 1 << ccb->ccb_slot; 19037acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ONCHIP; 19047acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 19057acdc34dSpatrick ap->ap_active_cnt++; 19067acdc34dSpatrick } 19077acdc34dSpatrick } 19087acdc34dSpatrick } 19097acdc34dSpatrick 19107acdc34dSpatrick void 19117acdc34dSpatrick ahci_issue_pending_ncq_commands(struct ahci_port *ap) 19127acdc34dSpatrick { 19137acdc34dSpatrick struct ahci_ccb *nextccb; 19147acdc34dSpatrick u_int32_t sact_change = 0; 19157acdc34dSpatrick 19167acdc34dSpatrick KASSERT(ap->ap_active_cnt == 0); 19177acdc34dSpatrick 19187acdc34dSpatrick nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 19197acdc34dSpatrick if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 19207acdc34dSpatrick return; 19217acdc34dSpatrick 19227acdc34dSpatrick /* Start all the NCQ commands at the head of the pending list. 19237acdc34dSpatrick * If a port multiplier is attached to the port, we can only 19247acdc34dSpatrick * issue commands for one of its ports at a time. 19257acdc34dSpatrick */ 19267acdc34dSpatrick if (ap->ap_sactive != 0 && 19277acdc34dSpatrick ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 19287acdc34dSpatrick return; 19297acdc34dSpatrick } 19307acdc34dSpatrick 19317acdc34dSpatrick ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 19327acdc34dSpatrick do { 19337acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 19347acdc34dSpatrick sact_change |= 1 << nextccb->ccb_slot; 19357acdc34dSpatrick nextccb->ccb_xa.state = ATA_S_ONCHIP; 19367acdc34dSpatrick nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 19377acdc34dSpatrick } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 19387acdc34dSpatrick (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 19397acdc34dSpatrick 19407acdc34dSpatrick ap->ap_sactive |= sact_change; 19417acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 19427acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 19437acdc34dSpatrick } 19447acdc34dSpatrick 19457acdc34dSpatrick void 19467acdc34dSpatrick ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 19477acdc34dSpatrick { 19487acdc34dSpatrick struct ahci_ccb *nextccb; 19497acdc34dSpatrick 19507acdc34dSpatrick nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 19517acdc34dSpatrick if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 19527acdc34dSpatrick if (last_was_ncq) { 19537acdc34dSpatrick KASSERT(nextccb->ccb_xa.pmp_port != 19547acdc34dSpatrick ap->ap_pmp_ncq_port); 19557acdc34dSpatrick /* otherwise it should have been started already */ 19567acdc34dSpatrick } else { 19577acdc34dSpatrick ap->ap_active_cnt--; 19587acdc34dSpatrick } 19597acdc34dSpatrick 19607acdc34dSpatrick /* Issue NCQ commands only when there are no outstanding 19617acdc34dSpatrick * standard commands, and previous NCQ commands for other 19627acdc34dSpatrick * PMP ports have finished. 19637acdc34dSpatrick */ 19647acdc34dSpatrick if (ap->ap_active == 0) 19657acdc34dSpatrick ahci_issue_pending_ncq_commands(ap); 19667acdc34dSpatrick else 19677acdc34dSpatrick KASSERT(ap->ap_active_cnt == 1); 19687acdc34dSpatrick } else if (nextccb) { 19697acdc34dSpatrick if (ap->ap_sactive != 0 || last_was_ncq) 19707acdc34dSpatrick KASSERT(ap->ap_active_cnt == 0); 19717acdc34dSpatrick 19727acdc34dSpatrick /* Wait for all NCQ commands to finish before issuing standard 19737acdc34dSpatrick * command. */ 19747acdc34dSpatrick if (ap->ap_sactive != 0) 19757acdc34dSpatrick return; 19767acdc34dSpatrick 19777acdc34dSpatrick /* Keep up to 2 standard commands on-chip at a time. */ 19787acdc34dSpatrick do { 19797acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 19807acdc34dSpatrick ap->ap_active |= 1 << nextccb->ccb_slot; 19817acdc34dSpatrick nextccb->ccb_xa.state = ATA_S_ONCHIP; 19827acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 19837acdc34dSpatrick if (last_was_ncq) 19847acdc34dSpatrick ap->ap_active_cnt++; 19857acdc34dSpatrick if (ap->ap_active_cnt == 2) 19867acdc34dSpatrick break; 19877acdc34dSpatrick KASSERT(ap->ap_active_cnt == 1); 19887acdc34dSpatrick nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 19897acdc34dSpatrick } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 19907acdc34dSpatrick } else if (!last_was_ncq) { 19917acdc34dSpatrick KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 19927acdc34dSpatrick 19937acdc34dSpatrick /* Standard command finished, none waiting to start. */ 19947acdc34dSpatrick ap->ap_active_cnt--; 19957acdc34dSpatrick } else { 19967acdc34dSpatrick KASSERT(ap->ap_active_cnt == 0); 19977acdc34dSpatrick 19987acdc34dSpatrick /* NCQ command finished. */ 19997acdc34dSpatrick } 20007acdc34dSpatrick } 20017acdc34dSpatrick 20027acdc34dSpatrick int 20037acdc34dSpatrick ahci_intr(void *arg) 20047acdc34dSpatrick { 20057acdc34dSpatrick struct ahci_softc *sc = arg; 20067acdc34dSpatrick u_int32_t is, ack = 0; 20077acdc34dSpatrick int port; 20087acdc34dSpatrick 20097acdc34dSpatrick /* Read global interrupt status */ 20107acdc34dSpatrick is = ahci_read(sc, AHCI_REG_IS); 20117acdc34dSpatrick if (is == 0 || is == 0xffffffff) 20127acdc34dSpatrick return (0); 20137acdc34dSpatrick ack = is; 20147acdc34dSpatrick 20157acdc34dSpatrick #ifdef AHCI_COALESCE 20167acdc34dSpatrick /* Check coalescing interrupt first */ 20177acdc34dSpatrick if (is & sc->sc_ccc_mask) { 20187acdc34dSpatrick DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 20197acdc34dSpatrick DEVNAME(sc)); 20207acdc34dSpatrick is &= ~sc->sc_ccc_mask; 20217acdc34dSpatrick is |= sc->sc_ccc_ports_cur; 20227acdc34dSpatrick } 20237acdc34dSpatrick #endif 20247acdc34dSpatrick 20257acdc34dSpatrick /* Process interrupts for each port */ 20267acdc34dSpatrick while (is) { 20277acdc34dSpatrick port = ffs(is) - 1; 20287acdc34dSpatrick if (sc->sc_ports[port]) 20297acdc34dSpatrick ahci_port_intr(sc->sc_ports[port], 20307acdc34dSpatrick AHCI_PREG_CI_ALL_SLOTS); 20317acdc34dSpatrick is &= ~(1 << port); 20327acdc34dSpatrick } 20337acdc34dSpatrick 20347acdc34dSpatrick /* Finally, acknowledge global interrupt */ 20357acdc34dSpatrick ahci_write(sc, AHCI_REG_IS, ack); 20367acdc34dSpatrick 20377acdc34dSpatrick return (1); 20387acdc34dSpatrick } 20397acdc34dSpatrick 20407acdc34dSpatrick u_int32_t 20417acdc34dSpatrick ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 20427acdc34dSpatrick { 20437acdc34dSpatrick struct ahci_softc *sc = ap->ap_sc; 20447acdc34dSpatrick u_int32_t is, ci_saved, ci_masked, processed = 0; 20457acdc34dSpatrick int slot, need_restart = 0; 20467acdc34dSpatrick int process_error = 0; 20477acdc34dSpatrick struct ahci_ccb *ccb; 20487acdc34dSpatrick volatile u_int32_t *active; 20497acdc34dSpatrick #ifdef DIAGNOSTIC 20507acdc34dSpatrick u_int32_t tmp; 20517acdc34dSpatrick #endif 20527acdc34dSpatrick 20537acdc34dSpatrick is = ahci_pread(ap, AHCI_PREG_IS); 20547acdc34dSpatrick 20557acdc34dSpatrick /* Ack port interrupt only if checking all command slots. */ 20567acdc34dSpatrick if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 20577acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, is); 20587acdc34dSpatrick 20597acdc34dSpatrick if (is) 20607acdc34dSpatrick DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 20617acdc34dSpatrick is, AHCI_PFMT_IS); 20627acdc34dSpatrick 20637acdc34dSpatrick if (ap->ap_sactive) { 20647acdc34dSpatrick /* Active NCQ commands - use SActive instead of CI */ 20657acdc34dSpatrick KASSERT(ap->ap_active == 0); 20667acdc34dSpatrick KASSERT(ap->ap_active_cnt == 0); 20677acdc34dSpatrick ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 20687acdc34dSpatrick active = &ap->ap_sactive; 20697acdc34dSpatrick } else { 20707acdc34dSpatrick /* Save CI */ 20717acdc34dSpatrick ci_saved = ahci_pread(ap, AHCI_PREG_CI); 20727acdc34dSpatrick active = &ap->ap_active; 20737acdc34dSpatrick } 20747acdc34dSpatrick 20757acdc34dSpatrick if (is & AHCI_PREG_IS_TFES) { 20767acdc34dSpatrick process_error = 1; 20777acdc34dSpatrick } else if (is & AHCI_PREG_IS_DHRS) { 20787acdc34dSpatrick u_int32_t tfd; 20797acdc34dSpatrick u_int32_t cmd; 20807acdc34dSpatrick u_int32_t serr; 20817acdc34dSpatrick 20827acdc34dSpatrick tfd = ahci_pread(ap, AHCI_PREG_TFD); 20837acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD); 20847acdc34dSpatrick serr = ahci_pread(ap, AHCI_PREG_SERR); 20857acdc34dSpatrick if ((tfd & AHCI_PREG_TFD_STS_ERR) && 20867acdc34dSpatrick (cmd & AHCI_PREG_CMD_CR) == 0) { 20877acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 20887acdc34dSpatrick " %b, DIAG: %b\n", PORTNAME(ap), tfd, 20897acdc34dSpatrick AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 20907acdc34dSpatrick AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 20917acdc34dSpatrick AHCI_PFMT_SERR_DIAG); 20927acdc34dSpatrick process_error = 1; 20937acdc34dSpatrick } else { 20947acdc34dSpatrick /* rfis copy back is in the normal execution path */ 20957acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 20967acdc34dSpatrick } 20977acdc34dSpatrick } 20987acdc34dSpatrick 20997acdc34dSpatrick /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 21007acdc34dSpatrick if (process_error) { 21017acdc34dSpatrick u_int32_t tfd, serr; 21027acdc34dSpatrick int err_slot; 21037acdc34dSpatrick 21047acdc34dSpatrick tfd = ahci_pread(ap, AHCI_PREG_TFD); 21057acdc34dSpatrick serr = ahci_pread(ap, AHCI_PREG_SERR); 21067acdc34dSpatrick 21077acdc34dSpatrick if (ap->ap_sactive == 0) { 21087acdc34dSpatrick /* Errored slot is easy to determine from CMD. */ 21097acdc34dSpatrick err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 21107acdc34dSpatrick AHCI_PREG_CMD)); 21117acdc34dSpatrick 21127acdc34dSpatrick if ((ci_saved & (1 << err_slot)) == 0) { 21137acdc34dSpatrick /* 21147acdc34dSpatrick * Hardware doesn't seem to report correct 21157acdc34dSpatrick * slot number. If there's only one 21167acdc34dSpatrick * outstanding command we can cope, 21177acdc34dSpatrick * otherwise fail all active commands. 21187acdc34dSpatrick */ 21197acdc34dSpatrick if (ap->ap_active_cnt == 1) 21207acdc34dSpatrick err_slot = ffs(ap->ap_active) - 1; 21217acdc34dSpatrick else 21227acdc34dSpatrick goto failall; 21237acdc34dSpatrick } 21247acdc34dSpatrick 21257acdc34dSpatrick ccb = &ap->ap_ccbs[err_slot]; 21267acdc34dSpatrick 21277acdc34dSpatrick /* Preserve received taskfile data from the RFIS. */ 21287acdc34dSpatrick memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 21297acdc34dSpatrick sizeof(struct ata_fis_d2h)); 21307acdc34dSpatrick } else 21317acdc34dSpatrick err_slot = -1; /* Must extract error from log page */ 21327acdc34dSpatrick 21337acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 21347acdc34dSpatrick " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 21357acdc34dSpatrick AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 21367acdc34dSpatrick AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 21377acdc34dSpatrick AHCI_PFMT_SERR_DIAG); 21387acdc34dSpatrick 21397acdc34dSpatrick /* Turn off ST to clear CI and SACT. */ 21407acdc34dSpatrick ahci_port_stop(ap, 0); 21417acdc34dSpatrick need_restart = 1; 21427acdc34dSpatrick 21437acdc34dSpatrick /* Clear SERR to enable capturing new errors. */ 21447acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, serr); 21457acdc34dSpatrick 21467acdc34dSpatrick /* Acknowledge the interrupts we can recover from. */ 21477acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 21487acdc34dSpatrick AHCI_PREG_IS_IFS); 21497acdc34dSpatrick is = ahci_pread(ap, AHCI_PREG_IS); 21507acdc34dSpatrick 21517acdc34dSpatrick /* If device hasn't cleared its busy status, try to idle it. */ 21527acdc34dSpatrick if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 21537acdc34dSpatrick 21547acdc34dSpatrick if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 21557acdc34dSpatrick (ap->ap_state == AP_S_ERROR_RECOVERY)) { 21567acdc34dSpatrick /* can't reset the port here, just make sure 21577acdc34dSpatrick * the operation fails and the port still works. 21587acdc34dSpatrick */ 21597acdc34dSpatrick } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 21607acdc34dSpatrick printf("%s: error on PMP port %d, idling " 21617acdc34dSpatrick "device\n", PORTNAME(ap), 21627acdc34dSpatrick ccb->ccb_xa.pmp_port); 21637acdc34dSpatrick if (ahci_pmp_port_softreset(ap, 21647acdc34dSpatrick ccb->ccb_xa.pmp_port) == 0) { 21657acdc34dSpatrick printf("%s: unable to softreset port " 21667acdc34dSpatrick "%d\n", PORTNAME(ap), 21677acdc34dSpatrick ccb->ccb_xa.pmp_port); 21687acdc34dSpatrick if (ahci_pmp_port_portreset(ap, 21697acdc34dSpatrick ccb->ccb_xa.pmp_port)) { 21707acdc34dSpatrick printf("%s: failed to port " 21717acdc34dSpatrick " reset %d, giving up on " 21727acdc34dSpatrick "it\n", PORTNAME(ap), 21737acdc34dSpatrick ccb->ccb_xa.pmp_port); 21747acdc34dSpatrick goto fatal; 21757acdc34dSpatrick } 21767acdc34dSpatrick } 21777acdc34dSpatrick } else { 21787acdc34dSpatrick printf("%s: attempting to idle device\n", 21797acdc34dSpatrick PORTNAME(ap)); 21807acdc34dSpatrick if (ahci_port_softreset(ap)) { 21817acdc34dSpatrick printf("%s: failed to soft reset " 21827acdc34dSpatrick "device\n", PORTNAME(ap)); 21837acdc34dSpatrick if (ahci_port_portreset(ap, 0)) { 21847acdc34dSpatrick printf("%s: failed to port " 21857acdc34dSpatrick "reset device, give up on " 21867acdc34dSpatrick "it\n", PORTNAME(ap)); 21877acdc34dSpatrick goto fatal; 21887acdc34dSpatrick } 21897acdc34dSpatrick } 21907acdc34dSpatrick } 21917acdc34dSpatrick 21927acdc34dSpatrick /* Had to reset device, can't gather extended info. */ 21937acdc34dSpatrick } else if (ap->ap_sactive) { 21947acdc34dSpatrick /* Recover the NCQ error from log page 10h. 21956f1d62e9Sjmatthew * We can only have queued commands active for one port 21966f1d62e9Sjmatthew * at a time, so we know which device errored. 21977acdc34dSpatrick */ 21987acdc34dSpatrick ahci_port_read_ncq_error(ap, &err_slot, 21997acdc34dSpatrick ap->ap_pmp_ncq_port); 22007acdc34dSpatrick if (err_slot < 0) 22017acdc34dSpatrick goto failall; 22027acdc34dSpatrick 22037acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 22047acdc34dSpatrick PORTNAME(ap), err_slot); 22057acdc34dSpatrick 22067acdc34dSpatrick ccb = &ap->ap_ccbs[err_slot]; 220792a87063Sjmatthew if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 220892a87063Sjmatthew printf("%s: NCQ errored slot %d is idle" 220992a87063Sjmatthew " (%08x active)\n", PORTNAME(ap), err_slot, 221092a87063Sjmatthew ci_saved); 221192a87063Sjmatthew goto failall; 221292a87063Sjmatthew } 22137acdc34dSpatrick } else { 22147acdc34dSpatrick /* Didn't reset, could gather extended info from log. */ 22157acdc34dSpatrick } 22167acdc34dSpatrick 22177acdc34dSpatrick /* 22187acdc34dSpatrick * If we couldn't determine the errored slot, reset the port 22197acdc34dSpatrick * and fail all the active slots. 22207acdc34dSpatrick */ 22217acdc34dSpatrick if (err_slot == -1) { 22227acdc34dSpatrick if (ahci_port_softreset(ap) != 0 && 22237acdc34dSpatrick ahci_port_portreset(ap, 0) != 0) { 22247acdc34dSpatrick printf("%s: couldn't reset after NCQ error, " 22257acdc34dSpatrick "disabling device.\n", PORTNAME(ap)); 22267acdc34dSpatrick goto fatal; 22277acdc34dSpatrick } 22287acdc34dSpatrick printf("%s: couldn't recover NCQ error, failing " 22297acdc34dSpatrick "all outstanding commands.\n", PORTNAME(ap)); 22307acdc34dSpatrick goto failall; 22317acdc34dSpatrick } 22327acdc34dSpatrick 22337acdc34dSpatrick /* Clear the failed command in saved CI so completion runs. */ 22347acdc34dSpatrick ci_saved &= ~(1 << err_slot); 22357acdc34dSpatrick 22367acdc34dSpatrick /* Note the error in the ata_xfer. */ 22377acdc34dSpatrick KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 22387acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ERROR; 22397acdc34dSpatrick 22407acdc34dSpatrick #ifdef DIAGNOSTIC 22417acdc34dSpatrick /* There may only be one outstanding standard command now. */ 22427acdc34dSpatrick if (ap->ap_sactive == 0) { 22437acdc34dSpatrick tmp = ci_saved; 22447acdc34dSpatrick if (tmp) { 22457acdc34dSpatrick slot = ffs(tmp) - 1; 22467acdc34dSpatrick tmp &= ~(1 << slot); 22477acdc34dSpatrick KASSERT(tmp == 0); 22487acdc34dSpatrick } 22497acdc34dSpatrick } 22507acdc34dSpatrick #endif 22517acdc34dSpatrick } 22527acdc34dSpatrick 22537acdc34dSpatrick /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 22547acdc34dSpatrick * when there's a normal SATA device attached. 22557acdc34dSpatrick */ 22567acdc34dSpatrick if ((ap->ap_state == AP_S_PMP_PROBE) && 22577acdc34dSpatrick (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 22587acdc34dSpatrick (is & AHCI_PREG_IS_IPMS)) { 22597acdc34dSpatrick slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 22607acdc34dSpatrick DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 22617acdc34dSpatrick PORTNAME(ap), slot); 22627acdc34dSpatrick 22637acdc34dSpatrick ccb = &ap->ap_ccbs[slot]; 22647acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ERROR; 22657acdc34dSpatrick 22667acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 22677acdc34dSpatrick is &= ~AHCI_PREG_IS_IPMS; 22687acdc34dSpatrick } 22697acdc34dSpatrick 22707acdc34dSpatrick /* ignore IFS errors while resetting a PMP port */ 22717acdc34dSpatrick if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 22727acdc34dSpatrick DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 22737acdc34dSpatrick "port\n", PORTNAME(ap)); 22747acdc34dSpatrick 22757acdc34dSpatrick need_restart = 1; 22767acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SERR, -1); 22777acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 22787acdc34dSpatrick is &= ~AHCI_PREG_IS_IFS; 22797acdc34dSpatrick goto failall; 22807acdc34dSpatrick } 22817acdc34dSpatrick 22827acdc34dSpatrick /* Check for remaining errors - they are fatal. */ 22837acdc34dSpatrick if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 22847acdc34dSpatrick AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 22857acdc34dSpatrick printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 22867acdc34dSpatrick PORTNAME(ap), is, AHCI_PFMT_IS); 22877acdc34dSpatrick 22887acdc34dSpatrick /* XXX try recovery first */ 22897acdc34dSpatrick goto fatal; 22907acdc34dSpatrick } 22917acdc34dSpatrick 22927acdc34dSpatrick /* Fail all outstanding commands if we know the port won't recover. */ 22937acdc34dSpatrick if (ap->ap_state == AP_S_FATAL_ERROR) { 22947acdc34dSpatrick fatal: 22957acdc34dSpatrick ap->ap_state = AP_S_FATAL_ERROR; 22967acdc34dSpatrick failall: 22977acdc34dSpatrick 22987acdc34dSpatrick /* Ensure port is shut down. */ 22997acdc34dSpatrick ahci_port_stop(ap, 1); 23007acdc34dSpatrick 23017acdc34dSpatrick /* Error all the active slots. */ 23027acdc34dSpatrick ci_masked = ci_saved & *active; 23037acdc34dSpatrick while (ci_masked) { 23047acdc34dSpatrick slot = ffs(ci_masked) - 1; 23057acdc34dSpatrick ccb = &ap->ap_ccbs[slot]; 23067acdc34dSpatrick ci_masked &= ~(1 << slot); 23077acdc34dSpatrick ccb->ccb_xa.state = ATA_S_ERROR; 23087acdc34dSpatrick } 23097acdc34dSpatrick 23107acdc34dSpatrick /* Run completion for all active slots. */ 23117acdc34dSpatrick ci_saved &= ~*active; 23127acdc34dSpatrick 23137acdc34dSpatrick /* Don't restart the port if our problems were deemed fatal. */ 23147acdc34dSpatrick if (ap->ap_state == AP_S_FATAL_ERROR) 23157acdc34dSpatrick need_restart = 0; 23167acdc34dSpatrick } 23177acdc34dSpatrick 23187acdc34dSpatrick /* 23197acdc34dSpatrick * CCB completion is detected by noticing its slot's bit in CI has 23207acdc34dSpatrick * changed to zero some time after we activated it. 23217acdc34dSpatrick * If we are polling, we may only be interested in particular slot(s). 23227acdc34dSpatrick */ 23237acdc34dSpatrick ci_masked = ~ci_saved & *active & ci_mask; 23247acdc34dSpatrick while (ci_masked) { 23257acdc34dSpatrick slot = ffs(ci_masked) - 1; 23267acdc34dSpatrick ccb = &ap->ap_ccbs[slot]; 23277acdc34dSpatrick ci_masked &= ~(1 << slot); 23287acdc34dSpatrick 23297acdc34dSpatrick DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 23307acdc34dSpatrick PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 23317acdc34dSpatrick " (error)" : ""); 23327acdc34dSpatrick 23337acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, 23347acdc34dSpatrick AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 23357acdc34dSpatrick ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 23367acdc34dSpatrick sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 23377acdc34dSpatrick 23387acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, 23397acdc34dSpatrick AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 23407acdc34dSpatrick ccb->ccb_slot * sizeof(struct ahci_cmd_table), 23417acdc34dSpatrick sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 23427acdc34dSpatrick 23437acdc34dSpatrick bus_dmamap_sync(sc->sc_dmat, 23447acdc34dSpatrick AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 23457acdc34dSpatrick sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 23467acdc34dSpatrick 23477acdc34dSpatrick *active &= ~(1 << ccb->ccb_slot); 23487acdc34dSpatrick /* Copy the rfis into the ccb if we were asked for it */ 23497acdc34dSpatrick if (ccb->ccb_xa.state == ATA_S_ONCHIP && 23507acdc34dSpatrick ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 23517acdc34dSpatrick memcpy(&ccb->ccb_xa.rfis, 23527acdc34dSpatrick ap->ap_rfis->rfis, 23537acdc34dSpatrick sizeof(struct ata_fis_d2h)); 23547acdc34dSpatrick } 23557acdc34dSpatrick 23567acdc34dSpatrick processed |= 1 << ccb->ccb_slot; 23576579b5f8Sjsg 23586579b5f8Sjsg ccb->ccb_done(ccb); 23597acdc34dSpatrick } 23607acdc34dSpatrick 23617acdc34dSpatrick if (need_restart) { 23627acdc34dSpatrick /* Restart command DMA on the port */ 23637acdc34dSpatrick ahci_port_start(ap, 0); 23647acdc34dSpatrick 23657acdc34dSpatrick /* Re-enable outstanding commands on port. */ 23667acdc34dSpatrick if (ci_saved) { 23677acdc34dSpatrick #ifdef DIAGNOSTIC 23687acdc34dSpatrick tmp = ci_saved; 23697acdc34dSpatrick while (tmp) { 23707acdc34dSpatrick slot = ffs(tmp) - 1; 23717acdc34dSpatrick tmp &= ~(1 << slot); 23727acdc34dSpatrick ccb = &ap->ap_ccbs[slot]; 23737acdc34dSpatrick KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 23747acdc34dSpatrick KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 23757acdc34dSpatrick (!!ap->ap_sactive)); 23767acdc34dSpatrick } 23777acdc34dSpatrick #endif 23787acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 23797acdc34dSpatrick "re-enabling%s slots %08x\n", PORTNAME(ap), 23807acdc34dSpatrick ap->ap_sactive ? " NCQ" : "", ci_saved); 23817acdc34dSpatrick 23827acdc34dSpatrick if (ap->ap_sactive) 23837acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 23847acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 23857acdc34dSpatrick } 23867acdc34dSpatrick } 23877acdc34dSpatrick 23887acdc34dSpatrick return (processed); 23897acdc34dSpatrick } 23907acdc34dSpatrick 23917acdc34dSpatrick struct ahci_ccb * 23927acdc34dSpatrick ahci_get_ccb(struct ahci_port *ap) 23937acdc34dSpatrick { 23947acdc34dSpatrick struct ahci_ccb *ccb; 23957acdc34dSpatrick 23967acdc34dSpatrick mtx_enter(&ap->ap_ccb_mtx); 23977acdc34dSpatrick ccb = TAILQ_FIRST(&ap->ap_ccb_free); 23987acdc34dSpatrick if (ccb != NULL) { 23997acdc34dSpatrick KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 24007acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 24017acdc34dSpatrick ccb->ccb_xa.state = ATA_S_SETUP; 24027acdc34dSpatrick } 24037acdc34dSpatrick mtx_leave(&ap->ap_ccb_mtx); 24047acdc34dSpatrick 24057acdc34dSpatrick return (ccb); 24067acdc34dSpatrick } 24077acdc34dSpatrick 24087acdc34dSpatrick void 24097acdc34dSpatrick ahci_put_ccb(struct ahci_ccb *ccb) 24107acdc34dSpatrick { 24117acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 24127acdc34dSpatrick 24137acdc34dSpatrick #ifdef DIAGNOSTIC 24147acdc34dSpatrick if (ccb->ccb_xa.state != ATA_S_COMPLETE && 24157acdc34dSpatrick ccb->ccb_xa.state != ATA_S_TIMEOUT && 24167acdc34dSpatrick ccb->ccb_xa.state != ATA_S_ERROR) { 24177acdc34dSpatrick printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 24187acdc34dSpatrick "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 24197acdc34dSpatrick ccb->ccb_slot); 24207acdc34dSpatrick } 24217acdc34dSpatrick #endif 24227acdc34dSpatrick 24237acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PUT; 24247acdc34dSpatrick mtx_enter(&ap->ap_ccb_mtx); 24257acdc34dSpatrick TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 24267acdc34dSpatrick mtx_leave(&ap->ap_ccb_mtx); 24277acdc34dSpatrick } 24287acdc34dSpatrick 24297acdc34dSpatrick struct ahci_ccb * 24307acdc34dSpatrick ahci_get_err_ccb(struct ahci_port *ap) 24317acdc34dSpatrick { 24327acdc34dSpatrick struct ahci_ccb *err_ccb; 24337acdc34dSpatrick u_int32_t sact; 24347acdc34dSpatrick 24357acdc34dSpatrick splassert(IPL_BIO); 24367acdc34dSpatrick 24377acdc34dSpatrick /* No commands may be active on the chip. */ 24387acdc34dSpatrick sact = ahci_pread(ap, AHCI_PREG_SACT); 24397acdc34dSpatrick if (sact != 0) 24407acdc34dSpatrick printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 24417acdc34dSpatrick KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 24427acdc34dSpatrick 24437acdc34dSpatrick #ifdef DIAGNOSTIC 24447acdc34dSpatrick KASSERT(ap->ap_err_busy == 0); 24457acdc34dSpatrick ap->ap_err_busy = 1; 24467acdc34dSpatrick #endif 24477acdc34dSpatrick /* Save outstanding command state. */ 24487acdc34dSpatrick ap->ap_err_saved_active = ap->ap_active; 24497acdc34dSpatrick ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 24507acdc34dSpatrick ap->ap_err_saved_sactive = ap->ap_sactive; 24517acdc34dSpatrick 24527acdc34dSpatrick /* 24537acdc34dSpatrick * Pretend we have no commands outstanding, so that completions won't 24547acdc34dSpatrick * run prematurely. 24557acdc34dSpatrick */ 24567acdc34dSpatrick ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 24577acdc34dSpatrick 24587acdc34dSpatrick /* 24597acdc34dSpatrick * Grab a CCB to use for error recovery. This should never fail, as 24607acdc34dSpatrick * we ask atascsi to reserve one for us at init time. 24617acdc34dSpatrick */ 24627acdc34dSpatrick err_ccb = ap->ap_ccb_err; 24637acdc34dSpatrick err_ccb->ccb_xa.flags = 0; 24647acdc34dSpatrick err_ccb->ccb_xa.state = ATA_S_SETUP; 24657acdc34dSpatrick err_ccb->ccb_done = ahci_empty_done; 24667acdc34dSpatrick 24677acdc34dSpatrick return (err_ccb); 24687acdc34dSpatrick } 24697acdc34dSpatrick 24707acdc34dSpatrick void 24717acdc34dSpatrick ahci_put_err_ccb(struct ahci_ccb *ccb) 24727acdc34dSpatrick { 24737acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 24747acdc34dSpatrick u_int32_t sact; 24757acdc34dSpatrick 24767acdc34dSpatrick splassert(IPL_BIO); 24777acdc34dSpatrick 24787acdc34dSpatrick #ifdef DIAGNOSTIC 24797acdc34dSpatrick KASSERT(ap->ap_err_busy); 24807acdc34dSpatrick #endif 24817acdc34dSpatrick /* No commands may be active on the chip */ 24827acdc34dSpatrick sact = ahci_pread(ap, AHCI_PREG_SACT); 24837acdc34dSpatrick if (sact != 0) 248477db1dd4Skrw printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact); 24857acdc34dSpatrick KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 24867acdc34dSpatrick 24877acdc34dSpatrick /* Done with the CCB */ 24887acdc34dSpatrick KASSERT(ccb == ap->ap_ccb_err); 24897acdc34dSpatrick 24907acdc34dSpatrick /* Restore outstanding command state */ 24917acdc34dSpatrick ap->ap_sactive = ap->ap_err_saved_sactive; 24927acdc34dSpatrick ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 24937acdc34dSpatrick ap->ap_active = ap->ap_err_saved_active; 24947acdc34dSpatrick 24957acdc34dSpatrick #ifdef DIAGNOSTIC 24967acdc34dSpatrick ap->ap_err_busy = 0; 24977acdc34dSpatrick #endif 24987acdc34dSpatrick } 24997acdc34dSpatrick 25007acdc34dSpatrick struct ahci_ccb * 25017acdc34dSpatrick ahci_get_pmp_ccb(struct ahci_port *ap) 25027acdc34dSpatrick { 25037acdc34dSpatrick struct ahci_ccb *ccb; 25047acdc34dSpatrick u_int32_t sact; 25057acdc34dSpatrick 25067acdc34dSpatrick /* some PMP commands need to be issued on slot 1, 25077acdc34dSpatrick * particularly the command that clears SRST and 25087acdc34dSpatrick * fetches the device signature. 25097acdc34dSpatrick * 25107acdc34dSpatrick * ensure the chip is idle and ccb 1 is available. 25117acdc34dSpatrick */ 25127acdc34dSpatrick splassert(IPL_BIO); 25137acdc34dSpatrick 25147acdc34dSpatrick sact = ahci_pread(ap, AHCI_PREG_SACT); 25157acdc34dSpatrick if (sact != 0) 25167acdc34dSpatrick printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 25177acdc34dSpatrick KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 25187acdc34dSpatrick 25197acdc34dSpatrick ccb = &ap->ap_ccbs[1]; 25207acdc34dSpatrick KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 25217acdc34dSpatrick ccb->ccb_xa.flags = 0; 25227acdc34dSpatrick ccb->ccb_done = ahci_pmp_cmd_done; 25237acdc34dSpatrick 25247acdc34dSpatrick mtx_enter(&ap->ap_ccb_mtx); 25257acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 25267acdc34dSpatrick mtx_leave(&ap->ap_ccb_mtx); 25277acdc34dSpatrick 25287acdc34dSpatrick return ccb; 25297acdc34dSpatrick } 25307acdc34dSpatrick 25317acdc34dSpatrick void 25327acdc34dSpatrick ahci_put_pmp_ccb(struct ahci_ccb *ccb) 25337acdc34dSpatrick { 25347acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 25357acdc34dSpatrick u_int32_t sact; 25367acdc34dSpatrick 25377acdc34dSpatrick /* make sure this is the right ccb */ 25387acdc34dSpatrick KASSERT(ccb == &ap->ap_ccbs[1]); 25397acdc34dSpatrick 25407acdc34dSpatrick /* No commands may be active on the chip */ 25417acdc34dSpatrick sact = ahci_pread(ap, AHCI_PREG_SACT); 25427acdc34dSpatrick if (sact != 0) 254377db1dd4Skrw printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact); 25447acdc34dSpatrick KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 25457acdc34dSpatrick 25467acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PUT; 25477acdc34dSpatrick mtx_enter(&ap->ap_ccb_mtx); 25487acdc34dSpatrick TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 25497acdc34dSpatrick mtx_leave(&ap->ap_ccb_mtx); 25507acdc34dSpatrick } 25517acdc34dSpatrick 25527acdc34dSpatrick int 25537acdc34dSpatrick ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 25547acdc34dSpatrick { 25557acdc34dSpatrick struct ahci_ccb *ccb; 25567acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 25577acdc34dSpatrick u_int32_t cmd; 25587acdc34dSpatrick struct ata_fis_h2d *fis; 25597acdc34dSpatrick int rc = EIO, oldstate; 25607acdc34dSpatrick 25617acdc34dSpatrick DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 25627acdc34dSpatrick oldstate = ap->ap_state; 25637acdc34dSpatrick ap->ap_state = AP_S_ERROR_RECOVERY; 25647acdc34dSpatrick 25657acdc34dSpatrick /* Save command register state. */ 25667acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 25677acdc34dSpatrick 25687acdc34dSpatrick /* Port should have been idled already. Start it. */ 25697acdc34dSpatrick KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 25707acdc34dSpatrick ahci_port_start(ap, 0); 25717acdc34dSpatrick 25727acdc34dSpatrick /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 25737acdc34dSpatrick ccb = ahci_get_err_ccb(ap); 25747acdc34dSpatrick ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 25757acdc34dSpatrick ccb->ccb_xa.data = ap->ap_err_scratch; 25767acdc34dSpatrick ccb->ccb_xa.datalen = 512; 25777acdc34dSpatrick cmd_slot = ccb->ccb_cmd_hdr; 2578414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 25797acdc34dSpatrick 25807acdc34dSpatrick fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 25817acdc34dSpatrick fis->type = ATA_FIS_TYPE_H2D; 25827acdc34dSpatrick fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 25837acdc34dSpatrick fis->command = ATA_C_READ_LOG_EXT; 25847acdc34dSpatrick fis->lba_low = 0x10; /* queued error log page (10h) */ 25857acdc34dSpatrick fis->sector_count = 1; /* number of sectors (1) */ 25867acdc34dSpatrick fis->sector_count_exp = 0; 25877acdc34dSpatrick fis->lba_mid = 0; /* starting offset */ 25887acdc34dSpatrick fis->lba_mid_exp = 0; 25897acdc34dSpatrick fis->device = 0; 25907acdc34dSpatrick 25913f8c2449Sdlg htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 25923f8c2449Sdlg (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 25937acdc34dSpatrick 25947acdc34dSpatrick if (ahci_load_prdt(ccb) != 0) { 25957acdc34dSpatrick rc = ENOMEM; /* XXX caller must abort all commands */ 25967acdc34dSpatrick goto err; 25977acdc34dSpatrick } 25987acdc34dSpatrick 25997acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 26007acdc34dSpatrick if (ahci_poll(ccb, 1000, NULL) != 0 || 26017acdc34dSpatrick ccb->ccb_xa.state == ATA_S_ERROR) 26027acdc34dSpatrick goto err; 26037acdc34dSpatrick 26047acdc34dSpatrick rc = 0; 26057acdc34dSpatrick err: 26067acdc34dSpatrick /* Abort our command, if it failed, by stopping command DMA. */ 26077acdc34dSpatrick if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 26087acdc34dSpatrick printf("%s: log page read failed, slot %d was still active.\n", 26097acdc34dSpatrick PORTNAME(ap), ccb->ccb_slot); 26107acdc34dSpatrick ahci_port_stop(ap, 0); 26117acdc34dSpatrick } 26127acdc34dSpatrick 26137acdc34dSpatrick /* Done with the error CCB now. */ 26147acdc34dSpatrick ahci_unload_prdt(ccb); 26157acdc34dSpatrick ahci_put_err_ccb(ccb); 26167acdc34dSpatrick 26177acdc34dSpatrick /* Extract failed register set and tags from the scratch space. */ 26187acdc34dSpatrick if (rc == 0) { 26197acdc34dSpatrick struct ata_log_page_10h *log; 26207acdc34dSpatrick int err_slot; 26217acdc34dSpatrick 26227acdc34dSpatrick log = (struct ata_log_page_10h *)ap->ap_err_scratch; 26237acdc34dSpatrick if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 26247acdc34dSpatrick /* Not queued bit was set - wasn't an NCQ error? */ 26257acdc34dSpatrick printf("%s: read NCQ error page, but not an NCQ " 26267acdc34dSpatrick "error?\n", PORTNAME(ap)); 26277acdc34dSpatrick rc = ESRCH; 26287acdc34dSpatrick } else { 26297acdc34dSpatrick /* Copy back the log record as a D2H register FIS. */ 26307acdc34dSpatrick *err_slotp = err_slot = log->err_regs.type & 26317acdc34dSpatrick ATA_LOG_10H_TYPE_TAG_MASK; 26327acdc34dSpatrick 26337acdc34dSpatrick ccb = &ap->ap_ccbs[err_slot]; 26347acdc34dSpatrick memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 26357acdc34dSpatrick sizeof(struct ata_fis_d2h)); 26367acdc34dSpatrick ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 26377acdc34dSpatrick ccb->ccb_xa.rfis.flags = 0; 26387acdc34dSpatrick } 26397acdc34dSpatrick } 26407acdc34dSpatrick 26417acdc34dSpatrick /* Restore saved CMD register state */ 26427acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 26437acdc34dSpatrick ap->ap_state = oldstate; 26447acdc34dSpatrick 26457acdc34dSpatrick return (rc); 26467acdc34dSpatrick } 26477acdc34dSpatrick 26487acdc34dSpatrick struct ahci_dmamem * 26497acdc34dSpatrick ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 26507acdc34dSpatrick { 26517acdc34dSpatrick struct ahci_dmamem *adm; 26527acdc34dSpatrick int nsegs; 26537acdc34dSpatrick 26547acdc34dSpatrick adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 26557acdc34dSpatrick if (adm == NULL) 26567acdc34dSpatrick return (NULL); 26577acdc34dSpatrick 26587acdc34dSpatrick adm->adm_size = size; 26597acdc34dSpatrick 26607acdc34dSpatrick if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 26617acdc34dSpatrick BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 26627acdc34dSpatrick goto admfree; 26637acdc34dSpatrick 26647acdc34dSpatrick if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 26657acdc34dSpatrick 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 26667acdc34dSpatrick goto destroy; 26677acdc34dSpatrick 26687acdc34dSpatrick if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2669b70c9e02Spatrick &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0) 26707acdc34dSpatrick goto free; 26717acdc34dSpatrick 26727acdc34dSpatrick if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 26737acdc34dSpatrick NULL, BUS_DMA_NOWAIT) != 0) 26747acdc34dSpatrick goto unmap; 26757acdc34dSpatrick 26767acdc34dSpatrick return (adm); 26777acdc34dSpatrick 26787acdc34dSpatrick unmap: 26797acdc34dSpatrick bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 26807acdc34dSpatrick free: 26817acdc34dSpatrick bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 26827acdc34dSpatrick destroy: 26837acdc34dSpatrick bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 26847acdc34dSpatrick admfree: 26851f559dcaSderaadt free(adm, M_DEVBUF, sizeof(*adm)); 26867acdc34dSpatrick 26877acdc34dSpatrick return (NULL); 26887acdc34dSpatrick } 26897acdc34dSpatrick 26907acdc34dSpatrick void 26917acdc34dSpatrick ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 26927acdc34dSpatrick { 26937acdc34dSpatrick bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 26947acdc34dSpatrick bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 26957acdc34dSpatrick bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 26967acdc34dSpatrick bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 26971f559dcaSderaadt free(adm, M_DEVBUF, sizeof(*adm)); 26987acdc34dSpatrick } 26997acdc34dSpatrick 27007acdc34dSpatrick u_int32_t 27017acdc34dSpatrick ahci_read(struct ahci_softc *sc, bus_size_t r) 27027acdc34dSpatrick { 27037acdc34dSpatrick bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 27047acdc34dSpatrick BUS_SPACE_BARRIER_READ); 27057acdc34dSpatrick return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 27067acdc34dSpatrick } 27077acdc34dSpatrick 27087acdc34dSpatrick void 27097acdc34dSpatrick ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 27107acdc34dSpatrick { 27117acdc34dSpatrick bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 27127acdc34dSpatrick bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 27137acdc34dSpatrick BUS_SPACE_BARRIER_WRITE); 27147acdc34dSpatrick } 27157acdc34dSpatrick 27167acdc34dSpatrick int 27177acdc34dSpatrick ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 27187acdc34dSpatrick u_int32_t target) 27197acdc34dSpatrick { 27207acdc34dSpatrick int i; 27217acdc34dSpatrick 27227acdc34dSpatrick for (i = 0; i < 1000; i++) { 27237acdc34dSpatrick if ((ahci_read(sc, r) & mask) != target) 27247acdc34dSpatrick return (0); 27257acdc34dSpatrick delay(1000); 27267acdc34dSpatrick } 27277acdc34dSpatrick 27287acdc34dSpatrick return (1); 27297acdc34dSpatrick } 27307acdc34dSpatrick 27317acdc34dSpatrick u_int32_t 27327acdc34dSpatrick ahci_pread(struct ahci_port *ap, bus_size_t r) 27337acdc34dSpatrick { 27347acdc34dSpatrick bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 27357acdc34dSpatrick BUS_SPACE_BARRIER_READ); 27367acdc34dSpatrick return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 27377acdc34dSpatrick } 27387acdc34dSpatrick 27397acdc34dSpatrick void 27407acdc34dSpatrick ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 27417acdc34dSpatrick { 27427acdc34dSpatrick bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 27437acdc34dSpatrick bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 27447acdc34dSpatrick BUS_SPACE_BARRIER_WRITE); 27457acdc34dSpatrick } 27467acdc34dSpatrick 27477acdc34dSpatrick int 27487acdc34dSpatrick ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 27497acdc34dSpatrick u_int32_t target, int n) 27507acdc34dSpatrick { 27517acdc34dSpatrick int i; 27527acdc34dSpatrick 27537acdc34dSpatrick for (i = 0; i < n * 1000; i++) { 27547acdc34dSpatrick if ((ahci_pread(ap, r) & mask) == target) 27557acdc34dSpatrick return (0); 27567acdc34dSpatrick delay(1000); 27577acdc34dSpatrick } 27587acdc34dSpatrick 27597acdc34dSpatrick return (1); 27607acdc34dSpatrick } 27617acdc34dSpatrick 27627acdc34dSpatrick int 27637acdc34dSpatrick ahci_ata_probe(void *xsc, int port, int lun) 27647acdc34dSpatrick { 27657acdc34dSpatrick struct ahci_softc *sc = xsc; 27667acdc34dSpatrick struct ahci_port *ap = sc->sc_ports[port]; 27677acdc34dSpatrick 27687acdc34dSpatrick if (ap == NULL) 27697acdc34dSpatrick return (ATA_PORT_T_NONE); 27707acdc34dSpatrick 27717acdc34dSpatrick if (lun != 0) { 27727acdc34dSpatrick int pmp_port = lun - 1; 27737acdc34dSpatrick if (pmp_port >= ap->ap_pmp_ports) { 27747acdc34dSpatrick return (ATA_PORT_T_NONE); 27757acdc34dSpatrick } 27767acdc34dSpatrick return (ahci_pmp_port_probe(ap, pmp_port)); 27777acdc34dSpatrick } else { 27787acdc34dSpatrick return (ahci_port_signature(ap)); 27797acdc34dSpatrick } 27807acdc34dSpatrick } 27817acdc34dSpatrick 27827acdc34dSpatrick void 27837acdc34dSpatrick ahci_ata_free(void *xsc, int port, int lun) 27847acdc34dSpatrick { 27857acdc34dSpatrick 27867acdc34dSpatrick } 27877acdc34dSpatrick 27887acdc34dSpatrick struct ata_xfer * 27897acdc34dSpatrick ahci_ata_get_xfer(void *aaa_cookie, int port) 27907acdc34dSpatrick { 27917acdc34dSpatrick struct ahci_softc *sc = aaa_cookie; 27927acdc34dSpatrick struct ahci_port *ap = sc->sc_ports[port]; 27937acdc34dSpatrick struct ahci_ccb *ccb; 27947acdc34dSpatrick 27957acdc34dSpatrick ccb = ahci_get_ccb(ap); 27967acdc34dSpatrick if (ccb == NULL) { 27977acdc34dSpatrick DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 27987acdc34dSpatrick PORTNAME(ap)); 27997acdc34dSpatrick return (NULL); 28007acdc34dSpatrick } 28017acdc34dSpatrick 28027acdc34dSpatrick DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 28037acdc34dSpatrick PORTNAME(ap), ccb->ccb_slot); 28047acdc34dSpatrick 28057acdc34dSpatrick return ((struct ata_xfer *)ccb); 28067acdc34dSpatrick } 28077acdc34dSpatrick 28087acdc34dSpatrick void 28097acdc34dSpatrick ahci_ata_put_xfer(struct ata_xfer *xa) 28107acdc34dSpatrick { 28117acdc34dSpatrick struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 28127acdc34dSpatrick 28137acdc34dSpatrick DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 28147acdc34dSpatrick 28157acdc34dSpatrick ahci_put_ccb(ccb); 28167acdc34dSpatrick } 28177acdc34dSpatrick 28187acdc34dSpatrick void 28197acdc34dSpatrick ahci_ata_cmd(struct ata_xfer *xa) 28207acdc34dSpatrick { 28217acdc34dSpatrick struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 28227acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 28237acdc34dSpatrick int s; 28243f8c2449Sdlg u_int16_t flags; 28257acdc34dSpatrick 28267acdc34dSpatrick if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 28277acdc34dSpatrick goto failcmd; 28287acdc34dSpatrick 28297acdc34dSpatrick ccb->ccb_done = ahci_ata_cmd_done; 28307acdc34dSpatrick 28317acdc34dSpatrick cmd_slot = ccb->ccb_cmd_hdr; 28323f8c2449Sdlg flags = 5 /* FIS length (in DWORDs) */; 28333f8c2449Sdlg flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 28347acdc34dSpatrick 28357acdc34dSpatrick if (xa->flags & ATA_F_WRITE) 28363f8c2449Sdlg flags |= AHCI_CMD_LIST_FLAG_W; 28377acdc34dSpatrick 28387acdc34dSpatrick if (xa->flags & ATA_F_PACKET) 28393f8c2449Sdlg flags |= AHCI_CMD_LIST_FLAG_A; 28403f8c2449Sdlg 28413f8c2449Sdlg htolem16(&cmd_slot->flags, flags); 28427acdc34dSpatrick 28437acdc34dSpatrick if (ahci_load_prdt(ccb) != 0) 28447acdc34dSpatrick goto failcmd; 28457acdc34dSpatrick 28467acdc34dSpatrick timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 28477acdc34dSpatrick 28487acdc34dSpatrick xa->state = ATA_S_PENDING; 28497acdc34dSpatrick 28507acdc34dSpatrick if (xa->flags & ATA_F_POLL) 28517acdc34dSpatrick ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 28527acdc34dSpatrick else { 28537acdc34dSpatrick s = splbio(); 28547acdc34dSpatrick timeout_add_msec(&xa->stimeout, xa->timeout); 28557acdc34dSpatrick ahci_start(ccb); 28567acdc34dSpatrick splx(s); 28577acdc34dSpatrick } 28587acdc34dSpatrick 28597acdc34dSpatrick return; 28607acdc34dSpatrick 28617acdc34dSpatrick failcmd: 28627acdc34dSpatrick s = splbio(); 28637acdc34dSpatrick xa->state = ATA_S_ERROR; 28647acdc34dSpatrick ata_complete(xa); 28657acdc34dSpatrick splx(s); 28667acdc34dSpatrick } 28677acdc34dSpatrick 28687acdc34dSpatrick void 28697acdc34dSpatrick ahci_pmp_cmd_done(struct ahci_ccb *ccb) 28707acdc34dSpatrick { 28717acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 28727acdc34dSpatrick 28737acdc34dSpatrick if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 28747acdc34dSpatrick ahci_issue_pending_commands(ccb->ccb_port, 28757acdc34dSpatrick xa->flags & ATA_F_NCQ); 28767acdc34dSpatrick 28777acdc34dSpatrick xa->state = ATA_S_COMPLETE; 28787acdc34dSpatrick } 28797acdc34dSpatrick 28807acdc34dSpatrick 28817acdc34dSpatrick void 28827acdc34dSpatrick ahci_ata_cmd_done(struct ahci_ccb *ccb) 28837acdc34dSpatrick { 28847acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 28857acdc34dSpatrick 28867acdc34dSpatrick timeout_del(&xa->stimeout); 28877acdc34dSpatrick 28887acdc34dSpatrick if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 28897acdc34dSpatrick ahci_issue_pending_commands(ccb->ccb_port, 28907acdc34dSpatrick xa->flags & ATA_F_NCQ); 28917acdc34dSpatrick 28927acdc34dSpatrick ahci_unload_prdt(ccb); 28937acdc34dSpatrick 28947acdc34dSpatrick if (xa->state == ATA_S_ONCHIP) 28957acdc34dSpatrick xa->state = ATA_S_COMPLETE; 28967acdc34dSpatrick #ifdef DIAGNOSTIC 28977acdc34dSpatrick else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 28987acdc34dSpatrick printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 28997acdc34dSpatrick "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 29007acdc34dSpatrick ccb->ccb_slot); 29017acdc34dSpatrick #endif 29027acdc34dSpatrick if (xa->state != ATA_S_TIMEOUT) 29037acdc34dSpatrick ata_complete(xa); 29047acdc34dSpatrick } 29057acdc34dSpatrick 29067acdc34dSpatrick void 29077acdc34dSpatrick ahci_ata_cmd_timeout(void *arg) 29087acdc34dSpatrick { 29097acdc34dSpatrick struct ahci_ccb *ccb = arg; 29107acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 29117acdc34dSpatrick struct ahci_port *ap = ccb->ccb_port; 29127acdc34dSpatrick int s, ccb_was_started, ncq_cmd; 29137acdc34dSpatrick volatile u_int32_t *active; 29147acdc34dSpatrick 29157acdc34dSpatrick s = splbio(); 29167acdc34dSpatrick 29177acdc34dSpatrick ncq_cmd = (xa->flags & ATA_F_NCQ); 29187acdc34dSpatrick active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 29197acdc34dSpatrick 29207acdc34dSpatrick if (ccb->ccb_xa.state == ATA_S_PENDING) { 29217acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 29227acdc34dSpatrick "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 29237acdc34dSpatrick TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 29247acdc34dSpatrick ccb_was_started = 0; 29257acdc34dSpatrick } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 29267acdc34dSpatrick 1 << ccb->ccb_slot)) { 29277acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 29287acdc34dSpatrick "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 29297acdc34dSpatrick goto ret; 29307acdc34dSpatrick } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 29317acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 29327acdc34dSpatrick "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 29337acdc34dSpatrick ISSET(*active, 1 << ccb->ccb_slot) ? 29347acdc34dSpatrick " but slot is still active?" : "."); 29357acdc34dSpatrick goto ret; 29367acdc34dSpatrick } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 29377acdc34dSpatrick AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 29387acdc34dSpatrick 1 << ccb->ccb_slot)) { 29397acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 29407acdc34dSpatrick "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 29417acdc34dSpatrick ccb->ccb_slot); 29427acdc34dSpatrick *active &= ~(1 << ccb->ccb_slot); 29437acdc34dSpatrick ccb->ccb_done(ccb); 29447acdc34dSpatrick goto ret; 29457acdc34dSpatrick } else { 29467acdc34dSpatrick ccb_was_started = 1; 29477acdc34dSpatrick } 29487acdc34dSpatrick 29497acdc34dSpatrick /* Complete the slot with a timeout error. */ 29507acdc34dSpatrick ccb->ccb_xa.state = ATA_S_TIMEOUT; 29517acdc34dSpatrick *active &= ~(1 << ccb->ccb_slot); 29527acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 29537acdc34dSpatrick ccb->ccb_done(ccb); /* This won't issue pending commands or run the 29547acdc34dSpatrick atascsi completion. */ 29557acdc34dSpatrick 29567acdc34dSpatrick /* Reset port to abort running command. */ 29577acdc34dSpatrick if (ccb_was_started) { 29587acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 29597acdc34dSpatrick "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 29607acdc34dSpatrick ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 29617acdc34dSpatrick if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 29627acdc34dSpatrick != 0) { 29637acdc34dSpatrick printf("%s: failed to reset port during timeout " 29647acdc34dSpatrick "handling, disabling it\n", PORTNAME(ap)); 29657acdc34dSpatrick ap->ap_state = AP_S_FATAL_ERROR; 29667acdc34dSpatrick } 29677acdc34dSpatrick 29687acdc34dSpatrick /* Restart any other commands that were aborted by the reset. */ 29697acdc34dSpatrick if (*active) { 29707acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 29717acdc34dSpatrick "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 29727acdc34dSpatrick *active); 29737acdc34dSpatrick if (ncq_cmd) 29747acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_SACT, *active); 29757acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, *active); 29767acdc34dSpatrick } 29777acdc34dSpatrick } 29787acdc34dSpatrick 29797acdc34dSpatrick /* Issue any pending commands now. */ 29807acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 29817acdc34dSpatrick if (ccb_was_started) 29827acdc34dSpatrick ahci_issue_pending_commands(ap, ncq_cmd); 29837acdc34dSpatrick else if (ap->ap_active == 0) 29847acdc34dSpatrick ahci_issue_pending_ncq_commands(ap); 29857acdc34dSpatrick 29867acdc34dSpatrick /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 29877acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 29887acdc34dSpatrick ata_complete(xa); 29897acdc34dSpatrick 29907acdc34dSpatrick DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 29917acdc34dSpatrick ret: 29927acdc34dSpatrick splx(s); 29937acdc34dSpatrick } 29947acdc34dSpatrick 29957acdc34dSpatrick void 29967acdc34dSpatrick ahci_empty_done(struct ahci_ccb *ccb) 29977acdc34dSpatrick { 29987acdc34dSpatrick if (ccb->ccb_xa.state != ATA_S_ERROR) 29997acdc34dSpatrick ccb->ccb_xa.state = ATA_S_COMPLETE; 30007acdc34dSpatrick } 30017acdc34dSpatrick 30027acdc34dSpatrick int 30037acdc34dSpatrick ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 30047acdc34dSpatrick { 30057acdc34dSpatrick struct ahci_ccb *ccb; 30067acdc34dSpatrick struct ata_fis_h2d *fis; 30077acdc34dSpatrick int error; 30087acdc34dSpatrick 3009189d7e68Skrw ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 30107acdc34dSpatrick ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 30117acdc34dSpatrick ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 30127acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 30137acdc34dSpatrick 3014414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 30157acdc34dSpatrick fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 30167acdc34dSpatrick fis->type = ATA_FIS_TYPE_H2D; 30177acdc34dSpatrick fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 30187acdc34dSpatrick fis->command = ATA_C_READ_PM; 30197acdc34dSpatrick fis->features = which; 30207acdc34dSpatrick fis->device = target | ATA_H2D_DEVICE_LBA; 30217acdc34dSpatrick fis->control = ATA_FIS_CONTROL_4BIT; 30227acdc34dSpatrick 30237acdc34dSpatrick if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 30247acdc34dSpatrick error = 1; 30257acdc34dSpatrick } else { 30267acdc34dSpatrick *datap = ccb->ccb_xa.rfis.sector_count | 30277acdc34dSpatrick (ccb->ccb_xa.rfis.lba_low << 8) | 30287acdc34dSpatrick (ccb->ccb_xa.rfis.lba_mid << 16) | 30297acdc34dSpatrick (ccb->ccb_xa.rfis.lba_high << 24); 30307acdc34dSpatrick error = 0; 30317acdc34dSpatrick } 30327acdc34dSpatrick ahci_put_pmp_ccb(ccb); 30337acdc34dSpatrick return (error); 30347acdc34dSpatrick } 30357acdc34dSpatrick 30367acdc34dSpatrick int 30377acdc34dSpatrick ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 30387acdc34dSpatrick { 30397acdc34dSpatrick struct ahci_ccb *ccb; 30407acdc34dSpatrick struct ata_fis_h2d *fis; 30417acdc34dSpatrick int error; 30427acdc34dSpatrick 3043189d7e68Skrw ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 30447acdc34dSpatrick ccb->ccb_xa.flags = ATA_F_POLL; 30457acdc34dSpatrick ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 30467acdc34dSpatrick ccb->ccb_xa.state = ATA_S_PENDING; 30477acdc34dSpatrick 3048414fa24dSdlg memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 30497acdc34dSpatrick fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 30507acdc34dSpatrick fis->type = ATA_FIS_TYPE_H2D; 30517acdc34dSpatrick fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 30527acdc34dSpatrick fis->command = ATA_C_WRITE_PM; 30537acdc34dSpatrick fis->features = which; 30547acdc34dSpatrick fis->device = target | ATA_H2D_DEVICE_LBA; 30557acdc34dSpatrick fis->sector_count = (u_int8_t)data; 30567acdc34dSpatrick fis->lba_low = (u_int8_t)(data >> 8); 30577acdc34dSpatrick fis->lba_mid = (u_int8_t)(data >> 16); 30587acdc34dSpatrick fis->lba_high = (u_int8_t)(data >> 24); 30597acdc34dSpatrick fis->control = ATA_FIS_CONTROL_4BIT; 30607acdc34dSpatrick 30617acdc34dSpatrick error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 30627acdc34dSpatrick ahci_put_pmp_ccb(ccb); 30637acdc34dSpatrick return (error); 30647acdc34dSpatrick } 30657acdc34dSpatrick 30667acdc34dSpatrick int 30677acdc34dSpatrick ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 30687acdc34dSpatrick { 30697acdc34dSpatrick int error; 30707acdc34dSpatrick 30717acdc34dSpatrick error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 30727acdc34dSpatrick if (error == 0) 30737acdc34dSpatrick error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 30747acdc34dSpatrick if (error) 30757acdc34dSpatrick *datap = 0; 30767acdc34dSpatrick 30777acdc34dSpatrick return (error); 30787acdc34dSpatrick } 30797acdc34dSpatrick 30807acdc34dSpatrick int 30817acdc34dSpatrick ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 30827acdc34dSpatrick { 30837acdc34dSpatrick u_int32_t chipid; 30847acdc34dSpatrick u_int32_t rev; 30857acdc34dSpatrick u_int32_t nports; 30867acdc34dSpatrick u_int32_t features; 30877acdc34dSpatrick u_int32_t enabled; 30887acdc34dSpatrick int s; 30897acdc34dSpatrick 30907acdc34dSpatrick s = splbio(); 30917acdc34dSpatrick 30927acdc34dSpatrick if (ahci_pmp_read(ap, 15, 0, &chipid) || 30937acdc34dSpatrick ahci_pmp_read(ap, 15, 1, &rev) || 30947acdc34dSpatrick ahci_pmp_read(ap, 15, 2, &nports) || 30957acdc34dSpatrick ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 30967acdc34dSpatrick ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 30977acdc34dSpatrick printf("%s: port multiplier identification failed\n", 30987acdc34dSpatrick PORTNAME(ap)); 30997acdc34dSpatrick splx(s); 31007acdc34dSpatrick return (1); 31017acdc34dSpatrick } 31027acdc34dSpatrick splx(s); 31037acdc34dSpatrick 31047acdc34dSpatrick nports &= 0x0F; 31057acdc34dSpatrick 31067acdc34dSpatrick /* ignore SEMB port on SiI3726 port multiplier chips */ 31077acdc34dSpatrick if (chipid == 0x37261095) { 31087acdc34dSpatrick nports--; 31097acdc34dSpatrick } 31107acdc34dSpatrick 31117acdc34dSpatrick printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 31127acdc34dSpatrick "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 31137acdc34dSpatrick SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 31147acdc34dSpatrick SATA_PFMT_PM_FEA); 31157acdc34dSpatrick 31167acdc34dSpatrick *ret_nports = nports; 31177acdc34dSpatrick return (0); 31187acdc34dSpatrick } 31197acdc34dSpatrick 31207acdc34dSpatrick 31217acdc34dSpatrick #ifdef HIBERNATE 31227acdc34dSpatrick void 31237acdc34dSpatrick ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 31247acdc34dSpatrick { 31257acdc34dSpatrick ccb->ccb_cmd_hdr->prdbc = 0; 31267acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 31277acdc34dSpatrick } 31287acdc34dSpatrick 31297acdc34dSpatrick int 31307acdc34dSpatrick ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 31317acdc34dSpatrick { 31327acdc34dSpatrick u_int32_t is, ci_saved; 31337acdc34dSpatrick int process_error = 0; 31347acdc34dSpatrick 31357acdc34dSpatrick is = ahci_pread(ap, AHCI_PREG_IS); 31367acdc34dSpatrick 31377acdc34dSpatrick ci_saved = ahci_pread(ap, AHCI_PREG_CI); 31387acdc34dSpatrick 31397acdc34dSpatrick if (is & AHCI_PREG_IS_DHRS) { 31407acdc34dSpatrick u_int32_t tfd; 31417acdc34dSpatrick u_int32_t cmd; 31427acdc34dSpatrick 31437acdc34dSpatrick tfd = ahci_pread(ap, AHCI_PREG_TFD); 31447acdc34dSpatrick cmd = ahci_pread(ap, AHCI_PREG_CMD); 31457acdc34dSpatrick if ((tfd & AHCI_PREG_TFD_STS_ERR) && 31467acdc34dSpatrick (cmd & AHCI_PREG_CMD_CR) == 0) { 31477acdc34dSpatrick process_error = 1; 31487acdc34dSpatrick } else { 31497acdc34dSpatrick ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 31507acdc34dSpatrick } 31517acdc34dSpatrick } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 31527acdc34dSpatrick AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 31537acdc34dSpatrick process_error = 1; 31547acdc34dSpatrick } 31557acdc34dSpatrick 31567acdc34dSpatrick /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 31577acdc34dSpatrick if (process_error) { 31587acdc34dSpatrick 31597acdc34dSpatrick /* Turn off ST to clear CI and SACT. */ 31607acdc34dSpatrick ahci_port_stop(ap, 0); 31617acdc34dSpatrick 31627acdc34dSpatrick /* just return an error indicator? we can't meaningfully 31637acdc34dSpatrick * recover, and on the way back out we'll DVACT_RESUME which 31647acdc34dSpatrick * resets and reinits the port. 31657acdc34dSpatrick */ 31667acdc34dSpatrick return (EIO); 31677acdc34dSpatrick } 31687acdc34dSpatrick 31697acdc34dSpatrick /* command is finished when the bit in CI for the slot goes to 0 */ 31707acdc34dSpatrick if (ci_saved & (1 << ccb->ccb_slot)) { 31717acdc34dSpatrick return (EAGAIN); 31727acdc34dSpatrick } 31737acdc34dSpatrick 31747acdc34dSpatrick return (0); 31757acdc34dSpatrick } 31767acdc34dSpatrick 31777acdc34dSpatrick void 31787acdc34dSpatrick ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 31797acdc34dSpatrick { 31807acdc34dSpatrick struct ata_xfer *xa = &ccb->ccb_xa; 3181014fa59dSmlarkin struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 31827acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 31837acdc34dSpatrick int i; 31847acdc34dSpatrick paddr_t data_phys; 31857acdc34dSpatrick u_int64_t data_bus_phys; 31867acdc34dSpatrick vaddr_t data_addr; 31877acdc34dSpatrick size_t seglen; 31887acdc34dSpatrick size_t buflen; 31897acdc34dSpatrick 31907acdc34dSpatrick if (xa->datalen == 0) { 31917acdc34dSpatrick ccb->ccb_cmd_hdr->prdtl = 0; 31927acdc34dSpatrick return; 31937acdc34dSpatrick } 31947acdc34dSpatrick 31957acdc34dSpatrick /* derived from i386/amd64 _bus_dma_load_buffer; 31967acdc34dSpatrick * for amd64 the buffer will always be dma safe. 31977acdc34dSpatrick */ 31987acdc34dSpatrick 31997acdc34dSpatrick buflen = xa->datalen; 32007acdc34dSpatrick data_addr = (vaddr_t)xa->data; 32017acdc34dSpatrick for (i = 0; buflen > 0; i++) { 32027acdc34dSpatrick pmap_extract(pmap_kernel(), data_addr, &data_phys); 32037acdc34dSpatrick data_bus_phys = data_phys; 32047acdc34dSpatrick 32057acdc34dSpatrick seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 32067acdc34dSpatrick if (buflen < seglen) 32077acdc34dSpatrick seglen = buflen; 32087acdc34dSpatrick 3209b1514dfeSdlg ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3210b1514dfeSdlg 32117acdc34dSpatrick data_addr += seglen; 32127acdc34dSpatrick buflen -= seglen; 32137acdc34dSpatrick } 32147acdc34dSpatrick 32153f8c2449Sdlg htolem16(&cmd_slot->prdtl, i); 32167acdc34dSpatrick } 32177acdc34dSpatrick 32187acdc34dSpatrick int 32197acdc34dSpatrick ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 32207acdc34dSpatrick int op, void *page) 32217acdc34dSpatrick { 32227acdc34dSpatrick /* we use the 'real' ahci_port and ahci_softc here, but 32237acdc34dSpatrick * never write to them 32247acdc34dSpatrick */ 32257acdc34dSpatrick struct { 32267acdc34dSpatrick struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 32277acdc34dSpatrick struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 32287acdc34dSpatrick /* cmd table isn't actually used because of mysteries */ 32297acdc34dSpatrick struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 32307acdc34dSpatrick struct ahci_port *ap; 32317acdc34dSpatrick struct ahci_ccb ccb_buf; 32327acdc34dSpatrick struct ahci_ccb *ccb; 32337acdc34dSpatrick struct ahci_cmd_hdr *hdr_buf; 32347acdc34dSpatrick int pmp_port; 3235a1645687Sderaadt daddr_t poffset; 3236a1645687Sderaadt size_t psize; 32377acdc34dSpatrick } *my = page; 32387acdc34dSpatrick struct ata_fis_h2d *fis; 32397acdc34dSpatrick u_int32_t sector_count; 32407acdc34dSpatrick struct ahci_cmd_hdr *cmd_slot; 32417acdc34dSpatrick int rc; 32427acdc34dSpatrick int timeout; 32433f8c2449Sdlg u_int16_t flags; 32447acdc34dSpatrick 32457acdc34dSpatrick if (op == HIB_INIT) { 32467acdc34dSpatrick struct device *disk; 32477acdc34dSpatrick struct device *scsibus; 32487acdc34dSpatrick struct ahci_softc *sc; 32497acdc34dSpatrick extern struct cfdriver sd_cd; 32507acdc34dSpatrick struct scsi_link *link; 32517acdc34dSpatrick struct scsibus_softc *bus_sc; 32527acdc34dSpatrick int port; 32537acdc34dSpatrick paddr_t page_phys; 32547acdc34dSpatrick u_int64_t item_phys; 32557acdc34dSpatrick u_int32_t cmd; 32567acdc34dSpatrick 3257a1645687Sderaadt my->poffset = blkno; 3258a1645687Sderaadt my->psize = size; 3259a1645687Sderaadt 32607acdc34dSpatrick /* map dev to an ahci port */ 32617acdc34dSpatrick disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 32627acdc34dSpatrick scsibus = disk->dv_parent; 32637acdc34dSpatrick sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 32647acdc34dSpatrick 32657acdc34dSpatrick /* find the scsi_link for the device, which has the port */ 32667acdc34dSpatrick port = -1; 32677acdc34dSpatrick bus_sc = (struct scsibus_softc *)scsibus; 326820f90945Skrw SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) { 32697acdc34dSpatrick if (link->device_softc == disk) { 32707acdc34dSpatrick port = link->target; 32717acdc34dSpatrick if (link->lun > 0) 32727acdc34dSpatrick my->pmp_port = link->lun - 1; 32737acdc34dSpatrick else 32747acdc34dSpatrick my->pmp_port = 0; 32757acdc34dSpatrick 32767acdc34dSpatrick break; 32777acdc34dSpatrick } 32787acdc34dSpatrick } 32797acdc34dSpatrick if (port == -1) { 32807acdc34dSpatrick /* don't know where the disk is */ 32817acdc34dSpatrick return (EIO); 32827acdc34dSpatrick } 32837acdc34dSpatrick 32847acdc34dSpatrick my->ap = sc->sc_ports[port]; 32857acdc34dSpatrick 32867acdc34dSpatrick /* we're going to use the first command slot, 32877acdc34dSpatrick * so ensure it's not already in use 32887acdc34dSpatrick */ 32897acdc34dSpatrick if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 32907acdc34dSpatrick /* this shouldn't happen, we should be idle */ 32917acdc34dSpatrick return (EIO); 32927acdc34dSpatrick } 32937acdc34dSpatrick 32947acdc34dSpatrick /* stop the port so we can relocate to the hibernate page */ 32957acdc34dSpatrick if (ahci_port_stop(my->ap, 1)) { 32967acdc34dSpatrick return (EIO); 32977acdc34dSpatrick } 32987acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 32997acdc34dSpatrick 33007acdc34dSpatrick pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 33017acdc34dSpatrick 33027acdc34dSpatrick /* Setup RFIS base address */ 33037acdc34dSpatrick item_phys = page_phys + ((void *)&my->rfis - page); 33047acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_FBU, 33057acdc34dSpatrick (u_int32_t)(item_phys >> 32)); 33067acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 33077acdc34dSpatrick 33087acdc34dSpatrick /* Enable FIS reception and activate port. */ 33097acdc34dSpatrick cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 33107acdc34dSpatrick cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 33117acdc34dSpatrick AHCI_PREG_CMD_SUD; 33127acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 33137acdc34dSpatrick AHCI_PREG_CMD_ICC_ACTIVE); 33147acdc34dSpatrick 33157acdc34dSpatrick /* Check whether port activated. */ 33167acdc34dSpatrick cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 33177acdc34dSpatrick if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 33187acdc34dSpatrick return (EIO); 33197acdc34dSpatrick } 33207acdc34dSpatrick 33217acdc34dSpatrick /* Set up the single CCB */ 33227acdc34dSpatrick my->ccb = &my->ccb_buf; 33237acdc34dSpatrick my->ccb->ccb_slot = 0; 33247acdc34dSpatrick my->ccb->ccb_port = my->ap; 33257acdc34dSpatrick 33267acdc34dSpatrick /* Setup command list base address */ 33277acdc34dSpatrick item_phys = page_phys + ((void *)&my->cmd_hdr - page); 33287acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_CLBU, 33297acdc34dSpatrick (u_int32_t)(item_phys >> 32)); 33307acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 33317acdc34dSpatrick 33327acdc34dSpatrick my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 33337acdc34dSpatrick 333477db1dd4Skrw /* use existing cmd table - moving to a new one fails */ 33357acdc34dSpatrick my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 33367acdc34dSpatrick pmap_extract(pmap_kernel(), 33377acdc34dSpatrick (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 33387acdc34dSpatrick &page_phys); 33397acdc34dSpatrick item_phys = page_phys; 33407acdc34dSpatrick #if 0 33417acdc34dSpatrick /* use cmd table in hibernate page (doesn't work) */ 33427acdc34dSpatrick my->ccb->ccb_cmd_table = &my->cmd_table; 33437acdc34dSpatrick item_phys = page_phys + ((void *)&my->cmd_table - page); 33447acdc34dSpatrick #endif 33454d3a7877Sdlg htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 33467acdc34dSpatrick 33477acdc34dSpatrick my->ccb->ccb_xa.fis = 33487acdc34dSpatrick (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 33497acdc34dSpatrick my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 33507acdc34dSpatrick my->ccb->ccb_xa.tag = 0; 33517acdc34dSpatrick 33527acdc34dSpatrick /* Wait for ICC change to complete */ 33537acdc34dSpatrick ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 33547acdc34dSpatrick 33557acdc34dSpatrick if (ahci_port_start(my->ap, 0)) { 33567acdc34dSpatrick return (EIO); 33577acdc34dSpatrick } 33587acdc34dSpatrick 33597acdc34dSpatrick /* Flush interrupts for port */ 33607acdc34dSpatrick ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 33617acdc34dSpatrick AHCI_PREG_IS)); 33627acdc34dSpatrick ahci_write(sc, AHCI_REG_IS, 1 << port); 33637acdc34dSpatrick 33647acdc34dSpatrick ahci_enable_interrupts(my->ap); 33657acdc34dSpatrick return (0); 33667acdc34dSpatrick } else if (op == HIB_DONE) { 33677acdc34dSpatrick ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 33687acdc34dSpatrick return (0); 33697acdc34dSpatrick } 33707acdc34dSpatrick 3371a1645687Sderaadt if (blkno > my->psize) 3372a1645687Sderaadt return (E2BIG); 3373a1645687Sderaadt blkno += my->poffset; 3374a1645687Sderaadt 33757acdc34dSpatrick /* build fis */ 33767acdc34dSpatrick sector_count = size / 512; /* dlg promises this is okay */ 33777acdc34dSpatrick my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 33787acdc34dSpatrick fis = my->ccb->ccb_xa.fis; 33797acdc34dSpatrick fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 33807acdc34dSpatrick fis->lba_low = blkno & 0xff; 33817acdc34dSpatrick fis->lba_mid = (blkno >> 8) & 0xff; 33827acdc34dSpatrick fis->lba_high = (blkno >> 16) & 0xff; 33837acdc34dSpatrick 33847acdc34dSpatrick if (sector_count > 0x100 || blkno > 0xfffffff) { 33857acdc34dSpatrick /* Use LBA48 */ 33867acdc34dSpatrick fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 33877acdc34dSpatrick ATA_C_READDMA_EXT; 33887acdc34dSpatrick fis->device = ATA_H2D_DEVICE_LBA; 33897acdc34dSpatrick fis->lba_low_exp = (blkno >> 24) & 0xff; 33907acdc34dSpatrick fis->lba_mid_exp = (blkno >> 32) & 0xff; 33917acdc34dSpatrick fis->lba_high_exp = (blkno >> 40) & 0xff; 33927acdc34dSpatrick fis->sector_count = sector_count & 0xff; 33937acdc34dSpatrick fis->sector_count_exp = (sector_count >> 8) & 0xff; 33947acdc34dSpatrick } else { 33957acdc34dSpatrick /* Use LBA */ 33967acdc34dSpatrick fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 33977acdc34dSpatrick fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 33987acdc34dSpatrick fis->sector_count = sector_count & 0xff; 33997acdc34dSpatrick } 34007acdc34dSpatrick 34017acdc34dSpatrick my->ccb->ccb_xa.data = (void *)addr; 34027acdc34dSpatrick my->ccb->ccb_xa.datalen = size; 34037acdc34dSpatrick my->ccb->ccb_xa.pmp_port = my->pmp_port; 34047acdc34dSpatrick my->ccb->ccb_xa.flags |= ATA_F_POLL; 34057acdc34dSpatrick 34067acdc34dSpatrick cmd_slot = my->ccb->ccb_cmd_hdr; 34073f8c2449Sdlg flags = 5; /* FIS length (in DWORDs) */ 34083f8c2449Sdlg flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 34097acdc34dSpatrick 34107acdc34dSpatrick if (op == HIB_W) 34113f8c2449Sdlg flags |= AHCI_CMD_LIST_FLAG_W; 34123f8c2449Sdlg 34133f8c2449Sdlg htolem16(&cmd_slot->flags, flags); 34147acdc34dSpatrick 34157acdc34dSpatrick ahci_hibernate_load_prdt(my->ccb); 34167acdc34dSpatrick 34177acdc34dSpatrick ahci_hibernate_io_start(my->ap, my->ccb); 34180b4b247eSmlarkin timeout = 1000000; 34197acdc34dSpatrick while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 34200b4b247eSmlarkin delay(1); 34217acdc34dSpatrick timeout--; 34227acdc34dSpatrick if (timeout == 0) { 34237acdc34dSpatrick return (EIO); 34247acdc34dSpatrick } 34257acdc34dSpatrick } 34267acdc34dSpatrick 34277acdc34dSpatrick return (0); 34287acdc34dSpatrick } 34297acdc34dSpatrick 34307acdc34dSpatrick #endif 3431