Lines Matching full:ch
64 static void ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus);
65 static void ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb);
70 static int ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
76 static void ahci_reset(struct ahci_channel *ch);
77 static void ahci_start(struct ahci_channel *ch, int fbs);
78 static void ahci_stop(struct ahci_channel *ch);
79 static void ahci_clo(struct ahci_channel *ch);
80 static void ahci_start_fr(struct ahci_channel *ch);
81 static void ahci_stop_fr(struct ahci_channel *ch);
82 static int ahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr);
83 static uint32_t ahci_ch_detval(struct ahci_channel *ch, uint32_t val);
85 static int ahci_sata_connect(struct ahci_channel *ch);
86 static int ahci_sata_phy_reset(struct ahci_channel *ch);
87 static int ahci_wait_ready(struct ahci_channel *ch, int t, int t0);
89 static void ahci_issue_recovery(struct ahci_channel *ch);
90 static void ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb);
91 static void ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb);
105 ahci_ch_detval(struct ahci_channel *ch, uint32_t val)
108 return ch->disablephy ? ATA_SC_DET_DISABLE : val;
488 "ch%d" : "%d", i);
723 ahci_attached(device_t dev, struct ahci_channel *ch)
728 ctlr->ch[ch->unit] = ch;
733 ahci_detached(device_t dev, struct ahci_channel *ch)
738 mtx_lock(&ch->mtx);
739 ctlr->ch[ch->unit] = NULL;
740 mtx_unlock(&ch->mtx);
748 struct ahci_channel *ch;
752 ch = ctlr->ch[n];
753 if (ch != NULL)
754 mtx_lock(&ch->mtx);
756 return (ch);
760 ahci_putch(struct ahci_channel *ch)
763 mtx_unlock(&ch->mtx);
777 struct ahci_channel *ch;
780 ch = arg1;
781 value = ch->disablephy;
786 mtx_lock(&ch->mtx);
787 ch->disablephy = value;
789 ahci_ch_deinit(ch->dev);
791 ahci_ch_init(ch->dev);
792 ahci_phy_check_events(ch, ATA_SE_PHY_CHANGED | ATA_SE_EXCHANGED);
794 mtx_unlock(&ch->mtx);
803 struct ahci_channel *ch = device_get_softc(dev);
810 ch->dev = dev;
811 ch->unit = (intptr_t)device_get_ivars(dev);
812 ch->caps = ctlr->caps;
813 ch->caps2 = ctlr->caps2;
814 ch->start = ctlr->ch_start;
815 ch->quirks = ctlr->quirks;
816 ch->vendorid = ctlr->vendorid;
817 ch->deviceid = ctlr->deviceid;
818 ch->subvendorid = ctlr->subvendorid;
819 ch->subdeviceid = ctlr->subdeviceid;
820 ch->numslots = ((ch->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1;
821 mtx_init(&ch->mtx, "AHCI channel lock", NULL, MTX_DEF);
822 ch->pm_level = 0;
824 device_get_unit(dev), "pm_level", &ch->pm_level);
825 STAILQ_INIT(&ch->doneq);
826 if (ch->pm_level > 3)
827 callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
828 callout_init_mtx(&ch->reset_timer, &ch->mtx, 0);
830 if ((ctlr->quirks & AHCI_Q_SATA1_UNIT0) && ch->unit == 0)
832 if (ch->quirks & AHCI_Q_SATA2)
837 ch->user[i].revision = sata_rev;
838 ch->user[i].mode = 0;
839 ch->user[i].bytecount = 8192;
840 ch->user[i].tags = ch->numslots;
841 ch->user[i].caps = 0;
842 ch->curr[i] = ch->user[i];
843 if (ch->pm_level) {
844 ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
848 ch->user[i].caps |= CTS_SATA_CAPS_H_DMAAA |
852 if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
855 ch->chcaps = ATA_INL(ch->r_mem, AHCI_P_CMD);
858 ch->chcaps |= AHCI_P_CMD_FBSCP;
859 if (ch->caps2 & AHCI_CAP2_SDS)
860 ch->chscaps = ATA_INL(ch->r_mem, AHCI_P_DEVSLP);
863 (ch->chcaps & AHCI_P_CMD_HPCP) ? " HPCP":"",
864 (ch->chcaps & AHCI_P_CMD_MPSP) ? " MPSP":"",
865 (ch->chcaps & AHCI_P_CMD_CPD) ? " CPD":"",
866 (ch->chcaps & AHCI_P_CMD_ESP) ? " ESP":"",
867 (ch->chcaps & AHCI_P_CMD_FBSCP) ? " FBSCP":"",
868 (ch->chscaps & AHCI_P_DEVSLP_DSP) ? " DSP":"");
872 mtx_lock(&ch->mtx);
875 if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
881 if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
883 ch, &ch->ih))) {
889 devq = cam_simq_alloc(ch->numslots);
896 ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch,
897 device_get_unit(dev), (struct mtx *)&ch->mtx,
898 (ch->quirks & AHCI_Q_NOCCS) ? 1 : min(2, ch->numslots),
899 (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0,
901 if (ch->sim == NULL) {
907 if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
912 if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
918 if (ch->pm_level > 3) {
919 callout_reset(&ch->pm_timer,
920 (ch->pm_level == 4) ? hz / 1000 : hz / 8,
921 ahci_ch_pm, ch);
923 mtx_unlock(&ch->mtx);
924 ahci_attached(device_get_parent(dev), ch);
928 CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, ch,
933 xpt_bus_deregister(cam_sim_path(ch->sim));
935 cam_sim_free(ch->sim, /*free_devq*/TRUE);
937 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
939 bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
940 mtx_unlock(&ch->mtx);
941 mtx_destroy(&ch->mtx);
948 struct ahci_channel *ch = device_get_softc(dev);
950 ahci_detached(device_get_parent(dev), ch);
951 mtx_lock(&ch->mtx);
952 xpt_async(AC_LOST_DEVICE, ch->path, NULL);
954 if (ch->resetting) {
955 ch->resetting = 0;
956 xpt_release_simq(ch->sim, TRUE);
958 xpt_free_path(ch->path);
959 xpt_bus_deregister(cam_sim_path(ch->sim));
960 cam_sim_free(ch->sim, /*free_devq*/TRUE);
961 mtx_unlock(&ch->mtx);
963 if (ch->pm_level > 3)
964 callout_drain(&ch->pm_timer);
965 callout_drain(&ch->reset_timer);
966 bus_teardown_intr(dev, ch->r_irq, ch->ih);
967 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
973 bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
974 mtx_destroy(&ch->mtx);
981 struct ahci_channel *ch = device_get_softc(dev);
985 ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
987 work = ch->dma.work_bus + AHCI_CL_OFFSET;
988 ATA_OUTL(ch->r_mem, AHCI_P_CLB, work & 0xffffffff);
989 ATA_OUTL(ch->r_mem, AHCI_P_CLBU, work >> 32);
990 work = ch->dma.rfis_bus;
991 ATA_OUTL(ch->r_mem, AHCI_P_FB, work & 0xffffffff);
992 ATA_OUTL(ch->r_mem, AHCI_P_FBU, work >> 32);
994 ATA_OUTL(ch->r_mem, AHCI_P_CMD,
996 ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) |
997 ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 )));
998 ahci_start_fr(ch);
999 ahci_start(ch, 1);
1006 struct ahci_channel *ch = device_get_softc(dev);
1009 ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
1011 ahci_stop(ch);
1012 ahci_stop_fr(ch);
1013 ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0);
1015 ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0);
1017 ATA_OUTL(ch->r_mem, AHCI_P_CMD, AHCI_P_CMD_SLUMBER);
1020 ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE);
1027 struct ahci_channel *ch = device_get_softc(dev);
1029 mtx_lock(&ch->mtx);
1030 xpt_freeze_simq(ch->sim, 1);
1032 if (ch->resetting) {
1033 ch->resetting = 0;
1034 callout_stop(&ch->reset_timer);
1035 xpt_release_simq(ch->sim, TRUE);
1037 while (ch->oslots)
1038 msleep(ch, &ch->mtx, PRIBIO, "ahcisusp", hz/100);
1040 mtx_unlock(&ch->mtx);
1047 struct ahci_channel *ch = device_get_softc(dev);
1049 mtx_lock(&ch->mtx);
1051 ahci_reset(ch);
1052 xpt_release_simq(ch->sim, TRUE);
1053 mtx_unlock(&ch->mtx);
1080 struct ahci_channel *ch = device_get_softc(dev);
1089 0, NULL, NULL, &ch->dma.work_tag);
1092 error = bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work,
1093 BUS_DMA_ZERO, &ch->dma.work_map);
1096 error = bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
1099 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
1102 ch->dma.work_bus = dcba.maddr;
1104 if (ch->chcaps & AHCI_P_CMD_FBSCP)
1111 0, NULL, NULL, &ch->dma.rfis_tag);
1114 error = bus_dmamem_alloc(ch->dma.rfis_tag, (void **)&ch->dma.rfis, 0,
1115 &ch->dma.rfis_map);
1118 error = bus_dmamap_load(ch->dma.rfis_tag, ch->dma.rfis_map, ch->dma.rfis,
1121 bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map);
1124 ch->dma.rfis_bus = dcba.maddr;
1130 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag);
1153 struct ahci_channel *ch = device_get_softc(dev);
1155 if (ch->dma.data_tag) {
1156 bus_dma_tag_destroy(ch->dma.data_tag);
1157 ch->dma.data_tag = NULL;
1159 if (ch->dma.rfis_bus) {
1160 bus_dmamap_unload(ch->dma.rfis_tag, ch->dma.rfis_map);
1161 bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map);
1162 ch->dma.rfis_bus = 0;
1163 ch->dma.rfis = NULL;
1165 if (ch->dma.work_bus) {
1166 bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
1167 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
1168 ch->dma.work_bus = 0;
1169 ch->dma.work = NULL;
1171 if (ch->dma.work_tag) {
1172 bus_dma_tag_destroy(ch->dma.work_tag);
1173 ch->dma.work_tag = NULL;
1180 struct ahci_channel *ch = device_get_softc(dev);
1184 bzero(ch->slot, sizeof(ch->slot));
1185 for (i = 0; i < ch->numslots; i++) {
1186 struct ahci_slot *slot = &ch->slot[i];
1188 slot->ch = ch;
1193 callout_init_mtx(&slot->timeout, &ch->mtx, 0);
1195 if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
1196 device_printf(ch->dev, "FAILURE - create data_map\n");
1203 struct ahci_channel *ch = device_get_softc(dev);
1207 for (i = 0; i < ch->numslots; i++) {
1208 struct ahci_slot *slot = &ch->slot[i];
1212 bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
1219 ahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr)
1222 if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
1223 ((ch->pm_level != 0 || ch->listening) && (serr & ATA_SE_EXCHANGED))) {
1224 u_int32_t status = ATA_INL(ch->r_mem, AHCI_P_SSTS);
1229 device_printf(ch->dev, "CONNECT requested\n");
1231 device_printf(ch->dev, "DISCONNECT requested\n");
1233 ahci_reset(ch);
1237 cam_sim_path(ch->sim),
1249 ahci_cpd_check_events(struct ahci_channel *ch)
1255 if (ch->pm_level == 0)
1258 status = ATA_INL(ch->r_mem, AHCI_P_CMD);
1263 dev = ch->dev;
1269 ahci_reset(ch);
1272 if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(ch->sim),
1281 ahci_notify_events(struct ahci_channel *ch, u_int32_t status)
1286 if (ch->caps & AHCI_CAP_SSNTF)
1287 ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status);
1289 device_printf(ch->dev, "SNTF 0x%04x\n", status);
1294 xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) {
1302 ahci_done(struct ahci_channel *ch, union ccb *ccb)
1305 mtx_assert(&ch->mtx, MA_OWNED);
1307 ch->batch == 0) {
1312 STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe);
1318 struct ahci_channel *ch = (struct ahci_channel *)arg;
1322 istatus = ATA_INL(ch->r_mem, AHCI_P_IS);
1324 mtx_lock(&ch->mtx);
1325 ahci_ch_intr_main(ch, istatus);
1326 mtx_unlock(&ch->mtx);
1332 struct ahci_channel *ch = (struct ahci_channel *)arg;
1338 istatus = ATA_INL(ch->r_mem, AHCI_P_IS);
1340 mtx_lock(&ch->mtx);
1341 ch->batch = 1;
1342 ahci_ch_intr_main(ch, istatus);
1343 ch->batch = 0;
1348 STAILQ_CONCAT(&tmp_doneq, &ch->doneq);
1349 mtx_unlock(&ch->mtx);
1359 struct ahci_channel *ch = (struct ahci_channel *)arg;
1362 if (ch->numrslots != 0)
1364 work = ATA_INL(ch->r_mem, AHCI_P_CMD);
1365 if (ch->pm_level == 4)
1369 ATA_OUTL(ch->r_mem, AHCI_P_CMD, work);
1373 ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
1380 ATA_OUTL(ch->r_mem, AHCI_P_IS, istatus);
1382 if (ch->numtslots != 0)
1383 cstatus = ATA_INL(ch->r_mem, AHCI_P_SACT);
1386 if (ch->numrslots != ch->numtslots)
1387 cstatus |= ATA_INL(ch->r_mem, AHCI_P_CI);
1390 (ch->pm_present || ch->curr[0].atapi != 0)) {
1391 if (ch->caps & AHCI_CAP_SSNTF)
1392 sntf = ATA_INL(ch->r_mem, AHCI_P_SNTF);
1393 else if (ch->fbs_enabled) {
1394 u_int8_t *fis = ch->dma.rfis + 0x58;
1404 u_int8_t *fis = ch->dma.rfis + 0x58;
1413 serr = ATA_INL(ch->r_mem, AHCI_P_SERR);
1415 ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr);
1416 reset = ahci_phy_check_events(ch, serr);
1421 ahci_cpd_check_events(ch);
1425 if (ch->quirks & AHCI_Q_NOCCS) {
1430 cstatus |= ch->rslots;
1435 ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) &
1439 // __func__, istatus, cstatus, sstatus, ch->rslots, ATA_INL(ch->r_mem, AHCI_P_TFD),
1440 // serr, ATA_INL(ch->r_mem, AHCI_P_FBS), ccs);
1442 if (ch->fbs_enabled) {
1443 uint32_t fbs = ATA_INL(ch->r_mem, AHCI_P_FBS);
1449 if (ch->numrslotspd[i] == 0)
1460 err = ch->rslots & cstatus;
1467 ok = ch->rslots & ~cstatus;
1468 for (i = 0; i < ch->numslots; i++) {
1470 ahci_end_transaction(&ch->slot[i], AHCI_ERR_NONE);
1474 if (ch->frozen) {
1475 union ccb *fccb = ch->frozen;
1476 ch->frozen = NULL;
1482 ahci_done(ch, fccb);
1484 for (i = 0; i < ch->numslots; i++) {
1489 ch->slot[i].ccb->ccb_h.target_id != port)
1494 if (ch->numtslotspd[
1495 ch->slot[i].ccb->ccb_h.target_id] == 0) {
1507 ch->fatalerr = 1;
1510 if (ch->numtslots == 0 && i != ccs && port != -2)
1516 ahci_end_transaction(&ch->slot[i], et);
1522 if (ch->rslots != 0 && !ch->recoverycmd)
1523 ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | AHCI_P_FBS_DEC);
1527 ahci_notify_events(ch, sntf);
1532 ahci_check_collision(struct ahci_channel *ch, union ccb *ccb)
1539 if (((~ch->oslots) & (0xffffffff >> (32 -
1540 ch->curr[t].tags))) == 0)
1543 if (ch->fbs_enabled) {
1545 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0)
1549 if (ch->numrslots != 0 && ch->numtslots == 0)
1552 if (ch->numtslots != 0 &&
1553 ch->taggedtarget != ccb->ccb_h.target_id)
1558 if (ch->fbs_enabled) {
1560 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0)
1564 if (ch->numrslots != 0 && ch->numtslots != 0)
1571 if (ch->numrslots != 0)
1575 if (ch->aslots != 0)
1582 ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb)
1588 tags = ch->numslots;
1591 tags = ch->curr[ccb->ccb_h.target_id].tags;
1592 if (ch->lastslot + 1 < tags)
1593 tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
1596 if (tag == 0 || tag + ch->lastslot >= tags)
1597 tag = ffs(~ch->oslots) - 1;
1599 tag += ch->lastslot;
1600 ch->lastslot = tag;
1602 slot = &ch->slot[tag];
1605 if (ch->numrslots == 0 && ch->pm_level > 3)
1606 callout_stop(&ch->pm_timer);
1608 ch->oslots |= (1 << tag);
1609 ch->numrslots++;
1610 ch->numrslotspd[ccb->ccb_h.target_id]++;
1613 ch->numtslots++;
1614 ch->numtslotspd[ccb->ccb_h.target_id]++;
1615 ch->taggedtarget = ccb->ccb_h.target_id;
1619 ch->aslots |= (1 << tag);
1622 bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
1635 struct ahci_channel *ch = slot->ch;
1641 device_printf(ch->dev, "DMA load error\n");
1647 ctp = (struct ahci_cmd_tab *)(ch->dma.work + slot->ct_offset);
1655 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1665 struct ahci_channel *ch = slot->ch;
1671 uint8_t *fis = ch->dma.rfis + 0x40;
1676 ctp = (struct ahci_cmd_tab *)(ch->dma.work + slot->ct_offset);
1678 if (!(fis_size = ahci_setup_fis(ch, ctp, ccb, slot->slot))) {
1679 device_printf(ch->dev, "Setting up SATA FIS failed\n");
1685 (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot));
1699 ahci_stop(ch);
1700 ahci_clo(ch);
1701 ahci_start(ch, 0);
1713 clp->cmd_table_phys = htole64(ch->dma.work_bus + slot->ct_offset);
1714 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1716 bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map,
1721 ATA_OUTL(ch->r_mem, AHCI_P_SACT, 1 << slot->slot);
1724 if (ch->fbs_enabled) {
1725 ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN |
1730 ch->rslots |= (1 << slot->slot);
1731 ATA_OUTL(ch->r_mem, AHCI_P_CI, (1 << slot->slot));
1740 if (!(ATA_INL(ch->r_mem, AHCI_P_CI) & (1 << slot->slot)))
1742 if ((ATA_INL(ch->r_mem, AHCI_P_TFD) & ATA_S_ERROR) &&
1745 device_printf(ch->dev,
1747 slot->slot, ATA_INL(ch->r_mem, AHCI_P_TFD));
1754 (ch->quirks & AHCI_Q_ATI_PMP_BUG) &&
1755 (ATA_INL(ch->r_mem, AHCI_P_IS) & AHCI_P_IX_IPM)) {
1766 if (ch->quirks & AHCI_Q_MRVL_SR_DEL)
1776 if ((ch->quirks & AHCI_Q_NOBSYRES) == 0 &&
1777 (ch->quirks & AHCI_Q_ATI_PMP_BUG) == 0 &&
1780 bus_dmamap_sync(ch->dma.rfis_tag,
1781 ch->dma.rfis_map, BUS_DMASYNC_POSTREAD);
1783 bus_dmamap_sync(ch->dma.rfis_tag,
1784 ch->dma.rfis_map, BUS_DMASYNC_PREREAD);
1792 device_printf(ch->dev, "Poll timeout on slot %d port %d\n",
1794 device_printf(ch->dev, "is %08x cs %08x ss %08x "
1796 ATA_INL(ch->r_mem, AHCI_P_IS),
1797 ATA_INL(ch->r_mem, AHCI_P_CI),
1798 ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots,
1799 ATA_INL(ch->r_mem, AHCI_P_TFD),
1800 ATA_INL(ch->r_mem, AHCI_P_SERR),
1801 ATA_INL(ch->r_mem, AHCI_P_CMD));
1807 ch->eslots |= (1 << slot->slot);
1819 ahci_process_timeout(struct ahci_channel *ch)
1823 mtx_assert(&ch->mtx, MA_OWNED);
1825 for (i = 0; i < ch->numslots; i++) {
1827 if (ch->slot[i].state < AHCI_SLOT_RUNNING)
1829 ahci_end_transaction(&ch->slot[i], AHCI_ERR_TIMEOUT);
1835 ahci_rearm_timeout(struct ahci_channel *ch)
1839 mtx_assert(&ch->mtx, MA_OWNED);
1840 for (i = 0; i < ch->numslots; i++) {
1841 struct ahci_slot *slot = &ch->slot[i];
1846 if ((ch->toslots & (1 << i)) == 0)
1859 struct ahci_channel *ch = slot->ch;
1860 device_t dev = ch->dev;
1872 sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT);
1873 ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK)
1876 ch->fbs_enabled || ch->wrongccs)
1878 else if ((ch->rslots & (1 << ccs)) == 0) {
1879 ch->wrongccs = 1;
1893 ATA_INL(ch->r_mem, AHCI_P_IS), ATA_INL(ch->r_mem, AHCI_P_CI),
1894 ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots,
1895 ATA_INL(ch->r_mem, AHCI_P_TFD), ATA_INL(ch->r_mem, AHCI_P_SERR),
1896 ATA_INL(ch->r_mem, AHCI_P_CMD));
1899 if (ch->frozen) {
1900 union ccb *fccb = ch->frozen;
1901 ch->frozen = NULL;
1907 ahci_done(ch, fccb);
1909 if (!ch->fbs_enabled && !ch->wrongccs) {
1911 ch->fatalerr = 1;
1913 ahci_end_transaction(&ch->slot[slot->slot], AHCI_ERR_TIMEOUT);
1915 for (i = 0; i < ch->numslots; i++) {
1917 if (ch->slot[i].state < AHCI_SLOT_RUNNING)
1919 ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT);
1923 if (ch->toslots == 0)
1924 xpt_freeze_simq(ch->sim, 1);
1925 ch->toslots |= (1 << slot->slot);
1926 if ((ch->rslots & ~ch->toslots) == 0)
1927 ahci_process_timeout(ch);
1930 ch->rslots & ~ch->toslots);
1938 struct ahci_channel *ch = slot->ch;
1944 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1947 (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot));
1957 u_int8_t *fis = ch->dma.rfis + 0x40;
1959 bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map,
1961 if (ch->fbs_enabled) {
1966 uint16_t tfd = ATA_INL(ch->r_mem, AHCI_P_TFD);
1985 if ((ch->quirks & AHCI_Q_ALTSIG) &&
1988 sig = ATA_INL(ch->r_mem, AHCI_P_SIG);
1998 (ch->quirks & AHCI_Q_NOCOUNT) == 0) {
2004 (ch->quirks & AHCI_Q_NOCOUNT) == 0) {
2010 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
2013 bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
2016 ch->eslots |= (1 << slot->slot);
2018 if ((et != AHCI_ERR_NONE) && (!ch->recoverycmd) &&
2032 ch->fatalerr = 1;
2048 ch->fatalerr = 1;
2049 if (!ch->recoverycmd) {
2050 xpt_freeze_simq(ch->sim, 1);
2057 if (!ch->recoverycmd) {
2058 xpt_freeze_simq(ch->sim, 1);
2065 ch->fatalerr = 1;
2069 ch->oslots &= ~(1 << slot->slot);
2070 ch->rslots &= ~(1 << slot->slot);
2071 ch->aslots &= ~(1 << slot->slot);
2075 ch->numrslots--;
2076 ch->numrslotspd[ccb->ccb_h.target_id]--;
2079 ch->numtslots--;
2080 ch->numtslotspd[ccb->ccb_h.target_id]--;
2084 lastto = (ch->toslots == (1 << slot->slot));
2085 ch->toslots &= ~(1 << slot->slot);
2087 xpt_release_simq(ch->sim, TRUE);
2096 ahci_begin_transaction(ch, ccb);
2101 ahci_process_read_log(ch, ccb);
2104 ahci_process_request_sense(ch, ccb);
2109 ch->hold[slot->slot] = ccb;
2110 ch->numhslots++;
2112 ahci_done(ch, ccb);
2114 if (ch->rslots == 0) {
2116 if (ch->toslots != 0 || ch->fatalerr) {
2117 ahci_reset(ch);
2120 if (ch->eslots != 0) {
2121 ahci_stop(ch);
2122 ahci_clo(ch);
2123 ahci_start(ch, 1);
2126 if (!ch->recoverycmd && ch->numhslots)
2127 ahci_issue_recovery(ch);
2130 } else if ((ch->rslots & ~ch->toslots) == 0 &&
2132 ahci_rearm_timeout(ch);
2134 if (ch->frozen && !ahci_check_collision(ch, ch->frozen)) {
2135 union ccb *fccb = ch->frozen;
2136 ch->frozen = NULL;
2137 ahci_begin_transaction(ch, fccb);
2138 xpt_release_simq(ch->sim, TRUE);
2141 if (ch->numrslots == 0 && ch->pm_level > 3 &&
2142 (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
2143 callout_schedule(&ch->pm_timer,
2144 (ch->pm_level == 4) ? hz / 1000 : hz / 8);
2149 ahci_issue_recovery(struct ahci_channel *ch)
2157 for (i = 0; i < ch->numslots; i++) {
2158 if (ch->hold[i])
2163 device_printf(ch->dev, "Unable to allocate recovery command\n");
2166 for (i = 0; i < ch->numslots; i++) {
2167 if (ch->hold[i] == NULL)
2169 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
2170 ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
2171 ahci_done(ch, ch->hold[i]);
2172 ch->hold[i] = NULL;
2173 ch->numhslots--;
2175 ahci_reset(ch);
2178 xpt_setup_ccb(&ccb->ccb_h, ch->hold[i]->ccb_h.path,
2179 ch->hold[i]->ccb_h.pinfo.priority);
2180 if (ch->hold[i]->ccb_h.func_code == XPT_ATA_IO) {
2190 device_printf(ch->dev,
2212 csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
2213 csio->dxfer_len = ch->hold[i]->csio.sense_len;
2220 ch->recoverycmd = 1;
2221 xpt_freeze_simq(ch->sim, 1);
2222 ahci_begin_transaction(ch, ccb);
2226 ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb)
2232 ch->recoverycmd = 0;
2237 for (i = 0; i < ch->numslots; i++) {
2238 if (!ch->hold[i])
2240 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
2243 res = &ch->hold[i]->ataio.res;
2256 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
2257 ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
2259 ahci_done(ch, ch->hold[i]);
2260 ch->hold[i] = NULL;
2261 ch->numhslots--;
2265 device_printf(ch->dev, "Error while READ LOG EXT\n");
2267 device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
2269 for (i = 0; i < ch->numslots; i++) {
2270 if (!ch->hold[i])
2272 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
2274 ahci_done(ch, ch->hold[i]);
2275 ch->hold[i] = NULL;
2276 ch->numhslots--;
2281 xpt_release_simq(ch->sim, TRUE);
2285 ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb)
2289 ch->recoverycmd = 0;
2293 ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
2295 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
2296 ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
2298 ahci_done(ch, ch->hold[i]);
2299 ch->hold[i] = NULL;
2300 ch->numhslots--;
2302 xpt_release_simq(ch->sim, TRUE);
2306 ahci_start(struct ahci_channel *ch, int fbs)
2311 if (ch->start)
2312 ch->start(ch);
2315 ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xFFFFFFFF);
2317 ATA_OUTL(ch->r_mem, AHCI_P_IS, 0xFFFFFFFF);
2319 if (ch->chcaps & AHCI_P_CMD_FBSCP) {
2320 ch->fbs_enabled = (fbs && ch->pm_present) ? 1 : 0;
2321 ATA_OUTL(ch->r_mem, AHCI_P_FBS,
2322 ch->fbs_enabled ? AHCI_P_FBS_EN : 0);
2325 cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2327 ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_ST |
2328 (ch->pm_present ? AHCI_P_CMD_PMA : 0));
2332 ahci_stop(struct ahci_channel *ch)
2338 cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2339 ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_ST);
2345 device_printf(ch->dev, "stopping AHCI engine failed\n");
2348 } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR);
2349 ch->eslots = 0;
2353 ahci_clo(struct ahci_channel *ch)
2359 if (ch->caps & AHCI_CAP_SCLO) {
2360 cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2362 ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd);
2367 device_printf(ch->dev, "executing CLO failed\n");
2370 } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO);
2375 ahci_stop_fr(struct ahci_channel *ch)
2381 cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2382 ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_FRE);
2388 device_printf(ch->dev, "stopping AHCI FR engine failed\n");
2391 } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR);
2395 ahci_start_fr(struct ahci_channel *ch)
2400 cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2401 ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_FRE);
2405 ahci_wait_ready(struct ahci_channel *ch, int t, int t0)
2410 while ((val = ATA_INL(ch->r_mem, AHCI_P_TFD)) &
2414 device_printf(ch->dev,
2425 device_printf(ch->dev, "AHCI reset: device ready after %dms\n",
2433 struct ahci_channel *ch = arg;
2435 if (ch->resetting == 0)
2437 ch->resetting--;
2438 if (ahci_wait_ready(ch, ch->resetting == 0 ? -1 : 0,
2439 (310 - ch->resetting) * 100) == 0) {
2440 ch->resetting = 0;
2441 ahci_start(ch, 1);
2442 xpt_release_simq(ch->sim, TRUE);
2445 if (ch->resetting == 0) {
2446 ahci_clo(ch);
2447 ahci_start(ch, 1);
2448 xpt_release_simq(ch->sim, TRUE);
2451 callout_schedule(&ch->reset_timer, hz / 10);
2455 ahci_reset(struct ahci_channel *ch)
2457 struct ahci_controller *ctlr = device_get_softc(device_get_parent(ch->dev));
2460 xpt_freeze_simq(ch->sim, 1);
2462 device_printf(ch->dev, "AHCI reset...\n");
2464 if (ch->resetting) {
2465 ch->resetting = 0;
2466 callout_stop(&ch->reset_timer);
2467 xpt_release_simq(ch->sim, TRUE);
2470 if (ch->frozen) {
2471 union ccb *fccb = ch->frozen;
2472 ch->frozen = NULL;
2478 ahci_done(ch, fccb);
2481 ahci_stop(ch);
2482 for (i = 0; i < ch->numslots; i++) {
2484 if (ch->slot[i].state < AHCI_SLOT_RUNNING)
2487 ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT);
2489 for (i = 0; i < ch->numslots; i++) {
2490 if (!ch->hold[i])
2492 ahci_done(ch, ch->hold[i]);
2493 ch->hold[i] = NULL;
2494 ch->numhslots--;
2496 if (ch->toslots != 0)
2497 xpt_release_simq(ch->sim, TRUE);
2498 ch->eslots = 0;
2499 ch->toslots = 0;
2500 ch->wrongccs = 0;
2501 ch->fatalerr = 0;
2503 xpt_async(AC_BUS_RESET, ch->path, NULL);
2505 ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
2507 if (!ahci_sata_phy_reset(ch)) {
2509 device_printf(ch->dev,
2511 ch->devices = 0;
2513 ATA_OUTL(ch->r_mem, AHCI_P_IE,
2514 (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) |
2516 xpt_release_simq(ch->sim, TRUE);
2520 device_printf(ch->dev, "AHCI reset: device found\n");
2522 if (ahci_wait_ready(ch, dumping ? 31000 : 0, 0)) {
2524 ahci_clo(ch);
2526 ch->resetting = 310;
2528 ch->devices = 1;
2530 ATA_OUTL(ch->r_mem, AHCI_P_IE,
2531 (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) |
2534 ((ch->pm_level == 0) ? AHCI_P_IX_PRC : 0) | AHCI_P_IX_PC |
2537 if (ch->resetting)
2538 callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, ch);
2540 ahci_start(ch, 1);
2541 xpt_release_simq(ch->sim, TRUE);
2546 ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
2557 ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA)
2603 ahci_sata_connect(struct ahci_channel *ch)
2612 timeoutslot = ((ch->quirks & AHCI_Q_SLOWDEV) ? 5000 : 1000);
2614 status = ATA_INL(ch->r_mem, AHCI_P_SSTS);
2623 device_printf(ch->dev, "SATA offline status=%08x\n",
2634 device_printf(ch->dev,
2641 device_printf(ch->dev, "SATA connect time=%dus status=%08x\n",
2645 ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xffffffff);
2650 ahci_sata_phy_reset(struct ahci_channel *ch)
2655 if (ch->listening) {
2656 val = ATA_INL(ch->r_mem, AHCI_P_CMD);
2658 ATA_OUTL(ch->r_mem, AHCI_P_CMD, val);
2659 ch->listening = 0;
2661 sata_rev = ch->user[ch->pm_present ? 15 : 0].revision;
2670 detval = ahci_ch_detval(ch, ATA_SC_DET_RESET);
2671 ATA_OUTL(ch->r_mem, AHCI_P_SCTL,
2675 detval = ahci_ch_detval(ch, ATA_SC_DET_IDLE);
2676 ATA_OUTL(ch->r_mem, AHCI_P_SCTL,
2677 detval | val | ((ch->pm_level > 0) ? 0 :
2679 if (!ahci_sata_connect(ch)) {
2680 if (ch->caps & AHCI_CAP_SSS) {
2681 val = ATA_INL(ch->r_mem, AHCI_P_CMD);
2683 ATA_OUTL(ch->r_mem, AHCI_P_CMD, val);
2684 ch->listening = 1;
2685 } else if (ch->pm_level > 0)
2686 ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE);
2693 ahci_check_ids(struct ahci_channel *ch, union ccb *ccb)
2696 if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) {
2698 ahci_done(ch, ccb);
2703 ahci_done(ch, ccb);
2712 struct ahci_channel *ch;
2717 ch = (struct ahci_channel *)cam_sim_softc(sim);
2722 if (ahci_check_ids(ch, ccb))
2724 if (ch->devices == 0 ||
2725 (ch->pm_present == 0 &&
2732 if (ahci_check_collision(ch, ccb)) {
2734 ch->frozen = ccb;
2736 xpt_freeze_simq(ch->sim, 1);
2739 ahci_begin_transaction(ch, ccb);
2750 if (ahci_check_ids(ch, ccb))
2753 d = &ch->curr[ccb->ccb_h.target_id];
2755 d = &ch->user[ccb->ccb_h.target_id];
2763 d->tags = min(ch->numslots, cts->xport_specific.sata.tags);
2765 ch->pm_present = cts->xport_specific.sata.pm_present;
2780 if (ahci_check_ids(ch, ccb))
2783 d = &ch->curr[ccb->ccb_h.target_id];
2785 d = &ch->user[ccb->ccb_h.target_id];
2794 (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
2795 status = ATA_INL(ch->r_mem, AHCI_P_SSTS) & ATA_SS_SPD_MASK;
2803 if (ch->pm_level) {
2804 if (ch->caps & (AHCI_CAP_PSC | AHCI_CAP_SSC))
2806 if (ch->caps2 & AHCI_CAP2_APST)
2809 if ((ch->caps & AHCI_CAP_SNCQ) &&
2810 (ch->quirks & AHCI_Q_NOAA) == 0)
2814 ch->user[ccb->ccb_h.target_id].caps;
2826 cts->xport_specific.sata.pm_present = ch->pm_present;
2837 ahci_reset(ch);
2850 if (ch->caps & AHCI_CAP_SNCQ)
2852 if (ch->caps & AHCI_CAP_SPM)
2856 if ((ch->quirks & AHCI_Q_NOAUX) == 0)
2859 if (ch->caps & AHCI_CAP_SPM)
2877 if (ch->quirks & AHCI_Q_MAXIO_64K)
2879 cpi->hba_vendor = ch->vendorid;
2880 cpi->hba_device = ch->deviceid;
2881 cpi->hba_subvendor = ch->subvendorid;
2882 cpi->hba_subdevice = ch->subdeviceid;
2890 ahci_done(ch, ccb);
2896 struct ahci_channel *ch = (struct ahci_channel *)cam_sim_softc(sim);
2900 istatus = ATA_INL(ch->r_mem, AHCI_P_IS);
2902 ahci_ch_intr_main(ch, istatus);
2903 if (ch->resetting != 0 &&
2904 (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) {
2905 ch->resetpolldiv = 1000;
2906 ahci_reset_to(ch);