1 /* $OpenBSD: ahci.c,v 1.16 2014/07/13 23:10:23 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/buf.h> 24 #include <sys/kernel.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/queue.h> 28 #include <sys/mutex.h> 29 #include <sys/pool.h> 30 31 #include <machine/bus.h> 32 33 #include <dev/ic/ahcireg.h> 34 #include <dev/ic/ahcivar.h> 35 36 #ifdef AHCI_DEBUG 37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 38 while (0) 39 #define AHCI_D_TIMEOUT 0x00 40 #define AHCI_D_VERBOSE 0x01 41 #define AHCI_D_INTR 0x02 42 #define AHCI_D_XFER 0x08 43 int ahcidebug = AHCI_D_VERBOSE; 44 #else 45 #define DPRINTF(m, f...) 46 #endif 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disk.h> 52 #include <sys/disklabel.h> 53 54 #include <scsi/scsi_all.h> 55 #include <scsi/scsiconf.h> 56 57 void ahci_hibernate_io_start(struct ahci_port *, 58 struct ahci_ccb *); 59 int ahci_hibernate_io_poll(struct ahci_port *, 60 struct ahci_ccb *); 61 void ahci_hibernate_load_prdt(struct ahci_ccb *); 62 63 int ahci_hibernate_io(dev_t dev, daddr_t blkno, 64 vaddr_t addr, size_t size, int wr, void *page); 65 #endif 66 67 struct cfdriver ahci_cd = { 68 NULL, "ahci", DV_DULL 69 }; 70 71 void ahci_enable_interrupts(struct ahci_port *); 72 73 int ahci_init(struct ahci_softc *); 74 int ahci_port_alloc(struct ahci_softc *, u_int); 75 void ahci_port_free(struct ahci_softc *, u_int); 76 int ahci_port_init(struct ahci_softc *, u_int); 77 78 int ahci_port_start(struct ahci_port *, int); 79 int ahci_port_stop(struct ahci_port *, int); 80 int ahci_port_clo(struct ahci_port *); 81 int ahci_port_softreset(struct ahci_port *); 82 int ahci_port_portreset(struct ahci_port *, int); 83 int ahci_port_signature(struct ahci_port *); 84 int ahci_pmp_port_softreset(struct ahci_port *, int); 85 int ahci_pmp_port_portreset(struct ahci_port *, int); 86 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 87 88 int ahci_load_prdt(struct ahci_ccb *); 89 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 90 u_int32_t, u_int32_t); 91 void ahci_unload_prdt(struct ahci_ccb *); 92 93 int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 94 void ahci_start(struct ahci_ccb *); 95 96 void ahci_issue_pending_ncq_commands(struct ahci_port *); 97 void ahci_issue_pending_commands(struct ahci_port *, int); 98 99 int ahci_intr(void *); 100 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 101 102 struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 103 void ahci_put_ccb(struct ahci_ccb *); 104 105 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 106 void ahci_put_err_ccb(struct ahci_ccb *); 107 108 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 109 void ahci_put_pmp_ccb(struct ahci_ccb *); 110 111 int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 112 113 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 114 void ahci_dmamem_free(struct ahci_softc *, 115 struct ahci_dmamem *); 116 117 u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 118 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 119 int ahci_wait_ne(struct ahci_softc *, bus_size_t, 120 u_int32_t, u_int32_t); 121 122 u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 123 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 124 int ahci_pwait_eq(struct ahci_port *, bus_size_t, 125 u_int32_t, u_int32_t, int); 126 void ahci_flush_tfd(struct ahci_port *ap); 127 u_int32_t ahci_active_mask(struct ahci_port *); 128 int ahci_port_detect_pmp(struct ahci_port *); 129 void ahci_pmp_probe_timeout(void *); 130 131 /* pmp operations */ 132 int ahci_pmp_read(struct ahci_port *, int, int, 133 u_int32_t *); 134 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 135 int ahci_pmp_phy_status(struct ahci_port *, int, 136 u_int32_t *); 137 int ahci_pmp_identify(struct ahci_port *, int *); 138 139 140 /* Wait for all bits in _b to be cleared */ 141 #define ahci_pwait_clr(_ap, _r, _b, _n) \ 142 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 143 144 /* Wait for all bits in _b to be set */ 145 #define ahci_pwait_set(_ap, _r, _b, _n) \ 146 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 147 148 149 150 /* provide methods for atascsi to call */ 151 int ahci_ata_probe(void *, int, int); 152 void ahci_ata_free(void *, int, int); 153 struct ata_xfer * ahci_ata_get_xfer(void *, int); 154 void ahci_ata_put_xfer(struct ata_xfer *); 155 void ahci_ata_cmd(struct ata_xfer *); 156 157 struct atascsi_methods ahci_atascsi_methods = { 158 ahci_ata_probe, 159 ahci_ata_free, 160 ahci_ata_get_xfer, 161 ahci_ata_put_xfer, 162 ahci_ata_cmd 163 }; 164 165 /* ccb completions */ 166 void ahci_ata_cmd_done(struct ahci_ccb *); 167 void ahci_pmp_cmd_done(struct ahci_ccb *); 168 void ahci_ata_cmd_timeout(void *); 169 void ahci_empty_done(struct ahci_ccb *); 170 171 int 172 ahci_attach(struct ahci_softc *sc) 173 { 174 struct atascsi_attach_args aaa; 175 u_int32_t pi; 176 int i; 177 178 if (ahci_init(sc) != 0) { 179 /* error already printed by ahci_init */ 180 goto unmap; 181 } 182 183 printf("\n"); 184 185 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 186 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 187 #ifdef AHCI_DEBUG 188 if (ahcidebug & AHCI_D_VERBOSE) { 189 const char *gen; 190 191 switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 192 case AHCI_REG_CAP_ISS_G1: 193 gen = "1 (1.5Gbps)"; 194 break; 195 case AHCI_REG_CAP_ISS_G2: 196 gen = "2 (3Gbps)"; 197 break; 198 case AHCI_REG_CAP_ISS_G3: 199 gen = "3 (6Gbps)"; 200 break; 201 default: 202 gen = "unknown"; 203 break; 204 } 205 206 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 207 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 208 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 209 } 210 #endif 211 212 pi = ahci_read(sc, AHCI_REG_PI); 213 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 214 DEVNAME(sc), pi); 215 216 #ifdef AHCI_COALESCE 217 /* Naive coalescing support - enable for all ports. */ 218 if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 219 u_int16_t ccc_timeout = 20; 220 u_int8_t ccc_numcomplete = 12; 221 u_int32_t ccc_ctl; 222 223 /* disable coalescing during reconfiguration. */ 224 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 225 ccc_ctl &= ~0x00000001; 226 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 227 228 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 229 if (pi & sc->sc_ccc_mask) { 230 /* A conflict with the implemented port list? */ 231 printf("%s: coalescing interrupt/implemented port list " 232 "conflict, PI: %08x, ccc_mask: %08x\n", 233 DEVNAME(sc), pi, sc->sc_ccc_mask); 234 sc->sc_ccc_mask = 0; 235 goto noccc; 236 } 237 238 /* ahci_port_start will enable each port when it starts. */ 239 sc->sc_ccc_ports = pi; 240 sc->sc_ccc_ports_cur = 0; 241 242 /* program thresholds and enable overall coalescing. */ 243 ccc_ctl &= ~0xffffff00; 244 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 245 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 246 ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 247 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 248 } 249 noccc: 250 #endif 251 for (i = 0; i < AHCI_MAX_PORTS; i++) { 252 if (!ISSET(pi, 1 << i)) { 253 /* dont allocate stuff if the port isnt implemented */ 254 continue; 255 } 256 257 if (ahci_port_alloc(sc, i) == ENOMEM) 258 goto freeports; 259 } 260 261 memset(&aaa, 0, sizeof(aaa)); 262 aaa.aaa_cookie = sc; 263 aaa.aaa_methods = &ahci_atascsi_methods; 264 aaa.aaa_minphys = NULL; 265 aaa.aaa_nports = AHCI_MAX_PORTS; 266 aaa.aaa_ncmds = sc->sc_ncmds - 1; 267 if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 268 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 269 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 270 /* XXX enabling ASAA_CAP_PMP_NCQ with FBS: 271 * - some error recovery work required (single device vs port 272 * errors) 273 * - probably need to look at storing our active ccb queue 274 * differently so we can group ncq and non-ncq commands 275 * for different ports. as long as we preserve the order for 276 * each port, we can reorder commands to get more ncq 277 * commands to run in parallel. 278 */ 279 } 280 281 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 282 283 /* Enable interrupts */ 284 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 285 286 return 0; 287 288 freeports: 289 for (i = 0; i < AHCI_MAX_PORTS; i++) { 290 if (sc->sc_ports[i] != NULL) 291 ahci_port_free(sc, i); 292 } 293 unmap: 294 /* Disable controller */ 295 ahci_write(sc, AHCI_REG_GHC, 0); 296 return 1; 297 } 298 299 int 300 ahci_detach(struct ahci_softc *sc, int flags) 301 { 302 int rv, i; 303 304 if (sc->sc_atascsi != NULL) { 305 rv = atascsi_detach(sc->sc_atascsi, flags); 306 if (rv != 0) 307 return (rv); 308 } 309 310 for (i = 0; i < AHCI_MAX_PORTS; i++) { 311 if (sc->sc_ports[i] != NULL) 312 ahci_port_free(sc, i); 313 } 314 315 return (0); 316 } 317 318 int 319 ahci_activate(struct device *self, int act) 320 { 321 struct ahci_softc *sc = (struct ahci_softc *)self; 322 int i, rv = 0; 323 324 switch (act) { 325 case DVACT_RESUME: 326 /* enable ahci (global interrupts disabled) */ 327 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 328 329 /* restore BIOS initialised parameters */ 330 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 331 332 for (i = 0; i < AHCI_MAX_PORTS; i++) { 333 if (sc->sc_ports[i] != NULL) 334 ahci_port_init(sc, i); 335 } 336 337 /* Enable interrupts */ 338 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 339 340 rv = config_activate_children(self, act); 341 break; 342 case DVACT_POWERDOWN: 343 rv = config_activate_children(self, act); 344 for (i = 0; i < AHCI_MAX_PORTS; i++) { 345 if (sc->sc_ports[i] != NULL) 346 ahci_port_stop(sc->sc_ports[i], 1); 347 } 348 break; 349 default: 350 rv = config_activate_children(self, act); 351 break; 352 } 353 return (rv); 354 } 355 356 int 357 ahci_init(struct ahci_softc *sc) 358 { 359 u_int32_t reg, cap, pi; 360 const char *revision; 361 362 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 363 AHCI_FMT_GHC); 364 365 /* save BIOS initialised parameters, enable staggered spin up */ 366 cap = ahci_read(sc, AHCI_REG_CAP); 367 cap &= AHCI_REG_CAP_SMPS; 368 cap |= AHCI_REG_CAP_SSS; 369 pi = ahci_read(sc, AHCI_REG_PI); 370 371 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 372 /* reset the controller */ 373 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 374 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 375 AHCI_REG_GHC_HR) != 0) { 376 printf(" unable to reset controller\n"); 377 return (1); 378 } 379 } 380 381 /* enable ahci (global interrupts disabled) */ 382 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 383 384 /* restore parameters */ 385 ahci_write(sc, AHCI_REG_CAP, cap); 386 ahci_write(sc, AHCI_REG_PI, pi); 387 388 /* check the revision */ 389 reg = ahci_read(sc, AHCI_REG_VS); 390 switch (reg) { 391 case AHCI_REG_VS_0_95: 392 revision = "0.95"; 393 break; 394 case AHCI_REG_VS_1_0: 395 revision = "1.0"; 396 break; 397 case AHCI_REG_VS_1_1: 398 revision = "1.1"; 399 break; 400 case AHCI_REG_VS_1_2: 401 revision = "1.2"; 402 break; 403 case AHCI_REG_VS_1_3: 404 revision = "1.3"; 405 break; 406 407 default: 408 printf(" unsupported AHCI revision 0x%08x\n", reg); 409 return (1); 410 } 411 412 printf(" AHCI %s", revision); 413 414 return (0); 415 } 416 417 void 418 ahci_enable_interrupts(struct ahci_port *ap) 419 { 420 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 421 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 422 AHCI_PREG_IE_UFE | 423 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 424 #ifdef AHCI_COALESCE 425 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 426 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 427 #else 428 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 429 #endif 430 ); 431 } 432 433 int 434 ahci_port_alloc(struct ahci_softc *sc, u_int port) 435 { 436 struct ahci_port *ap; 437 struct ahci_ccb *ccb; 438 u_int64_t dva; 439 u_int32_t cmd; 440 struct ahci_cmd_hdr *hdr; 441 struct ahci_cmd_table *table; 442 int i, rc = ENOMEM; 443 444 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 445 if (ap == NULL) { 446 printf("%s: unable to allocate memory for port %d\n", 447 DEVNAME(sc), port); 448 goto reterr; 449 } 450 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 451 if (ap->ap_err_scratch == NULL) { 452 printf("%s: unable to allocate DMA scratch buf for port %d\n", 453 DEVNAME(sc), port); 454 goto freeport; 455 } 456 457 #ifdef AHCI_DEBUG 458 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 459 DEVNAME(sc), port); 460 #endif 461 ap->ap_port = port; 462 sc->sc_ports[port] = ap; 463 464 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 465 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 466 printf("%s: unable to create register window for port %d\n", 467 DEVNAME(sc), port); 468 goto freeport; 469 } 470 471 ap->ap_sc = sc; 472 #ifdef AHCI_COALESCE 473 ap->ap_num = port; 474 #endif 475 TAILQ_INIT(&ap->ap_ccb_free); 476 TAILQ_INIT(&ap->ap_ccb_pending); 477 mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 478 479 /* Disable port interrupts */ 480 ahci_pwrite(ap, AHCI_PREG_IE, 0); 481 482 /* Sec 10.1.2 - deinitialise port if it is already running */ 483 cmd = ahci_pread(ap, AHCI_PREG_CMD); 484 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 485 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 486 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 487 int r; 488 489 r = ahci_port_stop(ap, 1); 490 if (r) { 491 printf("%s: unable to disable %s, ignoring port %d\n", 492 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 493 rc = ENXIO; 494 goto freeport; 495 } 496 497 /* Write DET to zero */ 498 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 499 } 500 501 /* XXX FBS - need to allocate 16x ahci_rfis struct? - but we don't 502 * know if there's a PMP attached or if the HBA supports FBS yet.. 503 * reallocate when we enable FBS? 504 */ 505 506 /* Allocate RFIS */ 507 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 508 if (ap->ap_dmamem_rfis == NULL) 509 goto nomem; 510 511 /* Setup RFIS base address */ 512 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 513 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 514 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 515 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 516 517 /* Enable FIS reception and activate port. */ 518 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 519 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 520 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 521 522 /* Check whether port activated. Skip it if not. */ 523 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 524 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 525 rc = ENXIO; 526 goto freeport; 527 } 528 529 /* Allocate a CCB for each command slot */ 530 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 531 M_DEVBUF, M_NOWAIT | M_ZERO); 532 if (ap->ap_ccbs == NULL) { 533 printf("%s: unable to allocate command list for port %d\n", 534 DEVNAME(sc), port); 535 goto freeport; 536 } 537 538 /* Command List Structures and Command Tables */ 539 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 540 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 541 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 542 sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 543 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 544 nomem: 545 printf("%s: unable to allocate DMA memory for port %d\n", 546 DEVNAME(sc), port); 547 goto freeport; 548 } 549 550 /* Setup command list base address */ 551 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 552 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 553 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 554 555 /* Split CCB allocation into CCBs and assign to command header/table */ 556 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 557 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 558 for (i = 0; i < sc->sc_ncmds; i++) { 559 ccb = &ap->ap_ccbs[i]; 560 561 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 562 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 563 &ccb->ccb_dmamap) != 0) { 564 printf("%s: unable to create dmamap for port %d " 565 "ccb %d\n", DEVNAME(sc), port, i); 566 goto freeport; 567 } 568 569 ccb->ccb_slot = i; 570 ccb->ccb_port = ap; 571 ccb->ccb_cmd_hdr = &hdr[i]; 572 ccb->ccb_cmd_table = &table[i]; 573 htolem64(&ccb->ccb_cmd_hdr->ctba, 574 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 575 ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 576 577 ccb->ccb_xa.fis = 578 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 579 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 580 ccb->ccb_xa.tag = i; 581 582 ccb->ccb_xa.state = ATA_S_COMPLETE; 583 ahci_put_ccb(ccb); 584 } 585 586 /* grab a ccb for use during error recovery */ 587 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 588 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 589 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 590 591 /* Wait for ICC change to complete */ 592 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 593 594 /* Reset port */ 595 rc = ahci_port_portreset(ap, 1); 596 597 switch (rc) { 598 case ENODEV: 599 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 600 case AHCI_PREG_SSTS_DET_DEV_NE: 601 printf("%s: device not communicating on port %d\n", 602 DEVNAME(sc), port); 603 break; 604 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 605 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 606 port); 607 break; 608 default: 609 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 610 "on port %d\n", DEVNAME(sc), port); 611 break; 612 } 613 goto freeport; 614 615 case EBUSY: 616 printf("%s: device on port %d didn't come ready, " 617 "TFD: 0x%b\n", DEVNAME(sc), port, 618 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 619 620 /* Try a soft reset to clear busy */ 621 rc = ahci_port_softreset(ap); 622 if (rc) { 623 printf("%s: unable to communicate " 624 "with device on port %d\n", DEVNAME(sc), port); 625 goto freeport; 626 } 627 break; 628 629 default: 630 break; 631 } 632 633 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 634 DEVNAME(sc), port, rc); 635 636 /* Enable command transfers on port */ 637 if (ahci_port_start(ap, 0)) { 638 printf("%s: failed to start command DMA on port %d, " 639 "disabling\n", DEVNAME(sc), port); 640 rc = ENXIO; /* couldn't start port */ 641 } 642 643 /* Flush interrupts for port */ 644 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 645 ahci_write(sc, AHCI_REG_IS, 1 << port); 646 647 ahci_enable_interrupts(ap); 648 649 freeport: 650 if (rc != 0) 651 ahci_port_free(sc, port); 652 reterr: 653 return (rc); 654 } 655 656 void 657 ahci_port_free(struct ahci_softc *sc, u_int port) 658 { 659 struct ahci_port *ap = sc->sc_ports[port]; 660 struct ahci_ccb *ccb; 661 662 /* Ensure port is disabled and its interrupts are flushed */ 663 if (ap->ap_sc) { 664 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 665 ahci_pwrite(ap, AHCI_PREG_IE, 0); 666 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 667 ahci_write(sc, AHCI_REG_IS, 1 << port); 668 } 669 670 if (ap->ap_ccb_err) 671 ahci_put_ccb(ap->ap_ccb_err); 672 673 if (ap->ap_ccbs) { 674 while ((ccb = ahci_get_ccb(ap)) != NULL) 675 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 676 free(ap->ap_ccbs, M_DEVBUF, 0); 677 } 678 679 if (ap->ap_dmamem_cmd_list) 680 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 681 if (ap->ap_dmamem_rfis) 682 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 683 if (ap->ap_dmamem_cmd_table) 684 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 685 if (ap->ap_err_scratch) 686 dma_free(ap->ap_err_scratch, DEV_BSIZE); 687 688 /* bus_space(9) says we dont free the subregions handle */ 689 690 free(ap, M_DEVBUF, 0); 691 sc->sc_ports[port] = NULL; 692 } 693 694 int 695 ahci_port_init(struct ahci_softc *sc, u_int port) 696 { 697 struct ahci_port *ap; 698 u_int64_t dva; 699 u_int32_t cmd; 700 int rc = ENOMEM; 701 702 ap = sc->sc_ports[port]; 703 #ifdef AHCI_DEBUG 704 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 705 DEVNAME(sc), port); 706 #endif 707 708 /* Disable port interrupts */ 709 ahci_pwrite(ap, AHCI_PREG_IE, 0); 710 711 /* Sec 10.1.2 - deinitialise port if it is already running */ 712 cmd = ahci_pread(ap, AHCI_PREG_CMD); 713 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 714 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 715 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 716 int r; 717 718 r = ahci_port_stop(ap, 1); 719 if (r) { 720 printf("%s: unable to disable %s, ignoring port %d\n", 721 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 722 rc = ENXIO; 723 goto reterr; 724 } 725 726 /* Write DET to zero */ 727 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 728 } 729 730 /* Setup RFIS base address */ 731 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 732 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 733 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 734 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 735 736 /* Enable FIS reception and activate port. */ 737 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 738 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 739 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 740 741 /* Check whether port activated. Skip it if not. */ 742 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 743 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 744 rc = ENXIO; 745 goto reterr; 746 } 747 748 /* Setup command list base address */ 749 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 750 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 751 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 752 753 /* Wait for ICC change to complete */ 754 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 755 756 /* Reset port */ 757 rc = ahci_port_portreset(ap, 1); 758 switch (rc) { 759 case ENODEV: 760 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 761 case AHCI_PREG_SSTS_DET_DEV_NE: 762 printf("%s: device not communicating on port %d\n", 763 DEVNAME(sc), port); 764 break; 765 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 766 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 767 port); 768 break; 769 default: 770 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 771 "on port %d\n", DEVNAME(sc), port); 772 break; 773 } 774 goto reterr; 775 776 case EBUSY: 777 printf("%s: device on port %d didn't come ready, " 778 "TFD: 0x%b\n", DEVNAME(sc), port, 779 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 780 781 /* Try a soft reset to clear busy */ 782 rc = ahci_port_softreset(ap); 783 if (rc) { 784 printf("%s: unable to communicate " 785 "with device on port %d\n", DEVNAME(sc), port); 786 goto reterr; 787 } 788 break; 789 790 default: 791 break; 792 } 793 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 794 DEVNAME(sc), port); 795 796 if (ap->ap_pmp_ports > 0) { 797 int p; 798 799 for (p = 0; p < ap->ap_pmp_ports; p++) { 800 int sig; 801 802 /* might need to do a portreset first here? */ 803 804 /* softreset the port */ 805 if (ahci_pmp_port_softreset(ap, p)) { 806 printf("%s.%d: unable to probe PMP port due to" 807 " softreset failure\n", PORTNAME(ap), p); 808 continue; 809 } 810 811 sig = ahci_port_signature(ap); 812 printf("%s.%d: port signature returned %d\n", 813 PORTNAME(ap), p, sig); 814 } 815 } 816 817 /* Enable command transfers on port */ 818 if (ahci_port_start(ap, 0)) { 819 printf("%s: failed to start command DMA on port %d, " 820 "disabling\n", DEVNAME(sc), port); 821 rc = ENXIO; /* couldn't start port */ 822 } 823 824 /* Flush interrupts for port */ 825 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 826 ahci_write(sc, AHCI_REG_IS, 1 << port); 827 828 ahci_enable_interrupts(ap); 829 830 reterr: 831 return (rc); 832 } 833 834 int 835 ahci_port_start(struct ahci_port *ap, int fre_only) 836 { 837 u_int32_t r; 838 839 /* XXX FBS: possibly turn FBS on here */ 840 841 /* Turn on FRE (and ST) */ 842 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 843 r |= AHCI_PREG_CMD_FRE; 844 if (!fre_only) 845 r |= AHCI_PREG_CMD_ST; 846 ahci_pwrite(ap, AHCI_PREG_CMD, r); 847 848 #ifdef AHCI_COALESCE 849 /* (Re-)enable coalescing on the port. */ 850 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 851 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 852 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 853 ap->ap_sc->sc_ccc_ports_cur); 854 } 855 #endif 856 857 /* Wait for CR to come on */ 858 if (!fre_only && 859 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 860 return (1); 861 862 return (0); 863 } 864 865 int 866 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 867 { 868 u_int32_t r; 869 870 #ifdef AHCI_COALESCE 871 /* Disable coalescing on the port while it is stopped. */ 872 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 873 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 874 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 875 ap->ap_sc->sc_ccc_ports_cur); 876 } 877 #endif 878 879 /* Turn off ST (and FRE) */ 880 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 881 r &= ~AHCI_PREG_CMD_ST; 882 if (stop_fis_rx) 883 r &= ~AHCI_PREG_CMD_FRE; 884 ahci_pwrite(ap, AHCI_PREG_CMD, r); 885 886 /* Wait for CR to go off */ 887 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 888 return (1); 889 890 /* Wait for FR to go off */ 891 if (stop_fis_rx && 892 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 893 return (2); 894 895 /* XXX FBS: possibly disable FBS here? */ 896 897 return (0); 898 } 899 900 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 901 int 902 ahci_port_clo(struct ahci_port *ap) 903 { 904 struct ahci_softc *sc = ap->ap_sc; 905 u_int32_t cmd; 906 907 /* Only attempt CLO if supported by controller */ 908 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 909 return (1); 910 911 /* Issue CLO */ 912 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 913 #ifdef DIAGNOSTIC 914 if (ISSET(cmd, AHCI_PREG_CMD_ST)) 915 printf("%s: CLO requested while port running\n", PORTNAME(ap)); 916 #endif 917 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 918 919 /* Wait for completion */ 920 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 921 printf("%s: CLO did not complete\n", PORTNAME(ap)); 922 return (1); 923 } 924 925 return (0); 926 } 927 928 /* AHCI soft reset, Section 10.4.1 */ 929 int 930 ahci_port_softreset(struct ahci_port *ap) 931 { 932 struct ahci_ccb *ccb = NULL; 933 struct ahci_cmd_hdr *cmd_slot; 934 u_int8_t *fis; 935 int s, rc = EIO, oldstate; 936 u_int32_t cmd; 937 938 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 939 940 s = splbio(); 941 oldstate = ap->ap_state; 942 ap->ap_state = AP_S_ERROR_RECOVERY; 943 944 /* Save previous command register state */ 945 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 946 947 /* Idle port */ 948 if (ahci_port_stop(ap, 0)) { 949 printf("%s: failed to stop port, cannot softreset\n", 950 PORTNAME(ap)); 951 goto err; 952 } 953 954 /* Request CLO if device appears hung */ 955 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 956 AHCI_PREG_TFD_STS_DRQ)) 957 ahci_port_clo(ap); 958 959 /* Clear port errors to permit TFD transfer */ 960 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 961 962 /* XXX FBS - need to ensure we don't enable FBS here, since we're 963 * resetting stuff 964 * (AHCI spec 9.3.8) 965 */ 966 /* Restart port */ 967 if (ahci_port_start(ap, 0)) { 968 printf("%s: failed to start port, cannot softreset\n", 969 PORTNAME(ap)); 970 goto err; 971 } 972 973 /* Check whether CLO worked */ 974 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 975 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 976 printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 977 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 978 ? "failed" : "unsupported"); 979 rc = EBUSY; 980 goto err; 981 } 982 983 /* Prep first D2H command with SRST feature & clear busy/reset flags */ 984 ccb = ahci_get_err_ccb(ap); 985 cmd_slot = ccb->ccb_cmd_hdr; 986 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 987 988 fis = ccb->ccb_cmd_table->cfis; 989 fis[0] = ATA_FIS_TYPE_H2D; 990 fis[15] = ATA_FIS_CONTROL_SRST; 991 992 cmd_slot->prdtl = 0; 993 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 994 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 995 AHCI_CMD_LIST_FLAG_W); 996 997 ccb->ccb_xa.state = ATA_S_PENDING; 998 if (ahci_poll(ccb, 1000, NULL) != 0) 999 goto err; 1000 1001 /* Prep second D2H command to read status and complete reset sequence */ 1002 fis[0] = ATA_FIS_TYPE_H2D; 1003 fis[15] = 0; 1004 1005 cmd_slot->prdtl = 0; 1006 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 1007 1008 ccb->ccb_xa.state = ATA_S_PENDING; 1009 if (ahci_poll(ccb, 1000, NULL) != 0) 1010 goto err; 1011 1012 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1013 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 1014 printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1015 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 1016 AHCI_PFMT_TFD_STS); 1017 rc = EBUSY; 1018 goto err; 1019 } 1020 1021 rc = 0; 1022 err: 1023 if (ccb != NULL) { 1024 /* Abort our command, if it failed, by stopping command DMA. */ 1025 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 1026 printf("%s: stopping the port, softreset slot %d was " 1027 "still active.\n", PORTNAME(ap), ccb->ccb_slot); 1028 ahci_port_stop(ap, 0); 1029 } 1030 ccb->ccb_xa.state = ATA_S_ERROR; 1031 ahci_put_err_ccb(ccb); 1032 } 1033 1034 /* Restore saved CMD register state */ 1035 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1036 ap->ap_state = oldstate; 1037 1038 splx(s); 1039 1040 return (rc); 1041 } 1042 1043 int 1044 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 1045 { 1046 struct ahci_ccb *ccb = NULL; 1047 u_int32_t data; 1048 int count; 1049 int rc; 1050 int s; 1051 struct ahci_cmd_hdr *cmd_slot; 1052 u_int8_t *fis; 1053 1054 /* XXX FBS: ensure fbs is disabled on ap, since we're resetting 1055 * devices (AHCI section 9.3.8) 1056 */ 1057 1058 s = splbio(); 1059 /* ignore spurious IFS errors while resetting */ 1060 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 1061 ap->ap_pmp_ignore_ifs = 1; 1062 1063 count = 2; 1064 rc = 0; 1065 do { 1066 if (ccb != NULL) { 1067 ahci_put_pmp_ccb(ccb); 1068 ccb = NULL; 1069 } 1070 1071 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1072 printf("%s.%d: unable to clear PHY status\n", 1073 PORTNAME(ap), pmp_port); 1074 } 1075 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1076 /* maybe don't do this on the first loop: */ 1077 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1078 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1079 1080 /* send first softreset FIS */ 1081 ccb = ahci_get_pmp_ccb(ap); 1082 cmd_slot = ccb->ccb_cmd_hdr; 1083 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1084 1085 fis = ccb->ccb_cmd_table->cfis; 1086 fis[0] = ATA_FIS_TYPE_H2D; 1087 fis[1] = pmp_port; 1088 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1089 1090 cmd_slot->prdtl = 0; 1091 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1092 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1093 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1094 1095 ccb->ccb_xa.state = ATA_S_PENDING; 1096 1097 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 1098 PORTNAME(ap), pmp_port); 1099 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1100 printf("%s.%d: PMP port softreset cmd failed\n", 1101 PORTNAME(ap), pmp_port); 1102 rc = EBUSY; 1103 if (count > 0) { 1104 /* probably delay a while to allow 1105 * it to settle down? 1106 */ 1107 } 1108 continue; 1109 } 1110 1111 /* send signature FIS */ 1112 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1113 fis[0] = ATA_FIS_TYPE_H2D; 1114 fis[1] = pmp_port; 1115 fis[15] = ATA_FIS_CONTROL_4BIT; 1116 1117 cmd_slot->prdtl = 0; 1118 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1119 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1120 1121 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 1122 PORTNAME(ap), pmp_port); 1123 ccb->ccb_xa.state = ATA_S_PENDING; 1124 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1125 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 1126 "failed\n", PORTNAME(ap), pmp_port); 1127 rc = EBUSY; 1128 if (count > 0) { 1129 /* sleep a while? */ 1130 } 1131 continue; 1132 } 1133 1134 fis[15] = 0; 1135 break; 1136 } while (count--); 1137 1138 if (ccb != NULL) { 1139 ahci_put_pmp_ccb(ccb); 1140 ccb = NULL; 1141 } 1142 1143 /* clean up a bit */ 1144 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1145 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1146 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1147 ap->ap_pmp_ignore_ifs = 0; 1148 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 1149 splx(s); 1150 1151 return (rc); 1152 } 1153 1154 int 1155 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 1156 { 1157 int sig; 1158 1159 ap->ap_state = AP_S_PMP_PORT_PROBE; 1160 1161 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 1162 pmp_port); 1163 if (ahci_pmp_port_portreset(ap, pmp_port)) { 1164 printf("%s.%d: unable to probe PMP port; portreset failed\n", 1165 PORTNAME(ap), pmp_port); 1166 ap->ap_state = AP_S_NORMAL; 1167 return (ATA_PORT_T_NONE); 1168 } 1169 1170 if (ahci_pmp_port_softreset(ap, pmp_port)) { 1171 printf("%s.%d: unable to probe PMP port due to softreset " 1172 "failure\n", PORTNAME(ap), pmp_port); 1173 ap->ap_state = AP_S_NORMAL; 1174 return (ATA_PORT_T_NONE); 1175 } 1176 1177 sig = ahci_port_signature(ap); 1178 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 1179 PORTNAME(ap), pmp_port, sig); 1180 ap->ap_state = AP_S_NORMAL; 1181 return (sig); 1182 } 1183 1184 1185 void 1186 ahci_flush_tfd(struct ahci_port *ap) 1187 { 1188 u_int32_t r; 1189 1190 r = ahci_pread(ap, AHCI_PREG_SERR); 1191 if (r & AHCI_PREG_SERR_DIAG_X) 1192 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1193 } 1194 1195 u_int32_t 1196 ahci_active_mask(struct ahci_port *ap) 1197 { 1198 u_int32_t mask; 1199 1200 mask = ahci_pread(ap, AHCI_PREG_CI); 1201 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 1202 mask |= ahci_pread(ap, AHCI_PREG_SACT); 1203 return mask; 1204 } 1205 1206 void 1207 ahci_pmp_probe_timeout(void *cookie) 1208 { 1209 struct ahci_ccb *ccb = cookie; 1210 struct ahci_port *ap = ccb->ccb_port; 1211 u_int32_t mask; 1212 1213 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 1214 switch (ccb->ccb_xa.state) { 1215 case ATA_S_PENDING: 1216 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1217 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1218 break; 1219 1220 case ATA_S_ONCHIP: 1221 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 1222 /* clear the command on-chip */ 1223 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 1224 ap->ap_sactive == 0); 1225 ahci_port_stop(ap, 0); 1226 ahci_port_start(ap, 0); 1227 1228 if (ahci_active_mask(ap) != 0) { 1229 ahci_port_stop(ap, 0); 1230 ahci_port_start(ap, 0); 1231 mask = ahci_active_mask(ap); 1232 if (mask != 0) { 1233 printf("%s: ahci_pmp_probe_timeout: failed to " 1234 "clear active cmds: %08x\n", PORTNAME(ap), 1235 mask); 1236 } 1237 } 1238 1239 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1240 ap->ap_active &= ~(1 << ccb->ccb_slot); 1241 KASSERT(ap->ap_active_cnt > 0); 1242 --ap->ap_active_cnt; 1243 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n", 1244 PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt); 1245 break; 1246 1247 default: 1248 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 1249 PORTNAME(ap), ccb->ccb_xa.state); 1250 } 1251 } 1252 1253 int 1254 ahci_port_signature(struct ahci_port *ap) 1255 { 1256 u_int32_t sig; 1257 1258 sig = ahci_pread(ap, AHCI_PREG_SIG); 1259 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 1260 return (ATA_PORT_T_ATAPI); 1261 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 1262 0xffff0000)) 1263 return (ATA_PORT_T_PM); 1264 else 1265 return (ATA_PORT_T_DISK); 1266 } 1267 1268 int 1269 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 1270 { 1271 u_int32_t cmd, data; 1272 int loop; 1273 int rc = 1; 1274 int s; 1275 1276 s = splbio(); 1277 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 1278 pmp_port); 1279 1280 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1281 1282 /* turn off power management and disable the PHY */ 1283 data = AHCI_PREG_SCTL_IPM_DISABLED; 1284 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 1285 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 1286 goto err; 1287 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1288 goto err; 1289 delay(10000); 1290 1291 /* start COMRESET */ 1292 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1293 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1294 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 1295 pmp_port); 1296 data |= AHCI_PREG_SCTL_SPD_GEN1; 1297 } else 1298 data |= AHCI_PREG_SCTL_SPD_ANY; 1299 1300 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1301 goto err; 1302 1303 /* give it a while to settle down */ 1304 delay(100000); 1305 1306 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1307 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 1308 pmp_port); 1309 } 1310 1311 /* start trying to negotiate */ 1312 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1313 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 1314 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1315 goto err; 1316 1317 /* give it a while to detect */ 1318 for (loop = 3; loop; --loop) { 1319 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1320 goto err; 1321 if (data & AHCI_PREG_SSTS_DET) 1322 break; 1323 delay(100000); 1324 } 1325 if (loop == 0) { 1326 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 1327 goto err; 1328 } 1329 1330 /* give it even longer to fully negotiate */ 1331 for (loop = 30; loop; --loop) { 1332 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1333 goto err; 1334 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 1335 break; 1336 delay(100000); 1337 } 1338 1339 if (loop == 0) { 1340 printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 1341 pmp_port); 1342 goto err; 1343 } 1344 1345 /* device detected */ 1346 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 1347 pmp_port); 1348 1349 /* clean up a bit */ 1350 delay(100000); 1351 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1352 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1353 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1354 1355 rc = 0; 1356 err: 1357 splx(s); 1358 return (rc); 1359 } 1360 1361 /* AHCI port reset, Section 10.4.2 */ 1362 int 1363 ahci_port_portreset(struct ahci_port *ap, int pmp) 1364 { 1365 u_int32_t cmd, r; 1366 int rc, s, retries = 0; 1367 1368 s = splbio(); 1369 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 1370 1371 /* Save previous command register state */ 1372 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1373 1374 /* Clear ST, ignoring failure */ 1375 ahci_port_stop(ap, 0); 1376 1377 /* Perform device detection */ 1378 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 1379 retry: 1380 delay(10000); 1381 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1382 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1383 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 1384 r |= AHCI_PREG_SCTL_SPD_GEN1; 1385 } else 1386 r |= AHCI_PREG_SCTL_SPD_ANY; 1387 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1388 delay(10000); /* wait at least 1ms for COMRESET to be sent */ 1389 r &= ~AHCI_PREG_SCTL_DET_INIT; 1390 r |= AHCI_PREG_SCTL_DET_NONE; 1391 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1392 delay(10000); 1393 1394 /* Wait for device to be detected and communications established */ 1395 if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET, 1396 AHCI_PREG_SSTS_DET_DEV, 1)) { 1397 rc = ENODEV; 1398 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 1399 /* this may be a port multiplier with no device 1400 * on port 0, so still do the pmp check if requested. 1401 */ 1402 } else { 1403 goto err; 1404 } 1405 } else { 1406 /* Clear SERR (incl X bit), so TFD can update */ 1407 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1408 1409 /* Wait for device to become ready */ 1410 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1411 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 1412 /* even if the device doesn't wake up, check if there's 1413 * a port multiplier there 1414 */ 1415 if (retries == 0) { 1416 retries = 1; 1417 goto retry; 1418 } 1419 rc = EBUSY; 1420 } else { 1421 rc = 0; 1422 } 1423 } 1424 1425 if (pmp != 0) { 1426 if (ahci_port_detect_pmp(ap) != 0) { 1427 rc = EBUSY; 1428 } 1429 } 1430 1431 err: 1432 /* Restore preserved port state */ 1433 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1434 splx(s); 1435 1436 return (rc); 1437 } 1438 1439 int 1440 ahci_port_detect_pmp(struct ahci_port *ap) 1441 { 1442 int count, pmp_rc, rc; 1443 u_int32_t r, cmd; 1444 struct ahci_cmd_hdr *cmd_slot; 1445 struct ahci_ccb *ccb = NULL; 1446 u_int8_t *fis = NULL; 1447 1448 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 1449 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 1450 return 0; 1451 } 1452 1453 rc = 0; 1454 pmp_rc = 0; 1455 count = 2; 1456 do { 1457 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 1458 count); 1459 if (ccb != NULL) { 1460 ahci_put_pmp_ccb(ccb); 1461 ccb = NULL; 1462 } 1463 ahci_port_stop(ap, 0); 1464 ap->ap_state = AP_S_PMP_PROBE; 1465 1466 /* set PMA in cmd reg */ 1467 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1468 if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 1469 cmd |= AHCI_PREG_CMD_PMA; 1470 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1471 } 1472 1473 /* Flush errors and request CLO unconditionally, 1474 * then start the port 1475 */ 1476 r = ahci_pread(ap, AHCI_PREG_SERR); 1477 if (r & AHCI_PREG_SERR_DIAG_X) 1478 ahci_pwrite(ap, AHCI_PREG_SERR, 1479 AHCI_PREG_SERR_DIAG_X); 1480 1481 /* Request CLO */ 1482 ahci_port_clo(ap); 1483 1484 /* Clear port errors to permit TFD transfer */ 1485 r = ahci_pread(ap, AHCI_PREG_SERR); 1486 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1487 1488 /* XXX FBS: ensure we don't enable FBS here, since 1489 * we're resetting the port 1490 * (AHCI section 9.3.8) 1491 */ 1492 /* Restart port */ 1493 if (ahci_port_start(ap, 0)) { 1494 rc = EBUSY; 1495 printf("%s: failed to start port, cannot probe PMP\n", 1496 PORTNAME(ap)); 1497 break; 1498 } 1499 1500 /* Check whether CLO worked */ 1501 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1502 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1503 u_int32_t cap; 1504 1505 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 1506 printf("%s: CLO %s, need port reset\n", 1507 PORTNAME(ap), 1508 ISSET(cap, AHCI_REG_CAP_SCLO) 1509 ? "failed" : "unsupported"); 1510 pmp_rc = EBUSY; 1511 break; 1512 } 1513 1514 /* Prep first command with SRST feature & 1515 * clear busy/reset flags 1516 */ 1517 ccb = ahci_get_pmp_ccb(ap); 1518 cmd_slot = ccb->ccb_cmd_hdr; 1519 memset(ccb->ccb_cmd_table, 0, 1520 sizeof(struct ahci_cmd_table)); 1521 1522 fis = ccb->ccb_cmd_table->cfis; 1523 fis[0] = ATA_FIS_TYPE_H2D; 1524 fis[1] = SATA_PMP_CONTROL_PORT; 1525 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1526 1527 cmd_slot->prdtl = 0; 1528 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1529 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1530 AHCI_CMD_LIST_FLAG_PMP); 1531 1532 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 1533 PORTNAME(ap)); 1534 ccb->ccb_xa.state = ATA_S_PENDING; 1535 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1536 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 1537 PORTNAME(ap)); 1538 pmp_rc = EBUSY; 1539 continue; 1540 } 1541 1542 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1543 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1544 printf("%s: port busy after first PMP probe FIS\n", 1545 PORTNAME(ap)); 1546 } 1547 1548 /* clear errors in case the device 1549 * didn't reset cleanly 1550 */ 1551 ahci_flush_tfd(ap); 1552 r = ahci_pread(ap, AHCI_PREG_SERR); 1553 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1554 1555 /* Prep second command to read status and 1556 * complete reset sequence 1557 */ 1558 memset(ccb->ccb_cmd_table, 0, 1559 sizeof(struct ahci_cmd_table)); 1560 fis[0] = ATA_FIS_TYPE_H2D; 1561 fis[1] = SATA_PMP_CONTROL_PORT; 1562 fis[15] = ATA_FIS_CONTROL_4BIT; 1563 1564 cmd_slot->prdtl = 0; 1565 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1566 AHCI_CMD_LIST_FLAG_PMP); 1567 1568 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 1569 PORTNAME(ap)); 1570 ccb->ccb_xa.state = ATA_S_PENDING; 1571 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1572 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 1573 "cmd failed\n", PORTNAME(ap)); 1574 pmp_rc = EBUSY; 1575 continue; 1576 } 1577 1578 /* apparently we need to retry at least once 1579 * to get the right signature 1580 */ 1581 fis[15] = 0; 1582 pmp_rc = 0; 1583 } while (--count); 1584 1585 if (ccb != NULL) { 1586 ahci_put_pmp_ccb(ccb); 1587 ccb = NULL; 1588 } 1589 1590 if (ap->ap_state == AP_S_PMP_PROBE) { 1591 ap->ap_state = AP_S_NORMAL; 1592 } 1593 1594 if (pmp_rc == 0) { 1595 if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 1596 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 1597 PORTNAME(ap)); 1598 pmp_rc = EBUSY; 1599 } else { 1600 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 1601 PORTNAME(ap)); 1602 } 1603 } 1604 1605 if (pmp_rc == 0) { 1606 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 1607 pmp_rc = EBUSY; 1608 } else { 1609 /* XXX enable FBS if available */ 1610 rc = 0; 1611 } 1612 } 1613 1614 /* if PMP detection failed, so turn off the PMA bit and 1615 * reset the port again 1616 */ 1617 if (pmp_rc != 0) { 1618 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 1619 "the port\n", PORTNAME(ap)); 1620 ahci_port_stop(ap, 0); 1621 ahci_port_clo(ap); 1622 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1623 cmd &= ~AHCI_PREG_CMD_PMA; 1624 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1625 1626 ahci_pwrite(ap, AHCI_PREG_IE, 0); 1627 ahci_port_stop(ap, 0); 1628 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 1629 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 1630 ahci_flush_tfd(ap); 1631 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1632 1633 ahci_pwrite(ap, AHCI_PREG_IS, -1); 1634 1635 ahci_enable_interrupts(ap); 1636 1637 ahci_port_portreset(ap, 0); 1638 } 1639 1640 return (rc); 1641 } 1642 1643 void 1644 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1645 u_int32_t flags) 1646 { 1647 flags |= len - 1; 1648 1649 htolem64(&prd->dba, addr); 1650 htolem32(&prd->flags, flags); 1651 } 1652 1653 int 1654 ahci_load_prdt(struct ahci_ccb *ccb) 1655 { 1656 struct ahci_port *ap = ccb->ccb_port; 1657 struct ahci_softc *sc = ap->ap_sc; 1658 struct ata_xfer *xa = &ccb->ccb_xa; 1659 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1660 bus_dmamap_t dmap = ccb->ccb_dmamap; 1661 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1662 int i, error; 1663 1664 if (xa->datalen == 0) { 1665 ccb->ccb_cmd_hdr->prdtl = 0; 1666 return (0); 1667 } 1668 1669 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 1670 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1671 if (error != 0) { 1672 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1673 return (1); 1674 } 1675 1676 for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1677 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1678 dmap->dm_segs[i].ds_len, 0); 1679 } 1680 1681 ahci_load_prdt_seg(&prdt[i], 1682 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1683 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 1684 1685 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 1686 1687 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1688 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 1689 BUS_DMASYNC_PREWRITE); 1690 1691 return (0); 1692 } 1693 1694 void 1695 ahci_unload_prdt(struct ahci_ccb *ccb) 1696 { 1697 struct ahci_port *ap = ccb->ccb_port; 1698 struct ahci_softc *sc = ap->ap_sc; 1699 struct ata_xfer *xa = &ccb->ccb_xa; 1700 bus_dmamap_t dmap = ccb->ccb_dmamap; 1701 1702 if (xa->datalen != 0) { 1703 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1704 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 1705 BUS_DMASYNC_POSTWRITE); 1706 1707 bus_dmamap_unload(sc->sc_dmat, dmap); 1708 1709 if (ccb->ccb_xa.flags & ATA_F_NCQ) 1710 xa->resid = 0; 1711 else 1712 xa->resid = xa->datalen - 1713 lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 1714 } 1715 } 1716 1717 int 1718 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 1719 { 1720 struct ahci_port *ap = ccb->ccb_port; 1721 int s; 1722 1723 s = splbio(); 1724 ahci_start(ccb); 1725 do { 1726 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 1727 1 << ccb->ccb_slot)) { 1728 splx(s); 1729 return (0); 1730 } 1731 if (ccb->ccb_xa.state == ATA_S_ERROR) { 1732 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 1733 PORTNAME(ap), ccb->ccb_slot); 1734 /* pretend it timed out? */ 1735 if (timeout_fn != NULL) { 1736 timeout_fn(ccb); 1737 } 1738 splx(s); 1739 return (1); 1740 } 1741 1742 delay(1000); 1743 } while (--timeout > 0); 1744 1745 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 1746 if (timeout_fn != NULL) 1747 timeout_fn(ccb); 1748 1749 splx(s); 1750 1751 return (1); 1752 } 1753 1754 void 1755 ahci_start(struct ahci_ccb *ccb) 1756 { 1757 struct ahci_port *ap = ccb->ccb_port; 1758 struct ahci_softc *sc = ap->ap_sc; 1759 1760 /* Zero transferred byte count before transfer */ 1761 ccb->ccb_cmd_hdr->prdbc = 0; 1762 1763 /* Sync command list entry and corresponding command table entry */ 1764 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1765 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 1766 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 1767 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1768 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 1769 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 1770 1771 /* Prepare RFIS area for write by controller */ 1772 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 1773 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 1774 1775 /* XXX FBS: need to figure out whether we still need to keep NCQ and 1776 * non-queued commands separate when FBS is in use. I guess probably 1777 * not? it's not particularly clear from the spec.. 1778 */ 1779 1780 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1781 /* Issue NCQ commands only when there are no outstanding 1782 * standard commands. */ 1783 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 1784 (ap->ap_sactive != 0 && 1785 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 1786 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1787 } else { 1788 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV 1789 * to the port number 1790 */ 1791 1792 KASSERT(ap->ap_active_cnt == 0); 1793 ap->ap_sactive |= (1 << ccb->ccb_slot); 1794 ccb->ccb_xa.state = ATA_S_ONCHIP; 1795 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 1796 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1797 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 1798 } 1799 } else { 1800 /* Wait for all NCQ commands to finish before issuing standard 1801 * command. */ 1802 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 1803 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1804 else if (ap->ap_active_cnt < 2) { 1805 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV to the 1806 * port number 1807 */ 1808 1809 ap->ap_active |= 1 << ccb->ccb_slot; 1810 ccb->ccb_xa.state = ATA_S_ONCHIP; 1811 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1812 ap->ap_active_cnt++; 1813 } 1814 } 1815 } 1816 1817 void 1818 ahci_issue_pending_ncq_commands(struct ahci_port *ap) 1819 { 1820 struct ahci_ccb *nextccb; 1821 u_int32_t sact_change = 0; 1822 1823 KASSERT(ap->ap_active_cnt == 0); 1824 1825 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1826 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 1827 return; 1828 1829 /* XXX FBS: 1830 * - set AHCI_PREG_FBS_DEV for each command 1831 * - one write to AHCI_PREG_CI per command 1832 */ 1833 1834 /* Start all the NCQ commands at the head of the pending list. 1835 * If a port multiplier is attached to the port, we can only 1836 * issue commands for one of its ports at a time. 1837 */ 1838 if (ap->ap_sactive != 0 && 1839 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 1840 return; 1841 } 1842 1843 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 1844 do { 1845 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1846 sact_change |= 1 << nextccb->ccb_slot; 1847 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1848 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1849 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 1850 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 1851 1852 ap->ap_sactive |= sact_change; 1853 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 1854 ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 1855 } 1856 1857 void 1858 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 1859 { 1860 struct ahci_ccb *nextccb; 1861 1862 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1863 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 1864 if (last_was_ncq) { 1865 KASSERT(nextccb->ccb_xa.pmp_port != 1866 ap->ap_pmp_ncq_port); 1867 /* otherwise it should have been started already */ 1868 } else { 1869 ap->ap_active_cnt--; 1870 } 1871 1872 /* Issue NCQ commands only when there are no outstanding 1873 * standard commands, and previous NCQ commands for other 1874 * PMP ports have finished. 1875 */ 1876 if (ap->ap_active == 0) 1877 ahci_issue_pending_ncq_commands(ap); 1878 else 1879 KASSERT(ap->ap_active_cnt == 1); 1880 } else if (nextccb) { 1881 if (ap->ap_sactive != 0 || last_was_ncq) 1882 KASSERT(ap->ap_active_cnt == 0); 1883 1884 /* Wait for all NCQ commands to finish before issuing standard 1885 * command. */ 1886 if (ap->ap_sactive != 0) 1887 return; 1888 1889 /* Keep up to 2 standard commands on-chip at a time. */ 1890 do { 1891 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1892 ap->ap_active |= 1 << nextccb->ccb_slot; 1893 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1894 /* XXX FBS: set AHCI_PREG_FBS_DEV here */ 1895 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 1896 if (last_was_ncq) 1897 ap->ap_active_cnt++; 1898 if (ap->ap_active_cnt == 2) 1899 break; 1900 KASSERT(ap->ap_active_cnt == 1); 1901 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1902 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 1903 } else if (!last_was_ncq) { 1904 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 1905 1906 /* Standard command finished, none waiting to start. */ 1907 ap->ap_active_cnt--; 1908 } else { 1909 KASSERT(ap->ap_active_cnt == 0); 1910 1911 /* NCQ command finished. */ 1912 } 1913 } 1914 1915 int 1916 ahci_intr(void *arg) 1917 { 1918 struct ahci_softc *sc = arg; 1919 u_int32_t is, ack = 0; 1920 int port; 1921 1922 /* Read global interrupt status */ 1923 is = ahci_read(sc, AHCI_REG_IS); 1924 if (is == 0 || is == 0xffffffff) 1925 return (0); 1926 ack = is; 1927 1928 #ifdef AHCI_COALESCE 1929 /* Check coalescing interrupt first */ 1930 if (is & sc->sc_ccc_mask) { 1931 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 1932 DEVNAME(sc)); 1933 is &= ~sc->sc_ccc_mask; 1934 is |= sc->sc_ccc_ports_cur; 1935 } 1936 #endif 1937 1938 /* Process interrupts for each port */ 1939 while (is) { 1940 port = ffs(is) - 1; 1941 if (sc->sc_ports[port]) 1942 ahci_port_intr(sc->sc_ports[port], 1943 AHCI_PREG_CI_ALL_SLOTS); 1944 is &= ~(1 << port); 1945 } 1946 1947 /* Finally, acknowledge global interrupt */ 1948 ahci_write(sc, AHCI_REG_IS, ack); 1949 1950 return (1); 1951 } 1952 1953 u_int32_t 1954 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 1955 { 1956 struct ahci_softc *sc = ap->ap_sc; 1957 u_int32_t is, ci_saved, ci_masked, processed = 0; 1958 int slot, need_restart = 0; 1959 int process_error = 0; 1960 struct ahci_ccb *ccb; 1961 volatile u_int32_t *active; 1962 #ifdef DIAGNOSTIC 1963 u_int32_t tmp; 1964 #endif 1965 1966 is = ahci_pread(ap, AHCI_PREG_IS); 1967 1968 /* Ack port interrupt only if checking all command slots. */ 1969 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 1970 ahci_pwrite(ap, AHCI_PREG_IS, is); 1971 1972 if (is) 1973 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 1974 is, AHCI_PFMT_IS); 1975 1976 if (ap->ap_sactive) { 1977 /* Active NCQ commands - use SActive instead of CI */ 1978 KASSERT(ap->ap_active == 0); 1979 KASSERT(ap->ap_active_cnt == 0); 1980 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 1981 active = &ap->ap_sactive; 1982 } else { 1983 /* Save CI */ 1984 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 1985 active = &ap->ap_active; 1986 } 1987 1988 if (is & AHCI_PREG_IS_TFES) { 1989 process_error = 1; 1990 } else if (is & AHCI_PREG_IS_DHRS) { 1991 u_int32_t tfd; 1992 u_int32_t cmd; 1993 u_int32_t serr; 1994 1995 tfd = ahci_pread(ap, AHCI_PREG_TFD); 1996 cmd = ahci_pread(ap, AHCI_PREG_CMD); 1997 serr = ahci_pread(ap, AHCI_PREG_SERR); 1998 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 1999 (cmd & AHCI_PREG_CMD_CR) == 0) { 2000 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 2001 " %b, DIAG: %b\n", PORTNAME(ap), tfd, 2002 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2003 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2004 AHCI_PFMT_SERR_DIAG); 2005 process_error = 1; 2006 } else { 2007 /* rfis copy back is in the normal execution path */ 2008 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2009 } 2010 } 2011 2012 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 2013 if (process_error) { 2014 u_int32_t tfd, serr; 2015 int err_slot; 2016 2017 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2018 serr = ahci_pread(ap, AHCI_PREG_SERR); 2019 2020 if (ap->ap_sactive == 0) { 2021 /* Errored slot is easy to determine from CMD. */ 2022 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 2023 AHCI_PREG_CMD)); 2024 2025 if ((ci_saved & (1 << err_slot)) == 0) { 2026 /* 2027 * Hardware doesn't seem to report correct 2028 * slot number. If there's only one 2029 * outstanding command we can cope, 2030 * otherwise fail all active commands. 2031 */ 2032 if (ap->ap_active_cnt == 1) 2033 err_slot = ffs(ap->ap_active) - 1; 2034 else 2035 goto failall; 2036 } 2037 2038 ccb = &ap->ap_ccbs[err_slot]; 2039 2040 /* Preserve received taskfile data from the RFIS. */ 2041 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2042 sizeof(struct ata_fis_d2h)); 2043 } else 2044 err_slot = -1; /* Must extract error from log page */ 2045 2046 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 2047 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 2048 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2049 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2050 AHCI_PFMT_SERR_DIAG); 2051 2052 /* Turn off ST to clear CI and SACT. */ 2053 ahci_port_stop(ap, 0); 2054 need_restart = 1; 2055 2056 /* Clear SERR to enable capturing new errors. */ 2057 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2058 2059 /* Acknowledge the interrupts we can recover from. */ 2060 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2061 AHCI_PREG_IS_IFS); 2062 is = ahci_pread(ap, AHCI_PREG_IS); 2063 2064 /* If device hasn't cleared its busy status, try to idle it. */ 2065 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2066 2067 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 2068 (ap->ap_state == AP_S_ERROR_RECOVERY)) { 2069 /* can't reset the port here, just make sure 2070 * the operation fails and the port still works. 2071 */ 2072 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 2073 printf("%s: error on PMP port %d, idling " 2074 "device\n", PORTNAME(ap), 2075 ccb->ccb_xa.pmp_port); 2076 if (ahci_pmp_port_softreset(ap, 2077 ccb->ccb_xa.pmp_port) == 0) { 2078 printf("%s: unable to softreset port " 2079 "%d\n", PORTNAME(ap), 2080 ccb->ccb_xa.pmp_port); 2081 if (ahci_pmp_port_portreset(ap, 2082 ccb->ccb_xa.pmp_port)) { 2083 printf("%s: failed to port " 2084 " reset %d, giving up on " 2085 "it\n", PORTNAME(ap), 2086 ccb->ccb_xa.pmp_port); 2087 goto fatal; 2088 } 2089 } 2090 } else { 2091 printf("%s: attempting to idle device\n", 2092 PORTNAME(ap)); 2093 if (ahci_port_softreset(ap)) { 2094 printf("%s: failed to soft reset " 2095 "device\n", PORTNAME(ap)); 2096 if (ahci_port_portreset(ap, 0)) { 2097 printf("%s: failed to port " 2098 "reset device, give up on " 2099 "it\n", PORTNAME(ap)); 2100 goto fatal; 2101 } 2102 } 2103 } 2104 2105 /* Had to reset device, can't gather extended info. */ 2106 } else if (ap->ap_sactive) { 2107 /* Recover the NCQ error from log page 10h. 2108 * XXX FBS: need to do things to figure out where the 2109 * error came from. without FBS, we know the PMP port 2110 * responsible because we can only have queued commands 2111 * active for one port at a time. 2112 */ 2113 ahci_port_read_ncq_error(ap, &err_slot, 2114 ap->ap_pmp_ncq_port); 2115 if (err_slot < 0) 2116 goto failall; 2117 2118 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 2119 PORTNAME(ap), err_slot); 2120 2121 ccb = &ap->ap_ccbs[err_slot]; 2122 } else { 2123 /* Didn't reset, could gather extended info from log. */ 2124 } 2125 2126 /* 2127 * If we couldn't determine the errored slot, reset the port 2128 * and fail all the active slots. 2129 */ 2130 if (err_slot == -1) { 2131 if (ahci_port_softreset(ap) != 0 && 2132 ahci_port_portreset(ap, 0) != 0) { 2133 printf("%s: couldn't reset after NCQ error, " 2134 "disabling device.\n", PORTNAME(ap)); 2135 goto fatal; 2136 } 2137 printf("%s: couldn't recover NCQ error, failing " 2138 "all outstanding commands.\n", PORTNAME(ap)); 2139 goto failall; 2140 } 2141 2142 /* Clear the failed command in saved CI so completion runs. */ 2143 ci_saved &= ~(1 << err_slot); 2144 2145 /* Note the error in the ata_xfer. */ 2146 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2147 ccb->ccb_xa.state = ATA_S_ERROR; 2148 2149 #ifdef DIAGNOSTIC 2150 /* There may only be one outstanding standard command now. */ 2151 if (ap->ap_sactive == 0) { 2152 tmp = ci_saved; 2153 if (tmp) { 2154 slot = ffs(tmp) - 1; 2155 tmp &= ~(1 << slot); 2156 KASSERT(tmp == 0); 2157 } 2158 } 2159 #endif 2160 } 2161 2162 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 2163 * when there's a normal SATA device attached. 2164 */ 2165 if ((ap->ap_state == AP_S_PMP_PROBE) && 2166 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 2167 (is & AHCI_PREG_IS_IPMS)) { 2168 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2169 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 2170 PORTNAME(ap), slot); 2171 2172 ccb = &ap->ap_ccbs[slot]; 2173 ccb->ccb_xa.state = ATA_S_ERROR; 2174 2175 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 2176 is &= ~AHCI_PREG_IS_IPMS; 2177 } 2178 2179 /* ignore IFS errors while resetting a PMP port */ 2180 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 2181 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 2182 "port\n", PORTNAME(ap)); 2183 2184 need_restart = 1; 2185 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2186 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2187 is &= ~AHCI_PREG_IS_IFS; 2188 goto failall; 2189 } 2190 2191 /* Check for remaining errors - they are fatal. */ 2192 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2193 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2194 printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 2195 PORTNAME(ap), is, AHCI_PFMT_IS); 2196 2197 /* XXX try recovery first */ 2198 goto fatal; 2199 } 2200 2201 /* Fail all outstanding commands if we know the port won't recover. */ 2202 if (ap->ap_state == AP_S_FATAL_ERROR) { 2203 fatal: 2204 ap->ap_state = AP_S_FATAL_ERROR; 2205 failall: 2206 2207 /* Ensure port is shut down. */ 2208 ahci_port_stop(ap, 1); 2209 2210 /* Error all the active slots. */ 2211 ci_masked = ci_saved & *active; 2212 while (ci_masked) { 2213 slot = ffs(ci_masked) - 1; 2214 ccb = &ap->ap_ccbs[slot]; 2215 ci_masked &= ~(1 << slot); 2216 ccb->ccb_xa.state = ATA_S_ERROR; 2217 } 2218 2219 /* Run completion for all active slots. */ 2220 ci_saved &= ~*active; 2221 2222 /* Don't restart the port if our problems were deemed fatal. */ 2223 if (ap->ap_state == AP_S_FATAL_ERROR) 2224 need_restart = 0; 2225 } 2226 2227 /* 2228 * CCB completion is detected by noticing its slot's bit in CI has 2229 * changed to zero some time after we activated it. 2230 * If we are polling, we may only be interested in particular slot(s). 2231 */ 2232 ci_masked = ~ci_saved & *active & ci_mask; 2233 while (ci_masked) { 2234 slot = ffs(ci_masked) - 1; 2235 ccb = &ap->ap_ccbs[slot]; 2236 ci_masked &= ~(1 << slot); 2237 2238 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2239 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2240 " (error)" : ""); 2241 2242 bus_dmamap_sync(sc->sc_dmat, 2243 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2244 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 2245 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 2246 2247 bus_dmamap_sync(sc->sc_dmat, 2248 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2249 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 2250 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 2251 2252 bus_dmamap_sync(sc->sc_dmat, 2253 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 2254 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 2255 2256 *active &= ~(1 << ccb->ccb_slot); 2257 /* Copy the rfis into the ccb if we were asked for it */ 2258 if (ccb->ccb_xa.state == ATA_S_ONCHIP && 2259 ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 2260 memcpy(&ccb->ccb_xa.rfis, 2261 ap->ap_rfis->rfis, 2262 sizeof(struct ata_fis_d2h)); 2263 } 2264 2265 ccb->ccb_done(ccb); 2266 2267 processed |= 1 << ccb->ccb_slot; 2268 } 2269 2270 if (need_restart) { 2271 /* Restart command DMA on the port */ 2272 ahci_port_start(ap, 0); 2273 2274 /* Re-enable outstanding commands on port. */ 2275 if (ci_saved) { 2276 #ifdef DIAGNOSTIC 2277 tmp = ci_saved; 2278 while (tmp) { 2279 slot = ffs(tmp) - 1; 2280 tmp &= ~(1 << slot); 2281 ccb = &ap->ap_ccbs[slot]; 2282 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2283 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 2284 (!!ap->ap_sactive)); 2285 } 2286 #endif 2287 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 2288 "re-enabling%s slots %08x\n", PORTNAME(ap), 2289 ap->ap_sactive ? " NCQ" : "", ci_saved); 2290 2291 /* XXX FBS: 2292 * - need to set AHCI_PREG_FBS_DEV for each command 2293 * - can't do multiple commands with a single write to 2294 * AHCI_PREG_CI 2295 */ 2296 if (ap->ap_sactive) 2297 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 2298 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 2299 } 2300 } 2301 2302 return (processed); 2303 } 2304 2305 struct ahci_ccb * 2306 ahci_get_ccb(struct ahci_port *ap) 2307 { 2308 struct ahci_ccb *ccb; 2309 2310 mtx_enter(&ap->ap_ccb_mtx); 2311 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2312 if (ccb != NULL) { 2313 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2314 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2315 ccb->ccb_xa.state = ATA_S_SETUP; 2316 } 2317 mtx_leave(&ap->ap_ccb_mtx); 2318 2319 return (ccb); 2320 } 2321 2322 void 2323 ahci_put_ccb(struct ahci_ccb *ccb) 2324 { 2325 struct ahci_port *ap = ccb->ccb_port; 2326 2327 #ifdef DIAGNOSTIC 2328 if (ccb->ccb_xa.state != ATA_S_COMPLETE && 2329 ccb->ccb_xa.state != ATA_S_TIMEOUT && 2330 ccb->ccb_xa.state != ATA_S_ERROR) { 2331 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 2332 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 2333 ccb->ccb_slot); 2334 } 2335 #endif 2336 2337 ccb->ccb_xa.state = ATA_S_PUT; 2338 mtx_enter(&ap->ap_ccb_mtx); 2339 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2340 mtx_leave(&ap->ap_ccb_mtx); 2341 } 2342 2343 struct ahci_ccb * 2344 ahci_get_err_ccb(struct ahci_port *ap) 2345 { 2346 struct ahci_ccb *err_ccb; 2347 u_int32_t sact; 2348 2349 splassert(IPL_BIO); 2350 2351 /* No commands may be active on the chip. */ 2352 sact = ahci_pread(ap, AHCI_PREG_SACT); 2353 if (sact != 0) 2354 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 2355 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2356 2357 #ifdef DIAGNOSTIC 2358 KASSERT(ap->ap_err_busy == 0); 2359 ap->ap_err_busy = 1; 2360 #endif 2361 /* Save outstanding command state. */ 2362 ap->ap_err_saved_active = ap->ap_active; 2363 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2364 ap->ap_err_saved_sactive = ap->ap_sactive; 2365 2366 /* 2367 * Pretend we have no commands outstanding, so that completions won't 2368 * run prematurely. 2369 */ 2370 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2371 2372 /* 2373 * Grab a CCB to use for error recovery. This should never fail, as 2374 * we ask atascsi to reserve one for us at init time. 2375 */ 2376 err_ccb = ap->ap_ccb_err; 2377 err_ccb->ccb_xa.flags = 0; 2378 err_ccb->ccb_xa.state = ATA_S_SETUP; 2379 err_ccb->ccb_done = ahci_empty_done; 2380 2381 return (err_ccb); 2382 } 2383 2384 void 2385 ahci_put_err_ccb(struct ahci_ccb *ccb) 2386 { 2387 struct ahci_port *ap = ccb->ccb_port; 2388 u_int32_t sact; 2389 2390 splassert(IPL_BIO); 2391 2392 #ifdef DIAGNOSTIC 2393 KASSERT(ap->ap_err_busy); 2394 #endif 2395 /* No commands may be active on the chip */ 2396 sact = ahci_pread(ap, AHCI_PREG_SACT); 2397 if (sact != 0) 2398 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2399 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2400 2401 #ifdef DIAGNOSTIC 2402 /* Done with the CCB */ 2403 KASSERT(ccb == ap->ap_ccb_err); 2404 #endif 2405 2406 /* Restore outstanding command state */ 2407 ap->ap_sactive = ap->ap_err_saved_sactive; 2408 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2409 ap->ap_active = ap->ap_err_saved_active; 2410 2411 #ifdef DIAGNOSTIC 2412 ap->ap_err_busy = 0; 2413 #endif 2414 } 2415 2416 struct ahci_ccb * 2417 ahci_get_pmp_ccb(struct ahci_port *ap) 2418 { 2419 struct ahci_ccb *ccb; 2420 u_int32_t sact; 2421 2422 /* some PMP commands need to be issued on slot 1, 2423 * particularly the command that clears SRST and 2424 * fetches the device signature. 2425 * 2426 * ensure the chip is idle and ccb 1 is available. 2427 */ 2428 splassert(IPL_BIO); 2429 2430 sact = ahci_pread(ap, AHCI_PREG_SACT); 2431 if (sact != 0) 2432 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 2433 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2434 2435 ccb = &ap->ap_ccbs[1]; 2436 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2437 ccb->ccb_xa.flags = 0; 2438 ccb->ccb_done = ahci_pmp_cmd_done; 2439 2440 mtx_enter(&ap->ap_ccb_mtx); 2441 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2442 mtx_leave(&ap->ap_ccb_mtx); 2443 2444 return ccb; 2445 } 2446 2447 void 2448 ahci_put_pmp_ccb(struct ahci_ccb *ccb) 2449 { 2450 struct ahci_port *ap = ccb->ccb_port; 2451 u_int32_t sact; 2452 2453 /* make sure this is the right ccb */ 2454 KASSERT(ccb == &ap->ap_ccbs[1]); 2455 2456 /* No commands may be active on the chip */ 2457 sact = ahci_pread(ap, AHCI_PREG_SACT); 2458 if (sact != 0) 2459 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2460 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2461 2462 ccb->ccb_xa.state = ATA_S_PUT; 2463 mtx_enter(&ap->ap_ccb_mtx); 2464 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2465 mtx_leave(&ap->ap_ccb_mtx); 2466 } 2467 2468 int 2469 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 2470 { 2471 struct ahci_ccb *ccb; 2472 struct ahci_cmd_hdr *cmd_slot; 2473 u_int32_t cmd; 2474 struct ata_fis_h2d *fis; 2475 int rc = EIO, oldstate; 2476 2477 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 2478 oldstate = ap->ap_state; 2479 ap->ap_state = AP_S_ERROR_RECOVERY; 2480 2481 /* Save command register state. */ 2482 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 2483 2484 /* Port should have been idled already. Start it. */ 2485 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 2486 ahci_port_start(ap, 0); 2487 2488 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 2489 ccb = ahci_get_err_ccb(ap); 2490 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 2491 ccb->ccb_xa.data = ap->ap_err_scratch; 2492 ccb->ccb_xa.datalen = 512; 2493 cmd_slot = ccb->ccb_cmd_hdr; 2494 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2495 2496 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2497 fis->type = ATA_FIS_TYPE_H2D; 2498 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 2499 fis->command = ATA_C_READ_LOG_EXT; 2500 fis->lba_low = 0x10; /* queued error log page (10h) */ 2501 fis->sector_count = 1; /* number of sectors (1) */ 2502 fis->sector_count_exp = 0; 2503 fis->lba_mid = 0; /* starting offset */ 2504 fis->lba_mid_exp = 0; 2505 fis->device = 0; 2506 2507 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 2508 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 2509 2510 if (ahci_load_prdt(ccb) != 0) { 2511 rc = ENOMEM; /* XXX caller must abort all commands */ 2512 goto err; 2513 } 2514 2515 ccb->ccb_xa.state = ATA_S_PENDING; 2516 if (ahci_poll(ccb, 1000, NULL) != 0 || 2517 ccb->ccb_xa.state == ATA_S_ERROR) 2518 goto err; 2519 2520 rc = 0; 2521 err: 2522 /* Abort our command, if it failed, by stopping command DMA. */ 2523 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 2524 printf("%s: log page read failed, slot %d was still active.\n", 2525 PORTNAME(ap), ccb->ccb_slot); 2526 ahci_port_stop(ap, 0); 2527 } 2528 2529 /* Done with the error CCB now. */ 2530 ahci_unload_prdt(ccb); 2531 ahci_put_err_ccb(ccb); 2532 2533 /* Extract failed register set and tags from the scratch space. */ 2534 if (rc == 0) { 2535 struct ata_log_page_10h *log; 2536 int err_slot; 2537 2538 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2539 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 2540 /* Not queued bit was set - wasn't an NCQ error? */ 2541 printf("%s: read NCQ error page, but not an NCQ " 2542 "error?\n", PORTNAME(ap)); 2543 rc = ESRCH; 2544 } else { 2545 /* Copy back the log record as a D2H register FIS. */ 2546 *err_slotp = err_slot = log->err_regs.type & 2547 ATA_LOG_10H_TYPE_TAG_MASK; 2548 2549 ccb = &ap->ap_ccbs[err_slot]; 2550 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2551 sizeof(struct ata_fis_d2h)); 2552 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2553 ccb->ccb_xa.rfis.flags = 0; 2554 } 2555 } 2556 2557 /* Restore saved CMD register state */ 2558 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 2559 ap->ap_state = oldstate; 2560 2561 return (rc); 2562 } 2563 2564 struct ahci_dmamem * 2565 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 2566 { 2567 struct ahci_dmamem *adm; 2568 int nsegs; 2569 2570 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 2571 if (adm == NULL) 2572 return (NULL); 2573 2574 adm->adm_size = size; 2575 2576 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2577 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2578 goto admfree; 2579 2580 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 2581 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 2582 goto destroy; 2583 2584 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2585 &adm->adm_kva, BUS_DMA_NOWAIT) != 0) 2586 goto free; 2587 2588 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 2589 NULL, BUS_DMA_NOWAIT) != 0) 2590 goto unmap; 2591 2592 return (adm); 2593 2594 unmap: 2595 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 2596 free: 2597 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2598 destroy: 2599 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2600 admfree: 2601 free(adm, M_DEVBUF, 0); 2602 2603 return (NULL); 2604 } 2605 2606 void 2607 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 2608 { 2609 bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 2610 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 2611 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2612 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2613 free(adm, M_DEVBUF, 0); 2614 } 2615 2616 u_int32_t 2617 ahci_read(struct ahci_softc *sc, bus_size_t r) 2618 { 2619 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2620 BUS_SPACE_BARRIER_READ); 2621 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 2622 } 2623 2624 void 2625 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 2626 { 2627 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 2628 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2629 BUS_SPACE_BARRIER_WRITE); 2630 } 2631 2632 int 2633 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 2634 u_int32_t target) 2635 { 2636 int i; 2637 2638 for (i = 0; i < 1000; i++) { 2639 if ((ahci_read(sc, r) & mask) != target) 2640 return (0); 2641 delay(1000); 2642 } 2643 2644 return (1); 2645 } 2646 2647 u_int32_t 2648 ahci_pread(struct ahci_port *ap, bus_size_t r) 2649 { 2650 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2651 BUS_SPACE_BARRIER_READ); 2652 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 2653 } 2654 2655 void 2656 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 2657 { 2658 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 2659 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2660 BUS_SPACE_BARRIER_WRITE); 2661 } 2662 2663 int 2664 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 2665 u_int32_t target, int n) 2666 { 2667 int i; 2668 2669 for (i = 0; i < n * 1000; i++) { 2670 if ((ahci_pread(ap, r) & mask) == target) 2671 return (0); 2672 delay(1000); 2673 } 2674 2675 return (1); 2676 } 2677 2678 int 2679 ahci_ata_probe(void *xsc, int port, int lun) 2680 { 2681 struct ahci_softc *sc = xsc; 2682 struct ahci_port *ap = sc->sc_ports[port]; 2683 2684 if (ap == NULL) 2685 return (ATA_PORT_T_NONE); 2686 2687 if (lun != 0) { 2688 int pmp_port = lun - 1; 2689 if (pmp_port >= ap->ap_pmp_ports) { 2690 return (ATA_PORT_T_NONE); 2691 } 2692 return (ahci_pmp_port_probe(ap, pmp_port)); 2693 } else { 2694 return (ahci_port_signature(ap)); 2695 } 2696 } 2697 2698 void 2699 ahci_ata_free(void *xsc, int port, int lun) 2700 { 2701 2702 } 2703 2704 struct ata_xfer * 2705 ahci_ata_get_xfer(void *aaa_cookie, int port) 2706 { 2707 struct ahci_softc *sc = aaa_cookie; 2708 struct ahci_port *ap = sc->sc_ports[port]; 2709 struct ahci_ccb *ccb; 2710 2711 ccb = ahci_get_ccb(ap); 2712 if (ccb == NULL) { 2713 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 2714 PORTNAME(ap)); 2715 return (NULL); 2716 } 2717 2718 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 2719 PORTNAME(ap), ccb->ccb_slot); 2720 2721 return ((struct ata_xfer *)ccb); 2722 } 2723 2724 void 2725 ahci_ata_put_xfer(struct ata_xfer *xa) 2726 { 2727 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2728 2729 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 2730 2731 ahci_put_ccb(ccb); 2732 } 2733 2734 void 2735 ahci_ata_cmd(struct ata_xfer *xa) 2736 { 2737 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2738 struct ahci_cmd_hdr *cmd_slot; 2739 int s; 2740 u_int16_t flags; 2741 2742 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 2743 goto failcmd; 2744 2745 ccb->ccb_done = ahci_ata_cmd_done; 2746 2747 cmd_slot = ccb->ccb_cmd_hdr; 2748 flags = 5 /* FIS length (in DWORDs) */; 2749 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 2750 2751 if (xa->flags & ATA_F_WRITE) 2752 flags |= AHCI_CMD_LIST_FLAG_W; 2753 2754 if (xa->flags & ATA_F_PACKET) 2755 flags |= AHCI_CMD_LIST_FLAG_A; 2756 2757 htolem16(&cmd_slot->flags, flags); 2758 2759 if (ahci_load_prdt(ccb) != 0) 2760 goto failcmd; 2761 2762 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 2763 2764 xa->state = ATA_S_PENDING; 2765 2766 if (xa->flags & ATA_F_POLL) 2767 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 2768 else { 2769 s = splbio(); 2770 timeout_add_msec(&xa->stimeout, xa->timeout); 2771 ahci_start(ccb); 2772 splx(s); 2773 } 2774 2775 return; 2776 2777 failcmd: 2778 s = splbio(); 2779 xa->state = ATA_S_ERROR; 2780 ata_complete(xa); 2781 splx(s); 2782 } 2783 2784 void 2785 ahci_pmp_cmd_done(struct ahci_ccb *ccb) 2786 { 2787 struct ata_xfer *xa = &ccb->ccb_xa; 2788 2789 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2790 ahci_issue_pending_commands(ccb->ccb_port, 2791 xa->flags & ATA_F_NCQ); 2792 2793 xa->state = ATA_S_COMPLETE; 2794 } 2795 2796 2797 void 2798 ahci_ata_cmd_done(struct ahci_ccb *ccb) 2799 { 2800 struct ata_xfer *xa = &ccb->ccb_xa; 2801 2802 timeout_del(&xa->stimeout); 2803 2804 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2805 ahci_issue_pending_commands(ccb->ccb_port, 2806 xa->flags & ATA_F_NCQ); 2807 2808 ahci_unload_prdt(ccb); 2809 2810 if (xa->state == ATA_S_ONCHIP) 2811 xa->state = ATA_S_COMPLETE; 2812 #ifdef DIAGNOSTIC 2813 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 2814 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 2815 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 2816 ccb->ccb_slot); 2817 #endif 2818 if (xa->state != ATA_S_TIMEOUT) 2819 ata_complete(xa); 2820 } 2821 2822 void 2823 ahci_ata_cmd_timeout(void *arg) 2824 { 2825 struct ahci_ccb *ccb = arg; 2826 struct ata_xfer *xa = &ccb->ccb_xa; 2827 struct ahci_port *ap = ccb->ccb_port; 2828 int s, ccb_was_started, ncq_cmd; 2829 volatile u_int32_t *active; 2830 2831 s = splbio(); 2832 2833 ncq_cmd = (xa->flags & ATA_F_NCQ); 2834 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 2835 2836 if (ccb->ccb_xa.state == ATA_S_PENDING) { 2837 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 2838 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 2839 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2840 ccb_was_started = 0; 2841 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 2842 1 << ccb->ccb_slot)) { 2843 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 2844 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 2845 goto ret; 2846 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2847 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 2848 "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 2849 ISSET(*active, 1 << ccb->ccb_slot) ? 2850 " but slot is still active?" : "."); 2851 goto ret; 2852 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 2853 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 2854 1 << ccb->ccb_slot)) { 2855 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 2856 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 2857 ccb->ccb_slot); 2858 *active &= ~(1 << ccb->ccb_slot); 2859 ccb->ccb_done(ccb); 2860 goto ret; 2861 } else { 2862 ccb_was_started = 1; 2863 } 2864 2865 /* Complete the slot with a timeout error. */ 2866 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2867 *active &= ~(1 << ccb->ccb_slot); 2868 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 2869 ccb->ccb_done(ccb); /* This won't issue pending commands or run the 2870 atascsi completion. */ 2871 2872 /* Reset port to abort running command. */ 2873 if (ccb_was_started) { 2874 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 2875 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 2876 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 2877 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 2878 != 0) { 2879 printf("%s: failed to reset port during timeout " 2880 "handling, disabling it\n", PORTNAME(ap)); 2881 ap->ap_state = AP_S_FATAL_ERROR; 2882 } 2883 2884 /* Restart any other commands that were aborted by the reset. */ 2885 if (*active) { 2886 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 2887 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 2888 *active); 2889 if (ncq_cmd) 2890 ahci_pwrite(ap, AHCI_PREG_SACT, *active); 2891 ahci_pwrite(ap, AHCI_PREG_CI, *active); 2892 } 2893 } 2894 2895 /* Issue any pending commands now. */ 2896 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 2897 if (ccb_was_started) 2898 ahci_issue_pending_commands(ap, ncq_cmd); 2899 else if (ap->ap_active == 0) 2900 ahci_issue_pending_ncq_commands(ap); 2901 2902 /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 2903 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 2904 ata_complete(xa); 2905 2906 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 2907 ret: 2908 splx(s); 2909 } 2910 2911 void 2912 ahci_empty_done(struct ahci_ccb *ccb) 2913 { 2914 if (ccb->ccb_xa.state != ATA_S_ERROR) 2915 ccb->ccb_xa.state = ATA_S_COMPLETE; 2916 } 2917 2918 int 2919 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 2920 { 2921 struct ahci_ccb *ccb; 2922 struct ata_fis_h2d *fis; 2923 int error; 2924 2925 ccb = ahci_get_pmp_ccb(ap); 2926 if (ccb == NULL) { 2927 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2928 return (1); 2929 } 2930 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 2931 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2932 ccb->ccb_xa.state = ATA_S_PENDING; 2933 2934 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2935 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2936 fis->type = ATA_FIS_TYPE_H2D; 2937 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 2938 fis->command = ATA_C_READ_PM; 2939 fis->features = which; 2940 fis->device = target | ATA_H2D_DEVICE_LBA; 2941 fis->control = ATA_FIS_CONTROL_4BIT; 2942 2943 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 2944 error = 1; 2945 } else { 2946 *datap = ccb->ccb_xa.rfis.sector_count | 2947 (ccb->ccb_xa.rfis.lba_low << 8) | 2948 (ccb->ccb_xa.rfis.lba_mid << 16) | 2949 (ccb->ccb_xa.rfis.lba_high << 24); 2950 error = 0; 2951 } 2952 ahci_put_pmp_ccb(ccb); 2953 return (error); 2954 } 2955 2956 int 2957 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 2958 { 2959 struct ahci_ccb *ccb; 2960 struct ata_fis_h2d *fis; 2961 int error; 2962 2963 ccb = ahci_get_pmp_ccb(ap); 2964 if (ccb == NULL) { 2965 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2966 return (1); 2967 } 2968 ccb->ccb_xa.flags = ATA_F_POLL; 2969 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2970 ccb->ccb_xa.state = ATA_S_PENDING; 2971 2972 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2973 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2974 fis->type = ATA_FIS_TYPE_H2D; 2975 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 2976 fis->command = ATA_C_WRITE_PM; 2977 fis->features = which; 2978 fis->device = target | ATA_H2D_DEVICE_LBA; 2979 fis->sector_count = (u_int8_t)data; 2980 fis->lba_low = (u_int8_t)(data >> 8); 2981 fis->lba_mid = (u_int8_t)(data >> 16); 2982 fis->lba_high = (u_int8_t)(data >> 24); 2983 fis->control = ATA_FIS_CONTROL_4BIT; 2984 2985 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 2986 ahci_put_pmp_ccb(ccb); 2987 return (error); 2988 } 2989 2990 int 2991 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 2992 { 2993 int error; 2994 2995 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 2996 if (error == 0) 2997 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 2998 if (error) 2999 *datap = 0; 3000 3001 return (error); 3002 } 3003 3004 int 3005 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 3006 { 3007 u_int32_t chipid; 3008 u_int32_t rev; 3009 u_int32_t nports; 3010 u_int32_t features; 3011 u_int32_t enabled; 3012 int s; 3013 3014 s = splbio(); 3015 3016 if (ahci_pmp_read(ap, 15, 0, &chipid) || 3017 ahci_pmp_read(ap, 15, 1, &rev) || 3018 ahci_pmp_read(ap, 15, 2, &nports) || 3019 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 3020 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 3021 printf("%s: port multiplier identification failed\n", 3022 PORTNAME(ap)); 3023 splx(s); 3024 return (1); 3025 } 3026 splx(s); 3027 3028 nports &= 0x0F; 3029 3030 /* ignore SEMB port on SiI3726 port multiplier chips */ 3031 if (chipid == 0x37261095) { 3032 nports--; 3033 } 3034 3035 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 3036 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 3037 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 3038 SATA_PFMT_PM_FEA); 3039 3040 *ret_nports = nports; 3041 return (0); 3042 } 3043 3044 3045 #ifdef HIBERNATE 3046 void 3047 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 3048 { 3049 ccb->ccb_cmd_hdr->prdbc = 0; 3050 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 3051 } 3052 3053 int 3054 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 3055 { 3056 u_int32_t is, ci_saved; 3057 int process_error = 0; 3058 3059 is = ahci_pread(ap, AHCI_PREG_IS); 3060 3061 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 3062 3063 if (is & AHCI_PREG_IS_DHRS) { 3064 u_int32_t tfd; 3065 u_int32_t cmd; 3066 u_int32_t serr; 3067 3068 tfd = ahci_pread(ap, AHCI_PREG_TFD); 3069 cmd = ahci_pread(ap, AHCI_PREG_CMD); 3070 serr = ahci_pread(ap, AHCI_PREG_SERR); 3071 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 3072 (cmd & AHCI_PREG_CMD_CR) == 0) { 3073 process_error = 1; 3074 } else { 3075 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 3076 } 3077 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 3078 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 3079 process_error = 1; 3080 } 3081 3082 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 3083 if (process_error) { 3084 3085 /* Turn off ST to clear CI and SACT. */ 3086 ahci_port_stop(ap, 0); 3087 3088 /* just return an error indicator? we can't meaningfully 3089 * recover, and on the way back out we'll DVACT_RESUME which 3090 * resets and reinits the port. 3091 */ 3092 return (EIO); 3093 } 3094 3095 /* command is finished when the bit in CI for the slot goes to 0 */ 3096 if (ci_saved & (1 << ccb->ccb_slot)) { 3097 return (EAGAIN); 3098 } 3099 3100 return (0); 3101 } 3102 3103 void 3104 ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 3105 { 3106 struct ata_xfer *xa = &ccb->ccb_xa; 3107 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd; 3108 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 3109 int i; 3110 paddr_t data_phys; 3111 u_int64_t data_bus_phys; 3112 vaddr_t data_addr; 3113 size_t seglen; 3114 size_t buflen; 3115 3116 if (xa->datalen == 0) { 3117 ccb->ccb_cmd_hdr->prdtl = 0; 3118 return; 3119 } 3120 3121 /* derived from i386/amd64 _bus_dma_load_buffer; 3122 * for amd64 the buffer will always be dma safe. 3123 */ 3124 3125 buflen = xa->datalen; 3126 data_addr = (vaddr_t)xa->data; 3127 for (i = 0; buflen > 0; i++) { 3128 prd = &prdt[i]; 3129 3130 pmap_extract(pmap_kernel(), data_addr, &data_phys); 3131 data_bus_phys = data_phys; 3132 3133 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 3134 if (buflen < seglen) 3135 seglen = buflen; 3136 3137 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3138 3139 data_addr += seglen; 3140 buflen -= seglen; 3141 } 3142 3143 htolem16(&cmd_slot->prdtl, i); 3144 } 3145 3146 int 3147 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 3148 int op, void *page) 3149 { 3150 /* we use the 'real' ahci_port and ahci_softc here, but 3151 * never write to them 3152 */ 3153 struct { 3154 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 3155 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 3156 /* cmd table isn't actually used because of mysteries */ 3157 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 3158 struct ahci_port *ap; 3159 struct ahci_ccb ccb_buf; 3160 struct ahci_ccb *ccb; 3161 struct ahci_cmd_hdr *hdr_buf; 3162 int pmp_port; 3163 daddr_t poffset; 3164 size_t psize; 3165 } *my = page; 3166 struct ata_fis_h2d *fis; 3167 u_int32_t sector_count; 3168 struct ahci_cmd_hdr *cmd_slot; 3169 int rc; 3170 int timeout; 3171 u_int16_t flags; 3172 3173 if (op == HIB_INIT) { 3174 struct device *disk; 3175 struct device *scsibus; 3176 struct ahci_softc *sc; 3177 extern struct cfdriver sd_cd; 3178 struct scsi_link *link; 3179 struct scsibus_softc *bus_sc; 3180 int port; 3181 paddr_t page_phys; 3182 u_int64_t item_phys; 3183 u_int32_t cmd; 3184 3185 my->poffset = blkno; 3186 my->psize = size; 3187 3188 /* map dev to an ahci port */ 3189 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 3190 scsibus = disk->dv_parent; 3191 sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 3192 3193 /* find the scsi_link for the device, which has the port */ 3194 port = -1; 3195 bus_sc = (struct scsibus_softc *)scsibus; 3196 SLIST_FOREACH(link, &bus_sc->sc_link, bus_list) { 3197 if (link->device_softc == disk) { 3198 /* link->adapter_softc == sc->sc_atascsi */ 3199 port = link->target; 3200 if (link->lun > 0) 3201 my->pmp_port = link->lun - 1; 3202 else 3203 my->pmp_port = 0; 3204 3205 break; 3206 } 3207 } 3208 if (port == -1) { 3209 /* don't know where the disk is */ 3210 return (EIO); 3211 } 3212 3213 my->ap = sc->sc_ports[port]; 3214 3215 /* we're going to use the first command slot, 3216 * so ensure it's not already in use 3217 */ 3218 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 3219 /* this shouldn't happen, we should be idle */ 3220 return (EIO); 3221 } 3222 3223 /* stop the port so we can relocate to the hibernate page */ 3224 if (ahci_port_stop(my->ap, 1)) { 3225 return (EIO); 3226 } 3227 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 3228 3229 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 3230 3231 /* Setup RFIS base address */ 3232 item_phys = page_phys + ((void *)&my->rfis - page); 3233 ahci_pwrite(my->ap, AHCI_PREG_FBU, 3234 (u_int32_t)(item_phys >> 32)); 3235 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 3236 3237 /* Enable FIS reception and activate port. */ 3238 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3239 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 3240 AHCI_PREG_CMD_SUD; 3241 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 3242 AHCI_PREG_CMD_ICC_ACTIVE); 3243 3244 /* Check whether port activated. */ 3245 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3246 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 3247 return (EIO); 3248 } 3249 3250 /* Set up the single CCB */ 3251 my->ccb = &my->ccb_buf; 3252 my->ccb->ccb_slot = 0; 3253 my->ccb->ccb_port = my->ap; 3254 3255 /* Setup command list base address */ 3256 item_phys = page_phys + ((void *)&my->cmd_hdr - page); 3257 ahci_pwrite(my->ap, AHCI_PREG_CLBU, 3258 (u_int32_t)(item_phys >> 32)); 3259 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 3260 3261 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 3262 3263 /* use existing cmd table - don't know why moving to a new one fails */ 3264 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 3265 pmap_extract(pmap_kernel(), 3266 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 3267 &page_phys); 3268 item_phys = page_phys; 3269 #if 0 3270 /* use cmd table in hibernate page (doesn't work) */ 3271 my->ccb->ccb_cmd_table = &my->cmd_table; 3272 item_phys = page_phys + ((void *)&my->cmd_table - page); 3273 #endif 3274 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 3275 3276 my->ccb->ccb_xa.fis = 3277 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 3278 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 3279 my->ccb->ccb_xa.tag = 0; 3280 3281 /* Wait for ICC change to complete */ 3282 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 3283 3284 if (ahci_port_start(my->ap, 0)) { 3285 return (EIO); 3286 } 3287 3288 /* Flush interrupts for port */ 3289 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 3290 AHCI_PREG_IS)); 3291 ahci_write(sc, AHCI_REG_IS, 1 << port); 3292 3293 ahci_enable_interrupts(my->ap); 3294 return (0); 3295 } else if (op == HIB_DONE) { 3296 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 3297 return (0); 3298 } 3299 3300 if (blkno > my->psize) 3301 return (E2BIG); 3302 blkno += my->poffset; 3303 3304 /* build fis */ 3305 sector_count = size / 512; /* dlg promises this is okay */ 3306 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 3307 fis = my->ccb->ccb_xa.fis; 3308 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 3309 fis->lba_low = blkno & 0xff; 3310 fis->lba_mid = (blkno >> 8) & 0xff; 3311 fis->lba_high = (blkno >> 16) & 0xff; 3312 3313 if (sector_count > 0x100 || blkno > 0xfffffff) { 3314 /* Use LBA48 */ 3315 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 3316 ATA_C_READDMA_EXT; 3317 fis->device = ATA_H2D_DEVICE_LBA; 3318 fis->lba_low_exp = (blkno >> 24) & 0xff; 3319 fis->lba_mid_exp = (blkno >> 32) & 0xff; 3320 fis->lba_high_exp = (blkno >> 40) & 0xff; 3321 fis->sector_count = sector_count & 0xff; 3322 fis->sector_count_exp = (sector_count >> 8) & 0xff; 3323 } else { 3324 /* Use LBA */ 3325 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 3326 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 3327 fis->sector_count = sector_count & 0xff; 3328 } 3329 3330 my->ccb->ccb_xa.data = (void *)addr; 3331 my->ccb->ccb_xa.datalen = size; 3332 my->ccb->ccb_xa.pmp_port = my->pmp_port; 3333 my->ccb->ccb_xa.flags |= ATA_F_POLL; 3334 3335 cmd_slot = my->ccb->ccb_cmd_hdr; 3336 flags = 5; /* FIS length (in DWORDs) */ 3337 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 3338 3339 if (op == HIB_W) 3340 flags |= AHCI_CMD_LIST_FLAG_W; 3341 3342 htolem16(&cmd_slot->flags, flags); 3343 3344 ahci_hibernate_load_prdt(my->ccb); 3345 3346 ahci_hibernate_io_start(my->ap, my->ccb); 3347 timeout = 1000; 3348 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 3349 delay(1000); 3350 timeout--; 3351 if (timeout == 0) { 3352 return (EIO); 3353 } 3354 } 3355 3356 return (0); 3357 } 3358 3359 #endif 3360