1 /* $OpenBSD: ahci.c,v 1.43 2024/11/22 09:29:41 jan Exp $ */ 2 3 /* 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/buf.h> 24 #include <sys/kernel.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/queue.h> 28 #include <sys/mutex.h> 29 #include <sys/pool.h> 30 31 #include <machine/bus.h> 32 33 #include <dev/ic/ahcireg.h> 34 #include <dev/ic/ahcivar.h> 35 36 #ifdef AHCI_DEBUG 37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 38 while (0) 39 #define AHCI_D_TIMEOUT 0x00 40 #define AHCI_D_VERBOSE 0x01 41 #define AHCI_D_INTR 0x02 42 #define AHCI_D_XFER 0x08 43 int ahcidebug = AHCI_D_VERBOSE; 44 #else 45 #define DPRINTF(m, f...) 46 #endif 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disk.h> 52 #include <sys/disklabel.h> 53 54 #include <scsi/scsi_all.h> 55 #include <scsi/scsiconf.h> 56 57 void ahci_hibernate_io_start(struct ahci_port *, 58 struct ahci_ccb *); 59 int ahci_hibernate_io_poll(struct ahci_port *, 60 struct ahci_ccb *); 61 void ahci_hibernate_load_prdt(struct ahci_ccb *); 62 63 int ahci_hibernate_io(dev_t dev, daddr_t blkno, 64 vaddr_t addr, size_t size, int wr, void *page); 65 #endif 66 67 struct cfdriver ahci_cd = { 68 NULL, "ahci", DV_DULL 69 }; 70 71 void ahci_enable_interrupts(struct ahci_port *); 72 73 int ahci_init(struct ahci_softc *); 74 int ahci_port_alloc(struct ahci_softc *, u_int); 75 void ahci_port_detect(struct ahci_softc *, u_int); 76 void ahci_port_free(struct ahci_softc *, u_int); 77 int ahci_port_init(struct ahci_softc *, u_int); 78 79 int ahci_default_port_start(struct ahci_port *, int); 80 int ahci_port_stop(struct ahci_port *, int); 81 int ahci_port_clo(struct ahci_port *); 82 int ahci_port_softreset(struct ahci_port *); 83 void ahci_port_comreset(struct ahci_port *); 84 int ahci_port_portreset(struct ahci_port *, int); 85 void ahci_port_portreset_start(struct ahci_port *); 86 int ahci_port_portreset_poll(struct ahci_port *); 87 void ahci_port_portreset_wait(struct ahci_port *); 88 int ahci_port_portreset_finish(struct ahci_port *, int); 89 int ahci_port_signature(struct ahci_port *); 90 int ahci_pmp_port_softreset(struct ahci_port *, int); 91 int ahci_pmp_port_portreset(struct ahci_port *, int); 92 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 93 94 int ahci_load_prdt(struct ahci_ccb *); 95 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 96 u_int32_t, u_int32_t); 97 void ahci_unload_prdt(struct ahci_ccb *); 98 99 int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 100 void ahci_start(struct ahci_ccb *); 101 102 void ahci_issue_pending_ncq_commands(struct ahci_port *); 103 void ahci_issue_pending_commands(struct ahci_port *, int); 104 105 int ahci_intr(void *); 106 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 107 108 struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 109 void ahci_put_ccb(struct ahci_ccb *); 110 111 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 112 void ahci_put_err_ccb(struct ahci_ccb *); 113 114 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 115 void ahci_put_pmp_ccb(struct ahci_ccb *); 116 117 int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 118 119 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 120 void ahci_dmamem_free(struct ahci_softc *, 121 struct ahci_dmamem *); 122 123 u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 124 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 125 int ahci_wait_ne(struct ahci_softc *, bus_size_t, 126 u_int32_t, u_int32_t); 127 128 u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 129 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 130 int ahci_pwait_eq(struct ahci_port *, bus_size_t, 131 u_int32_t, u_int32_t, int); 132 void ahci_flush_tfd(struct ahci_port *ap); 133 u_int32_t ahci_active_mask(struct ahci_port *); 134 int ahci_port_detect_pmp(struct ahci_port *); 135 void ahci_pmp_probe_timeout(void *); 136 137 /* pmp operations */ 138 int ahci_pmp_read(struct ahci_port *, int, int, 139 u_int32_t *); 140 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 141 int ahci_pmp_phy_status(struct ahci_port *, int, 142 u_int32_t *); 143 int ahci_pmp_identify(struct ahci_port *, int *); 144 145 146 /* Wait for all bits in _b to be cleared */ 147 #define ahci_pwait_clr(_ap, _r, _b, _n) \ 148 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 149 150 /* Wait for all bits in _b to be set */ 151 #define ahci_pwait_set(_ap, _r, _b, _n) \ 152 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 153 154 155 156 /* provide methods for atascsi to call */ 157 int ahci_ata_probe(void *, int, int); 158 void ahci_ata_free(void *, int, int); 159 struct ata_xfer * ahci_ata_get_xfer(void *, int); 160 void ahci_ata_put_xfer(struct ata_xfer *); 161 void ahci_ata_cmd(struct ata_xfer *); 162 163 const struct atascsi_methods ahci_atascsi_methods = { 164 ahci_ata_probe, 165 ahci_ata_free, 166 ahci_ata_get_xfer, 167 ahci_ata_put_xfer, 168 ahci_ata_cmd 169 }; 170 171 /* ccb completions */ 172 void ahci_ata_cmd_done(struct ahci_ccb *); 173 void ahci_pmp_cmd_done(struct ahci_ccb *); 174 void ahci_ata_cmd_timeout(void *); 175 void ahci_empty_done(struct ahci_ccb *); 176 177 int 178 ahci_attach(struct ahci_softc *sc) 179 { 180 struct atascsi_attach_args aaa; 181 u_int32_t pi; 182 int i, j, done; 183 184 if (sc->sc_port_start == NULL) 185 sc->sc_port_start = ahci_default_port_start; 186 187 if (ahci_init(sc) != 0) { 188 /* error already printed by ahci_init */ 189 goto unmap; 190 } 191 192 printf("\n"); 193 194 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 195 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 196 #ifdef AHCI_DEBUG 197 if (ahcidebug & AHCI_D_VERBOSE) { 198 const char *gen; 199 200 switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 201 case AHCI_REG_CAP_ISS_G1: 202 gen = "1 (1.5Gbps)"; 203 break; 204 case AHCI_REG_CAP_ISS_G2: 205 gen = "2 (3.0Gb/s)"; 206 break; 207 case AHCI_REG_CAP_ISS_G3: 208 gen = "3 (6.0Gb/s)"; 209 break; 210 default: 211 gen = "unknown"; 212 break; 213 } 214 215 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 216 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 217 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 218 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc), 219 ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2); 220 } 221 #endif 222 223 pi = ahci_read(sc, AHCI_REG_PI); 224 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 225 DEVNAME(sc), pi); 226 227 #ifdef AHCI_COALESCE 228 /* Naive coalescing support - enable for all ports. */ 229 if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 230 u_int16_t ccc_timeout = 20; 231 u_int8_t ccc_numcomplete = 12; 232 u_int32_t ccc_ctl; 233 234 /* disable coalescing during reconfiguration. */ 235 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 236 ccc_ctl &= ~0x00000001; 237 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 238 239 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 240 if (pi & sc->sc_ccc_mask) { 241 /* A conflict with the implemented port list? */ 242 printf("%s: coalescing interrupt/implemented port list " 243 "conflict, PI: %08x, ccc_mask: %08x\n", 244 DEVNAME(sc), pi, sc->sc_ccc_mask); 245 sc->sc_ccc_mask = 0; 246 goto noccc; 247 } 248 249 /* ahci_port_start will enable each port when it starts. */ 250 sc->sc_ccc_ports = pi; 251 sc->sc_ccc_ports_cur = 0; 252 253 /* program thresholds and enable overall coalescing. */ 254 ccc_ctl &= ~0xffffff00; 255 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 256 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 257 ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 258 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 259 } 260 noccc: 261 #endif 262 /* 263 * Given that ahci_port_alloc() will grab one CCB for error recovery 264 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds 265 * pretend at least 2 command slots for devices without NCQ support. 266 * That way, also at least 1 slot is made available for atascsi(4). 267 */ 268 sc->sc_ncmds = max(2, sc->sc_ncmds); 269 for (i = 0; i < AHCI_MAX_PORTS; i++) { 270 if (!ISSET(pi, 1U << i)) { 271 /* don't allocate stuff if the port isn't implemented */ 272 continue; 273 } 274 275 if (ahci_port_alloc(sc, i) == ENOMEM) 276 goto freeports; 277 278 if (sc->sc_ports[i] != NULL) 279 ahci_port_portreset_start(sc->sc_ports[i]); 280 } 281 282 /* 283 * Poll for device detection until all ports report a device, or one 284 * second has elapsed. 285 */ 286 for (i = 0; i < 1000; i++) { 287 done = 1; 288 for (j = 0; j < AHCI_MAX_PORTS; j++) { 289 if (sc->sc_ports[j] == NULL) 290 continue; 291 292 if (ahci_port_portreset_poll(sc->sc_ports[j])) 293 done = 0; 294 } 295 296 if (done) 297 break; 298 299 delay(1000); 300 } 301 302 /* 303 * Finish device detection on all ports that initialized. 304 */ 305 for (i = 0; i < AHCI_MAX_PORTS; i++) { 306 if (sc->sc_ports[i] != NULL) 307 ahci_port_detect(sc, i); 308 } 309 310 memset(&aaa, 0, sizeof(aaa)); 311 aaa.aaa_cookie = sc; 312 aaa.aaa_methods = &ahci_atascsi_methods; 313 aaa.aaa_minphys = NULL; 314 aaa.aaa_nports = AHCI_MAX_PORTS; 315 aaa.aaa_ncmds = sc->sc_ncmds - 1; 316 if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 317 sc->sc_ncmds > 2 && 318 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 319 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 320 } 321 322 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 323 324 /* Flush all residual bits of the interrupt status register */ 325 ahci_write(sc, AHCI_REG_IS, ahci_read(sc, AHCI_REG_IS)); 326 327 /* Enable interrupts */ 328 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 329 330 return 0; 331 332 freeports: 333 for (i = 0; i < AHCI_MAX_PORTS; i++) { 334 if (sc->sc_ports[i] != NULL) 335 ahci_port_free(sc, i); 336 } 337 unmap: 338 /* Disable controller */ 339 ahci_write(sc, AHCI_REG_GHC, 0); 340 return 1; 341 } 342 343 int 344 ahci_detach(struct ahci_softc *sc, int flags) 345 { 346 int rv, i; 347 348 if (sc->sc_atascsi != NULL) { 349 rv = atascsi_detach(sc->sc_atascsi, flags); 350 if (rv != 0) 351 return (rv); 352 } 353 354 for (i = 0; i < AHCI_MAX_PORTS; i++) { 355 if (sc->sc_ports[i] != NULL) 356 ahci_port_free(sc, i); 357 } 358 359 return (0); 360 } 361 362 int 363 ahci_activate(struct device *self, int act) 364 { 365 struct ahci_softc *sc = (struct ahci_softc *)self; 366 int i, rv = 0; 367 368 switch (act) { 369 case DVACT_RESUME: 370 /* enable ahci (global interrupts disabled) */ 371 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 372 373 /* restore BIOS initialised parameters */ 374 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 375 376 for (i = 0; i < AHCI_MAX_PORTS; i++) { 377 if (sc->sc_ports[i] != NULL) 378 ahci_port_init(sc, i); 379 } 380 381 /* Enable interrupts */ 382 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 383 384 rv = config_activate_children(self, act); 385 break; 386 case DVACT_POWERDOWN: 387 rv = config_activate_children(self, act); 388 for (i = 0; i < AHCI_MAX_PORTS; i++) { 389 if (sc->sc_ports[i] != NULL) 390 ahci_port_stop(sc->sc_ports[i], 1); 391 } 392 break; 393 default: 394 rv = config_activate_children(self, act); 395 break; 396 } 397 return (rv); 398 } 399 400 int 401 ahci_init(struct ahci_softc *sc) 402 { 403 u_int32_t reg, cap, pi; 404 const char *revision; 405 406 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 407 AHCI_FMT_GHC); 408 409 /* save BIOS initialised parameters, enable staggered spin up */ 410 cap = ahci_read(sc, AHCI_REG_CAP); 411 cap &= AHCI_REG_CAP_SMPS; 412 cap |= AHCI_REG_CAP_SSS; 413 pi = ahci_read(sc, AHCI_REG_PI); 414 415 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 416 /* reset the controller */ 417 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 418 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 419 AHCI_REG_GHC_HR) != 0) { 420 printf(" unable to reset controller\n"); 421 return (1); 422 } 423 } 424 425 /* enable ahci (global interrupts disabled) */ 426 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 427 428 /* restore parameters */ 429 ahci_write(sc, AHCI_REG_CAP, cap); 430 ahci_write(sc, AHCI_REG_PI, pi); 431 432 /* check the revision */ 433 reg = ahci_read(sc, AHCI_REG_VS); 434 switch (reg) { 435 case AHCI_REG_VS_0_95: 436 revision = "0.95"; 437 break; 438 case AHCI_REG_VS_1_0: 439 revision = "1.0"; 440 break; 441 case AHCI_REG_VS_1_1: 442 revision = "1.1"; 443 break; 444 case AHCI_REG_VS_1_2: 445 revision = "1.2"; 446 break; 447 case AHCI_REG_VS_1_3: 448 revision = "1.3"; 449 break; 450 case AHCI_REG_VS_1_3_1: 451 revision = "1.3.1"; 452 break; 453 454 default: 455 printf(" unsupported AHCI revision 0x%08x\n", reg); 456 return (1); 457 } 458 459 printf(" AHCI %s", revision); 460 461 return (0); 462 } 463 464 void 465 ahci_enable_interrupts(struct ahci_port *ap) 466 { 467 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 468 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 469 AHCI_PREG_IE_UFE | 470 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 471 #ifdef AHCI_COALESCE 472 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 473 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 474 #else 475 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 476 #endif 477 ); 478 } 479 480 int 481 ahci_port_alloc(struct ahci_softc *sc, u_int port) 482 { 483 struct ahci_port *ap; 484 struct ahci_ccb *ccb; 485 u_int64_t dva; 486 u_int32_t cmd; 487 struct ahci_cmd_hdr *hdr; 488 struct ahci_cmd_table *table; 489 int i, rc = ENOMEM; 490 491 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 492 if (ap == NULL) { 493 printf("%s: unable to allocate memory for port %d\n", 494 DEVNAME(sc), port); 495 goto reterr; 496 } 497 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 498 if (ap->ap_err_scratch == NULL) { 499 printf("%s: unable to allocate DMA scratch buf for port %d\n", 500 DEVNAME(sc), port); 501 free(ap, M_DEVBUF, sizeof(*ap)); 502 goto reterr; 503 } 504 505 #ifdef AHCI_DEBUG 506 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 507 DEVNAME(sc), port); 508 #endif 509 ap->ap_port = port; 510 sc->sc_ports[port] = ap; 511 512 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 513 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 514 printf("%s: unable to create register window for port %d\n", 515 DEVNAME(sc), port); 516 goto freeport; 517 } 518 519 ap->ap_sc = sc; 520 #ifdef AHCI_COALESCE 521 ap->ap_num = port; 522 #endif 523 TAILQ_INIT(&ap->ap_ccb_free); 524 TAILQ_INIT(&ap->ap_ccb_pending); 525 mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 526 527 /* Disable port interrupts */ 528 ahci_pwrite(ap, AHCI_PREG_IE, 0); 529 530 /* Sec 10.1.2 - deinitialise port if it is already running */ 531 cmd = ahci_pread(ap, AHCI_PREG_CMD); 532 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 533 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 534 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 535 int r; 536 537 r = ahci_port_stop(ap, 1); 538 if (r) { 539 printf("%s: unable to disable %s, ignoring port %d\n", 540 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 541 rc = ENXIO; 542 goto freeport; 543 } 544 545 /* Write DET to zero */ 546 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 547 } 548 549 /* Allocate RFIS */ 550 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 551 if (ap->ap_dmamem_rfis == NULL) 552 goto nomem; 553 554 /* Setup RFIS base address */ 555 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 556 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 557 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 558 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 559 560 /* Enable FIS reception and activate port. */ 561 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 562 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 563 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 564 565 /* Check whether port activated. Skip it if not. */ 566 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 567 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 568 rc = ENXIO; 569 goto freeport; 570 } 571 572 /* Allocate a CCB for each command slot */ 573 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 574 M_DEVBUF, M_NOWAIT | M_ZERO); 575 if (ap->ap_ccbs == NULL) { 576 printf("%s: unable to allocate command list for port %d\n", 577 DEVNAME(sc), port); 578 goto freeport; 579 } 580 581 /* Command List Structures and Command Tables */ 582 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 583 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 584 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 585 sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 586 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 587 nomem: 588 printf("%s: unable to allocate DMA memory for port %d\n", 589 DEVNAME(sc), port); 590 goto freeport; 591 } 592 593 /* Setup command list base address */ 594 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 595 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 596 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 597 598 /* Split CCB allocation into CCBs and assign to command header/table */ 599 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 600 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 601 for (i = 0; i < sc->sc_ncmds; i++) { 602 ccb = &ap->ap_ccbs[i]; 603 604 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 605 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 606 &ccb->ccb_dmamap) != 0) { 607 printf("%s: unable to create dmamap for port %d " 608 "ccb %d\n", DEVNAME(sc), port, i); 609 goto freeport; 610 } 611 612 ccb->ccb_slot = i; 613 ccb->ccb_port = ap; 614 ccb->ccb_cmd_hdr = &hdr[i]; 615 ccb->ccb_cmd_table = &table[i]; 616 htolem64(&ccb->ccb_cmd_hdr->ctba, 617 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 618 ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 619 620 ccb->ccb_xa.fis = 621 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 622 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 623 ccb->ccb_xa.tag = i; 624 625 ccb->ccb_xa.state = ATA_S_COMPLETE; 626 ahci_put_ccb(ccb); 627 } 628 629 /* grab a ccb for use during error recovery */ 630 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 631 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 632 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 633 634 /* Wait for ICC change to complete */ 635 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 636 rc = 0; 637 638 freeport: 639 if (rc != 0) 640 ahci_port_free(sc, port); 641 reterr: 642 return (rc); 643 } 644 645 void 646 ahci_port_detect(struct ahci_softc *sc, u_int port) 647 { 648 struct ahci_port *ap; 649 const char *speed; 650 int rc; 651 652 ap = sc->sc_ports[port]; 653 654 rc = ahci_port_portreset_finish(ap, 1); 655 switch (rc) { 656 case ENODEV: 657 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 658 case AHCI_PREG_SSTS_DET_DEV_NE: 659 printf("%s: device not communicating on port %d\n", 660 DEVNAME(sc), port); 661 break; 662 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 663 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 664 port); 665 break; 666 default: 667 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 668 "on port %d\n", DEVNAME(sc), port); 669 break; 670 } 671 goto freeport; 672 673 case EBUSY: 674 printf("%s: device on port %d didn't come ready, " 675 "TFD: 0x%b\n", DEVNAME(sc), port, 676 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 677 678 /* Try a soft reset to clear busy */ 679 rc = ahci_port_softreset(ap); 680 if (rc) { 681 printf("%s: unable to communicate " 682 "with device on port %d\n", DEVNAME(sc), port); 683 goto freeport; 684 } 685 break; 686 687 default: 688 break; 689 } 690 691 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 692 DEVNAME(sc), port, rc); 693 694 /* Read current link speed */ 695 switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { 696 case AHCI_PREG_SSTS_SPD_GEN1: 697 speed = "1.5Gb/s"; 698 break; 699 case AHCI_PREG_SSTS_SPD_GEN2: 700 speed = "3.0Gb/s"; 701 break; 702 case AHCI_PREG_SSTS_SPD_GEN3: 703 speed = "6.0Gb/s"; 704 break; 705 default: 706 speed = NULL; 707 break; 708 } 709 if (speed != NULL) 710 printf("%s: port %d: %s\n", PORTNAME(ap), port, speed); 711 712 /* Enable command transfers on port */ 713 if (ahci_port_start(ap, 0)) { 714 printf("%s: failed to start command DMA on port %d, " 715 "disabling\n", DEVNAME(sc), port); 716 rc = ENXIO; /* couldn't start port */ 717 } 718 719 /* Flush interrupts for port */ 720 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 721 ahci_write(sc, AHCI_REG_IS, 1 << port); 722 723 ahci_enable_interrupts(ap); 724 freeport: 725 if (rc != 0) 726 ahci_port_free(sc, port); 727 } 728 729 void 730 ahci_port_free(struct ahci_softc *sc, u_int port) 731 { 732 struct ahci_port *ap = sc->sc_ports[port]; 733 struct ahci_ccb *ccb; 734 735 /* Ensure port is disabled and its interrupts are flushed */ 736 if (ap->ap_sc) { 737 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 738 ahci_pwrite(ap, AHCI_PREG_IE, 0); 739 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 740 ahci_write(sc, AHCI_REG_IS, 1 << port); 741 } 742 743 if (ap->ap_ccb_err) 744 ahci_put_ccb(ap->ap_ccb_err); 745 746 if (ap->ap_ccbs) { 747 while ((ccb = ahci_get_ccb(ap)) != NULL) 748 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 749 free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb)); 750 } 751 752 if (ap->ap_dmamem_cmd_list) 753 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 754 if (ap->ap_dmamem_rfis) 755 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 756 if (ap->ap_dmamem_cmd_table) 757 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 758 if (ap->ap_err_scratch) 759 dma_free(ap->ap_err_scratch, DEV_BSIZE); 760 761 /* bus_space(9) says we dont free the subregions handle */ 762 763 free(ap, M_DEVBUF, sizeof(*ap)); 764 sc->sc_ports[port] = NULL; 765 } 766 767 int 768 ahci_port_init(struct ahci_softc *sc, u_int port) 769 { 770 struct ahci_port *ap; 771 u_int64_t dva; 772 u_int32_t cmd; 773 int rc = ENOMEM; 774 775 ap = sc->sc_ports[port]; 776 #ifdef AHCI_DEBUG 777 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 778 DEVNAME(sc), port); 779 #endif 780 781 /* Disable port interrupts */ 782 ahci_pwrite(ap, AHCI_PREG_IE, 0); 783 784 /* Sec 10.1.2 - deinitialise port if it is already running */ 785 cmd = ahci_pread(ap, AHCI_PREG_CMD); 786 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 787 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 788 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 789 int r; 790 791 r = ahci_port_stop(ap, 1); 792 if (r) { 793 printf("%s: unable to disable %s, ignoring port %d\n", 794 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 795 rc = ENXIO; 796 goto reterr; 797 } 798 799 /* Write DET to zero */ 800 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 801 } 802 803 /* Setup RFIS base address */ 804 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 805 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 806 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 807 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 808 809 /* Enable FIS reception and activate port. */ 810 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 811 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 812 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 813 814 /* Check whether port activated. Skip it if not. */ 815 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 816 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 817 rc = ENXIO; 818 goto reterr; 819 } 820 821 /* Setup command list base address */ 822 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 823 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 824 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 825 826 /* Wait for ICC change to complete */ 827 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 828 829 /* Reset port */ 830 rc = ahci_port_portreset(ap, 1); 831 switch (rc) { 832 case ENODEV: 833 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 834 case AHCI_PREG_SSTS_DET_DEV_NE: 835 printf("%s: device not communicating on port %d\n", 836 DEVNAME(sc), port); 837 break; 838 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 839 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 840 port); 841 break; 842 default: 843 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 844 "on port %d\n", DEVNAME(sc), port); 845 break; 846 } 847 goto reterr; 848 849 case EBUSY: 850 printf("%s: device on port %d didn't come ready, " 851 "TFD: 0x%b\n", DEVNAME(sc), port, 852 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 853 854 /* Try a soft reset to clear busy */ 855 rc = ahci_port_softreset(ap); 856 if (rc) { 857 printf("%s: unable to communicate " 858 "with device on port %d\n", DEVNAME(sc), port); 859 goto reterr; 860 } 861 break; 862 863 default: 864 break; 865 } 866 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 867 DEVNAME(sc), port); 868 869 if (ap->ap_pmp_ports > 0) { 870 int p; 871 872 for (p = 0; p < ap->ap_pmp_ports; p++) { 873 int sig; 874 875 /* might need to do a portreset first here? */ 876 877 /* softreset the port */ 878 if (ahci_pmp_port_softreset(ap, p)) { 879 printf("%s.%d: unable to probe PMP port due to" 880 " softreset failure\n", PORTNAME(ap), p); 881 continue; 882 } 883 884 sig = ahci_port_signature(ap); 885 printf("%s.%d: port signature returned %d\n", 886 PORTNAME(ap), p, sig); 887 } 888 } 889 890 /* Enable command transfers on port */ 891 if (ahci_port_start(ap, 0)) { 892 printf("%s: failed to start command DMA on port %d, " 893 "disabling\n", DEVNAME(sc), port); 894 rc = ENXIO; /* couldn't start port */ 895 } 896 897 /* Flush interrupts for port */ 898 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 899 ahci_write(sc, AHCI_REG_IS, 1 << port); 900 901 ahci_enable_interrupts(ap); 902 903 reterr: 904 return (rc); 905 } 906 907 int 908 ahci_default_port_start(struct ahci_port *ap, int fre_only) 909 { 910 u_int32_t r; 911 912 /* Turn on FRE (and ST) */ 913 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 914 r |= AHCI_PREG_CMD_FRE; 915 if (!fre_only) 916 r |= AHCI_PREG_CMD_ST; 917 ahci_pwrite(ap, AHCI_PREG_CMD, r); 918 919 #ifdef AHCI_COALESCE 920 /* (Re-)enable coalescing on the port. */ 921 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 922 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 923 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 924 ap->ap_sc->sc_ccc_ports_cur); 925 } 926 #endif 927 928 return (0); 929 } 930 931 int 932 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 933 { 934 u_int32_t r; 935 936 #ifdef AHCI_COALESCE 937 /* Disable coalescing on the port while it is stopped. */ 938 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 939 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 940 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 941 ap->ap_sc->sc_ccc_ports_cur); 942 } 943 #endif 944 945 /* Turn off ST (and FRE) */ 946 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 947 r &= ~AHCI_PREG_CMD_ST; 948 if (stop_fis_rx) 949 r &= ~AHCI_PREG_CMD_FRE; 950 ahci_pwrite(ap, AHCI_PREG_CMD, r); 951 952 /* Wait for CR to go off */ 953 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 954 return (1); 955 956 /* Wait for FR to go off */ 957 if (stop_fis_rx && 958 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 959 return (2); 960 961 return (0); 962 } 963 964 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 965 int 966 ahci_port_clo(struct ahci_port *ap) 967 { 968 struct ahci_softc *sc = ap->ap_sc; 969 u_int32_t cmd; 970 971 /* Only attempt CLO if supported by controller */ 972 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 973 return (1); 974 975 /* Issue CLO */ 976 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 977 #ifdef DIAGNOSTIC 978 if (ISSET(cmd, AHCI_PREG_CMD_ST)) 979 printf("%s: CLO requested while port running\n", PORTNAME(ap)); 980 #endif 981 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 982 983 /* Wait for completion */ 984 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 985 printf("%s: CLO did not complete\n", PORTNAME(ap)); 986 return (1); 987 } 988 989 return (0); 990 } 991 992 /* AHCI soft reset, Section 10.4.1 */ 993 int 994 ahci_port_softreset(struct ahci_port *ap) 995 { 996 struct ahci_ccb *ccb = NULL; 997 struct ahci_cmd_hdr *cmd_slot; 998 u_int8_t *fis; 999 int s, rc = EIO, oldstate; 1000 u_int32_t cmd; 1001 1002 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 1003 1004 s = splbio(); 1005 oldstate = ap->ap_state; 1006 ap->ap_state = AP_S_ERROR_RECOVERY; 1007 1008 /* Save previous command register state */ 1009 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1010 1011 /* Idle port */ 1012 if (ahci_port_stop(ap, 0)) { 1013 printf("%s: failed to stop port, cannot softreset\n", 1014 PORTNAME(ap)); 1015 goto err; 1016 } 1017 1018 /* Request CLO if device appears hung */ 1019 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 1020 AHCI_PREG_TFD_STS_DRQ)) 1021 ahci_port_clo(ap); 1022 1023 /* Clear port errors to permit TFD transfer */ 1024 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1025 1026 /* Restart port */ 1027 if (ahci_port_start(ap, 0)) { 1028 printf("%s: failed to start port, cannot softreset\n", 1029 PORTNAME(ap)); 1030 goto err; 1031 } 1032 1033 /* Check whether CLO worked */ 1034 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1035 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1036 printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 1037 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 1038 ? "failed" : "unsupported"); 1039 rc = EBUSY; 1040 goto err; 1041 } 1042 1043 /* Prep first D2H command with SRST feature & clear busy/reset flags */ 1044 ccb = ahci_get_err_ccb(ap); 1045 cmd_slot = ccb->ccb_cmd_hdr; 1046 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1047 1048 fis = ccb->ccb_cmd_table->cfis; 1049 fis[0] = ATA_FIS_TYPE_H2D; 1050 fis[15] = ATA_FIS_CONTROL_SRST; 1051 1052 cmd_slot->prdtl = 0; 1053 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1054 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1055 AHCI_CMD_LIST_FLAG_W); 1056 1057 ccb->ccb_xa.state = ATA_S_PENDING; 1058 if (ahci_poll(ccb, 1000, NULL) != 0) 1059 goto err; 1060 1061 /* Prep second D2H command to read status and complete reset sequence */ 1062 fis[0] = ATA_FIS_TYPE_H2D; 1063 fis[15] = 0; 1064 1065 cmd_slot->prdtl = 0; 1066 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 1067 1068 ccb->ccb_xa.state = ATA_S_PENDING; 1069 if (ahci_poll(ccb, 1000, NULL) != 0) 1070 goto err; 1071 1072 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1073 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 1074 printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1075 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 1076 AHCI_PFMT_TFD_STS); 1077 rc = EBUSY; 1078 goto err; 1079 } 1080 1081 rc = 0; 1082 err: 1083 if (ccb != NULL) { 1084 /* Abort our command, if it failed, by stopping command DMA. */ 1085 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 1086 printf("%s: stopping the port, softreset slot %d was " 1087 "still active.\n", PORTNAME(ap), ccb->ccb_slot); 1088 ahci_port_stop(ap, 0); 1089 } 1090 ccb->ccb_xa.state = ATA_S_ERROR; 1091 ahci_put_err_ccb(ccb); 1092 } 1093 1094 /* Restore saved CMD register state */ 1095 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1096 ap->ap_state = oldstate; 1097 1098 splx(s); 1099 1100 return (rc); 1101 } 1102 1103 int 1104 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 1105 { 1106 struct ahci_ccb *ccb = NULL; 1107 u_int32_t data; 1108 int count; 1109 int rc; 1110 int s; 1111 struct ahci_cmd_hdr *cmd_slot; 1112 u_int8_t *fis; 1113 1114 s = splbio(); 1115 /* ignore spurious IFS errors while resetting */ 1116 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 1117 ap->ap_pmp_ignore_ifs = 1; 1118 1119 count = 2; 1120 rc = 0; 1121 do { 1122 if (ccb != NULL) { 1123 ahci_put_pmp_ccb(ccb); 1124 ccb = NULL; 1125 } 1126 1127 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1128 printf("%s.%d: unable to clear PHY status\n", 1129 PORTNAME(ap), pmp_port); 1130 } 1131 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1132 /* maybe don't do this on the first loop: */ 1133 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1134 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1135 1136 /* send first softreset FIS */ 1137 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 1138 cmd_slot = ccb->ccb_cmd_hdr; 1139 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1140 1141 fis = ccb->ccb_cmd_table->cfis; 1142 fis[0] = ATA_FIS_TYPE_H2D; 1143 fis[1] = pmp_port; 1144 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1145 1146 cmd_slot->prdtl = 0; 1147 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1148 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1149 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1150 1151 ccb->ccb_xa.state = ATA_S_PENDING; 1152 1153 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 1154 PORTNAME(ap), pmp_port); 1155 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1156 printf("%s.%d: PMP port softreset cmd failed\n", 1157 PORTNAME(ap), pmp_port); 1158 rc = EBUSY; 1159 if (count > 0) { 1160 /* probably delay a while to allow 1161 * it to settle down? 1162 */ 1163 } 1164 continue; 1165 } 1166 1167 /* send signature FIS */ 1168 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1169 fis[0] = ATA_FIS_TYPE_H2D; 1170 fis[1] = pmp_port; 1171 fis[15] = ATA_FIS_CONTROL_4BIT; 1172 1173 cmd_slot->prdtl = 0; 1174 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1175 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1176 1177 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 1178 PORTNAME(ap), pmp_port); 1179 ccb->ccb_xa.state = ATA_S_PENDING; 1180 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1181 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 1182 "failed\n", PORTNAME(ap), pmp_port); 1183 rc = EBUSY; 1184 if (count > 0) { 1185 /* sleep a while? */ 1186 } 1187 continue; 1188 } 1189 1190 fis[15] = 0; 1191 break; 1192 } while (count--); 1193 1194 if (ccb != NULL) { 1195 ahci_put_pmp_ccb(ccb); 1196 ccb = NULL; 1197 } 1198 1199 /* clean up a bit */ 1200 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1201 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1202 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1203 ap->ap_pmp_ignore_ifs = 0; 1204 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 1205 splx(s); 1206 1207 return (rc); 1208 } 1209 1210 int 1211 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 1212 { 1213 int sig; 1214 1215 ap->ap_state = AP_S_PMP_PORT_PROBE; 1216 1217 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 1218 pmp_port); 1219 if (ahci_pmp_port_portreset(ap, pmp_port)) { 1220 printf("%s.%d: unable to probe PMP port; portreset failed\n", 1221 PORTNAME(ap), pmp_port); 1222 ap->ap_state = AP_S_NORMAL; 1223 return (ATA_PORT_T_NONE); 1224 } 1225 1226 if (ahci_pmp_port_softreset(ap, pmp_port)) { 1227 printf("%s.%d: unable to probe PMP port due to softreset " 1228 "failure\n", PORTNAME(ap), pmp_port); 1229 ap->ap_state = AP_S_NORMAL; 1230 return (ATA_PORT_T_NONE); 1231 } 1232 1233 sig = ahci_port_signature(ap); 1234 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 1235 PORTNAME(ap), pmp_port, sig); 1236 ap->ap_state = AP_S_NORMAL; 1237 return (sig); 1238 } 1239 1240 1241 void 1242 ahci_flush_tfd(struct ahci_port *ap) 1243 { 1244 u_int32_t r; 1245 1246 r = ahci_pread(ap, AHCI_PREG_SERR); 1247 if (r & AHCI_PREG_SERR_DIAG_X) 1248 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1249 } 1250 1251 u_int32_t 1252 ahci_active_mask(struct ahci_port *ap) 1253 { 1254 u_int32_t mask; 1255 1256 mask = ahci_pread(ap, AHCI_PREG_CI); 1257 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 1258 mask |= ahci_pread(ap, AHCI_PREG_SACT); 1259 return mask; 1260 } 1261 1262 void 1263 ahci_pmp_probe_timeout(void *cookie) 1264 { 1265 struct ahci_ccb *ccb = cookie; 1266 struct ahci_port *ap = ccb->ccb_port; 1267 u_int32_t mask; 1268 1269 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 1270 switch (ccb->ccb_xa.state) { 1271 case ATA_S_PENDING: 1272 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1273 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1274 break; 1275 1276 case ATA_S_ONCHIP: 1277 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 1278 /* clear the command on-chip */ 1279 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 1280 ap->ap_sactive == 0); 1281 ahci_port_stop(ap, 0); 1282 ahci_port_start(ap, 0); 1283 1284 if (ahci_active_mask(ap) != 0) { 1285 ahci_port_stop(ap, 0); 1286 ahci_port_start(ap, 0); 1287 mask = ahci_active_mask(ap); 1288 if (mask != 0) { 1289 printf("%s: ahci_pmp_probe_timeout: failed to " 1290 "clear active cmds: %08x\n", PORTNAME(ap), 1291 mask); 1292 } 1293 } 1294 1295 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1296 ap->ap_active &= ~(1 << ccb->ccb_slot); 1297 KASSERT(ap->ap_active_cnt > 0); 1298 --ap->ap_active_cnt; 1299 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, " 1300 "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot, 1301 ap->ap_active, ap->ap_active_cnt); 1302 break; 1303 1304 default: 1305 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 1306 PORTNAME(ap), ccb->ccb_xa.state); 1307 } 1308 } 1309 1310 int 1311 ahci_port_signature(struct ahci_port *ap) 1312 { 1313 u_int32_t sig; 1314 1315 sig = ahci_pread(ap, AHCI_PREG_SIG); 1316 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 1317 return (ATA_PORT_T_ATAPI); 1318 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 1319 0xffff0000)) 1320 return (ATA_PORT_T_PM); 1321 else 1322 return (ATA_PORT_T_DISK); 1323 } 1324 1325 int 1326 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 1327 { 1328 u_int32_t cmd, data; 1329 int loop; 1330 int rc = 1; 1331 int s; 1332 1333 s = splbio(); 1334 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 1335 pmp_port); 1336 1337 /* Save previous command register state */ 1338 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1339 1340 /* turn off power management and disable the PHY */ 1341 data = AHCI_PREG_SCTL_IPM_DISABLED; 1342 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 1343 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 1344 goto err; 1345 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1346 goto err; 1347 delay(10000); 1348 1349 /* start COMRESET */ 1350 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1351 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1352 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 1353 pmp_port); 1354 data |= AHCI_PREG_SCTL_SPD_GEN1; 1355 } else 1356 data |= AHCI_PREG_SCTL_SPD_ANY; 1357 1358 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1359 goto err; 1360 1361 /* give it a while to settle down */ 1362 delay(100000); 1363 1364 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1365 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 1366 pmp_port); 1367 } 1368 1369 /* start trying to negotiate */ 1370 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1371 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 1372 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1373 goto err; 1374 1375 /* give it a while to detect */ 1376 for (loop = 3; loop; --loop) { 1377 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1378 goto err; 1379 if (data & AHCI_PREG_SSTS_DET) 1380 break; 1381 delay(100000); 1382 } 1383 if (loop == 0) { 1384 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 1385 goto err; 1386 } 1387 1388 /* give it even longer to fully negotiate */ 1389 for (loop = 30; loop; --loop) { 1390 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1391 goto err; 1392 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 1393 break; 1394 delay(100000); 1395 } 1396 1397 if (loop == 0) { 1398 printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 1399 pmp_port); 1400 goto err; 1401 } 1402 1403 /* device detected */ 1404 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 1405 pmp_port); 1406 1407 /* clean up a bit */ 1408 delay(100000); 1409 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1410 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1411 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1412 1413 rc = 0; 1414 err: 1415 /* Restore preserved port state */ 1416 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1417 splx(s); 1418 return (rc); 1419 } 1420 1421 /* AHCI port reset, Section 10.4.2 */ 1422 1423 void 1424 ahci_port_comreset(struct ahci_port *ap) 1425 { 1426 u_int32_t r; 1427 1428 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1429 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1430 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 1431 r |= AHCI_PREG_SCTL_SPD_GEN1; 1432 } else 1433 r |= AHCI_PREG_SCTL_SPD_ANY; 1434 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1435 delay(10000); /* wait at least 1ms for COMRESET to be sent */ 1436 r &= ~AHCI_PREG_SCTL_DET_INIT; 1437 r |= AHCI_PREG_SCTL_DET_NONE; 1438 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1439 delay(10000); 1440 } 1441 1442 void 1443 ahci_port_portreset_start(struct ahci_port *ap) 1444 { 1445 int s; 1446 1447 s = splbio(); 1448 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 1449 1450 /* Save previous command register state */ 1451 ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1452 1453 /* Clear ST, ignoring failure */ 1454 ahci_port_stop(ap, 0); 1455 1456 /* Perform device detection */ 1457 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 1458 delay(10000); 1459 ahci_port_comreset(ap); 1460 splx(s); 1461 } 1462 1463 int 1464 ahci_port_portreset_poll(struct ahci_port *ap) 1465 { 1466 if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) != 1467 AHCI_PREG_SSTS_DET_DEV) 1468 return (EAGAIN); 1469 return (0); 1470 } 1471 1472 void 1473 ahci_port_portreset_wait(struct ahci_port *ap) 1474 { 1475 int i; 1476 1477 for (i = 0; i < 1000; i++) { 1478 if (ahci_port_portreset_poll(ap) == 0) 1479 break; 1480 delay(1000); 1481 } 1482 } 1483 1484 int 1485 ahci_port_portreset_finish(struct ahci_port *ap, int pmp) 1486 { 1487 int rc, s, retries = 0; 1488 1489 s = splbio(); 1490 retry: 1491 if (ahci_port_portreset_poll(ap)) { 1492 rc = ENODEV; 1493 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 1494 /* this may be a port multiplier with no device 1495 * on port 0, so still do the pmp check if requested. 1496 */ 1497 } else { 1498 goto err; 1499 } 1500 } else { 1501 /* Clear SERR (incl X bit), so TFD can update */ 1502 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1503 1504 /* Wait for device to become ready */ 1505 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1506 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 1507 /* even if the device doesn't wake up, check if there's 1508 * a port multiplier there 1509 */ 1510 if (retries == 0) { 1511 retries = 1; 1512 ahci_port_comreset(ap); 1513 ahci_port_portreset_wait(ap); 1514 goto retry; 1515 } 1516 rc = EBUSY; 1517 } else { 1518 rc = 0; 1519 } 1520 } 1521 1522 if (pmp != 0) { 1523 if (ahci_port_detect_pmp(ap)) { 1524 /* reset again without pmp support */ 1525 pmp = 0; 1526 retries = 0; 1527 ahci_port_comreset(ap); 1528 ahci_port_portreset_wait(ap); 1529 goto retry; 1530 } 1531 } 1532 1533 err: 1534 /* Restore preserved port state */ 1535 ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd); 1536 ap->ap_saved_cmd = 0; 1537 splx(s); 1538 1539 return (rc); 1540 } 1541 1542 int 1543 ahci_port_portreset(struct ahci_port *ap, int pmp) 1544 { 1545 ahci_port_portreset_start(ap); 1546 ahci_port_portreset_wait(ap); 1547 return (ahci_port_portreset_finish(ap, pmp)); 1548 } 1549 1550 int 1551 ahci_port_detect_pmp(struct ahci_port *ap) 1552 { 1553 int count, pmp_rc, rc; 1554 u_int32_t r, cmd; 1555 struct ahci_cmd_hdr *cmd_slot; 1556 struct ahci_ccb *ccb = NULL; 1557 u_int8_t *fis = NULL; 1558 1559 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 1560 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 1561 return 0; 1562 } 1563 1564 rc = 0; 1565 pmp_rc = 0; 1566 count = 2; 1567 do { 1568 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 1569 count); 1570 if (ccb != NULL) { 1571 ahci_put_pmp_ccb(ccb); 1572 ccb = NULL; 1573 } 1574 ahci_port_stop(ap, 0); 1575 ap->ap_state = AP_S_PMP_PROBE; 1576 1577 /* set PMA in cmd reg */ 1578 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1579 if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 1580 cmd |= AHCI_PREG_CMD_PMA; 1581 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1582 } 1583 1584 /* Flush errors and request CLO unconditionally, 1585 * then start the port 1586 */ 1587 r = ahci_pread(ap, AHCI_PREG_SERR); 1588 if (r & AHCI_PREG_SERR_DIAG_X) 1589 ahci_pwrite(ap, AHCI_PREG_SERR, 1590 AHCI_PREG_SERR_DIAG_X); 1591 1592 /* Request CLO */ 1593 ahci_port_clo(ap); 1594 1595 /* Clear port errors to permit TFD transfer */ 1596 r = ahci_pread(ap, AHCI_PREG_SERR); 1597 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1598 1599 /* Restart port */ 1600 if (ahci_port_start(ap, 0)) { 1601 rc = EBUSY; 1602 printf("%s: failed to start port, cannot probe PMP\n", 1603 PORTNAME(ap)); 1604 break; 1605 } 1606 1607 /* Check whether CLO worked */ 1608 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1609 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1610 u_int32_t cap; 1611 1612 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 1613 printf("%s: CLO %s, need port reset\n", 1614 PORTNAME(ap), 1615 ISSET(cap, AHCI_REG_CAP_SCLO) 1616 ? "failed" : "unsupported"); 1617 pmp_rc = EBUSY; 1618 break; 1619 } 1620 1621 /* Prep first command with SRST feature & 1622 * clear busy/reset flags 1623 */ 1624 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 1625 cmd_slot = ccb->ccb_cmd_hdr; 1626 memset(ccb->ccb_cmd_table, 0, 1627 sizeof(struct ahci_cmd_table)); 1628 1629 fis = ccb->ccb_cmd_table->cfis; 1630 fis[0] = ATA_FIS_TYPE_H2D; 1631 fis[1] = SATA_PMP_CONTROL_PORT; 1632 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1633 1634 cmd_slot->prdtl = 0; 1635 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1636 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1637 AHCI_CMD_LIST_FLAG_PMP); 1638 1639 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 1640 PORTNAME(ap)); 1641 ccb->ccb_xa.state = ATA_S_PENDING; 1642 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1643 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 1644 PORTNAME(ap)); 1645 pmp_rc = EBUSY; 1646 continue; 1647 } 1648 1649 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1650 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1651 printf("%s: port busy after first PMP probe FIS\n", 1652 PORTNAME(ap)); 1653 } 1654 1655 /* clear errors in case the device 1656 * didn't reset cleanly 1657 */ 1658 ahci_flush_tfd(ap); 1659 r = ahci_pread(ap, AHCI_PREG_SERR); 1660 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1661 1662 /* Prep second command to read status and 1663 * complete reset sequence 1664 */ 1665 memset(ccb->ccb_cmd_table, 0, 1666 sizeof(struct ahci_cmd_table)); 1667 fis[0] = ATA_FIS_TYPE_H2D; 1668 fis[1] = SATA_PMP_CONTROL_PORT; 1669 fis[15] = ATA_FIS_CONTROL_4BIT; 1670 1671 cmd_slot->prdtl = 0; 1672 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1673 AHCI_CMD_LIST_FLAG_PMP); 1674 1675 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 1676 PORTNAME(ap)); 1677 ccb->ccb_xa.state = ATA_S_PENDING; 1678 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1679 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 1680 "cmd failed\n", PORTNAME(ap)); 1681 pmp_rc = EBUSY; 1682 continue; 1683 } 1684 1685 /* apparently we need to retry at least once 1686 * to get the right signature 1687 */ 1688 fis[15] = 0; 1689 pmp_rc = 0; 1690 } while (--count); 1691 1692 if (ccb != NULL) { 1693 ahci_put_pmp_ccb(ccb); 1694 ccb = NULL; 1695 } 1696 1697 if (ap->ap_state == AP_S_PMP_PROBE) { 1698 ap->ap_state = AP_S_NORMAL; 1699 } 1700 1701 if (pmp_rc == 0) { 1702 if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 1703 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 1704 PORTNAME(ap)); 1705 pmp_rc = EBUSY; 1706 } else { 1707 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 1708 PORTNAME(ap)); 1709 } 1710 } 1711 1712 if (pmp_rc == 0) { 1713 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 1714 pmp_rc = EBUSY; 1715 } else { 1716 rc = 0; 1717 } 1718 } 1719 1720 /* if PMP detection failed, so turn off the PMA bit and 1721 * reset the port again 1722 */ 1723 if (pmp_rc != 0) { 1724 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 1725 "the port\n", PORTNAME(ap)); 1726 ahci_port_stop(ap, 0); 1727 ahci_port_clo(ap); 1728 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1729 cmd &= ~AHCI_PREG_CMD_PMA; 1730 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1731 1732 ahci_pwrite(ap, AHCI_PREG_IE, 0); 1733 ahci_port_stop(ap, 0); 1734 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 1735 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 1736 ahci_flush_tfd(ap); 1737 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1738 1739 ahci_pwrite(ap, AHCI_PREG_IS, -1); 1740 1741 ahci_enable_interrupts(ap); 1742 1743 rc = pmp_rc; 1744 } 1745 1746 return (rc); 1747 } 1748 1749 void 1750 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1751 u_int32_t flags) 1752 { 1753 flags |= len - 1; 1754 1755 htolem64(&prd->dba, addr); 1756 htolem32(&prd->flags, flags); 1757 } 1758 1759 int 1760 ahci_load_prdt(struct ahci_ccb *ccb) 1761 { 1762 struct ahci_port *ap = ccb->ccb_port; 1763 struct ahci_softc *sc = ap->ap_sc; 1764 struct ata_xfer *xa = &ccb->ccb_xa; 1765 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1766 bus_dmamap_t dmap = ccb->ccb_dmamap; 1767 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1768 int i, error; 1769 1770 if (xa->datalen == 0) { 1771 ccb->ccb_cmd_hdr->prdtl = 0; 1772 return (0); 1773 } 1774 1775 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 1776 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1777 if (error != 0) { 1778 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1779 return (1); 1780 } 1781 1782 for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1783 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1784 dmap->dm_segs[i].ds_len, 0); 1785 } 1786 1787 ahci_load_prdt_seg(&prdt[i], 1788 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1789 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 1790 1791 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 1792 1793 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1794 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 1795 BUS_DMASYNC_PREWRITE); 1796 1797 return (0); 1798 } 1799 1800 void 1801 ahci_unload_prdt(struct ahci_ccb *ccb) 1802 { 1803 struct ahci_port *ap = ccb->ccb_port; 1804 struct ahci_softc *sc = ap->ap_sc; 1805 struct ata_xfer *xa = &ccb->ccb_xa; 1806 bus_dmamap_t dmap = ccb->ccb_dmamap; 1807 1808 if (xa->datalen != 0) { 1809 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1810 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 1811 BUS_DMASYNC_POSTWRITE); 1812 1813 bus_dmamap_unload(sc->sc_dmat, dmap); 1814 1815 if (ccb->ccb_xa.flags & ATA_F_NCQ) 1816 xa->resid = 0; 1817 else 1818 xa->resid = xa->datalen - 1819 lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 1820 } 1821 } 1822 1823 int 1824 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 1825 { 1826 struct ahci_port *ap = ccb->ccb_port; 1827 int s; 1828 1829 s = splbio(); 1830 ahci_start(ccb); 1831 do { 1832 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 1833 1 << ccb->ccb_slot)) { 1834 splx(s); 1835 return (0); 1836 } 1837 if (ccb->ccb_xa.state == ATA_S_ERROR) { 1838 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 1839 PORTNAME(ap), ccb->ccb_slot); 1840 /* pretend it timed out? */ 1841 if (timeout_fn != NULL) { 1842 timeout_fn(ccb); 1843 } 1844 splx(s); 1845 return (1); 1846 } 1847 1848 delay(1000); 1849 } while (--timeout > 0); 1850 1851 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 1852 if (timeout_fn != NULL) 1853 timeout_fn(ccb); 1854 1855 splx(s); 1856 1857 return (1); 1858 } 1859 1860 void 1861 ahci_start(struct ahci_ccb *ccb) 1862 { 1863 struct ahci_port *ap = ccb->ccb_port; 1864 struct ahci_softc *sc = ap->ap_sc; 1865 1866 /* Zero transferred byte count before transfer */ 1867 ccb->ccb_cmd_hdr->prdbc = 0; 1868 1869 /* Sync command list entry and corresponding command table entry */ 1870 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1871 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 1872 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 1873 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1874 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 1875 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 1876 1877 /* Prepare RFIS area for write by controller */ 1878 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 1879 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 1880 1881 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1882 /* Issue NCQ commands only when there are no outstanding 1883 * standard commands. */ 1884 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 1885 (ap->ap_sactive != 0 && 1886 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 1887 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1888 } else { 1889 KASSERT(ap->ap_active_cnt == 0); 1890 ap->ap_sactive |= (1 << ccb->ccb_slot); 1891 ccb->ccb_xa.state = ATA_S_ONCHIP; 1892 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 1893 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1894 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 1895 } 1896 } else { 1897 /* Wait for all NCQ commands to finish before issuing standard 1898 * command. */ 1899 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 1900 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1901 else if (ap->ap_active_cnt < 2) { 1902 ap->ap_active |= 1 << ccb->ccb_slot; 1903 ccb->ccb_xa.state = ATA_S_ONCHIP; 1904 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1905 ap->ap_active_cnt++; 1906 } 1907 } 1908 } 1909 1910 void 1911 ahci_issue_pending_ncq_commands(struct ahci_port *ap) 1912 { 1913 struct ahci_ccb *nextccb; 1914 u_int32_t sact_change = 0; 1915 1916 KASSERT(ap->ap_active_cnt == 0); 1917 1918 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1919 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 1920 return; 1921 1922 /* Start all the NCQ commands at the head of the pending list. 1923 * If a port multiplier is attached to the port, we can only 1924 * issue commands for one of its ports at a time. 1925 */ 1926 if (ap->ap_sactive != 0 && 1927 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 1928 return; 1929 } 1930 1931 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 1932 do { 1933 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1934 sact_change |= 1 << nextccb->ccb_slot; 1935 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1936 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1937 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 1938 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 1939 1940 ap->ap_sactive |= sact_change; 1941 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 1942 ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 1943 } 1944 1945 void 1946 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 1947 { 1948 struct ahci_ccb *nextccb; 1949 1950 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1951 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 1952 if (last_was_ncq) { 1953 KASSERT(nextccb->ccb_xa.pmp_port != 1954 ap->ap_pmp_ncq_port); 1955 /* otherwise it should have been started already */ 1956 } else { 1957 ap->ap_active_cnt--; 1958 } 1959 1960 /* Issue NCQ commands only when there are no outstanding 1961 * standard commands, and previous NCQ commands for other 1962 * PMP ports have finished. 1963 */ 1964 if (ap->ap_active == 0) 1965 ahci_issue_pending_ncq_commands(ap); 1966 else 1967 KASSERT(ap->ap_active_cnt == 1); 1968 } else if (nextccb) { 1969 if (ap->ap_sactive != 0 || last_was_ncq) 1970 KASSERT(ap->ap_active_cnt == 0); 1971 1972 /* Wait for all NCQ commands to finish before issuing standard 1973 * command. */ 1974 if (ap->ap_sactive != 0) 1975 return; 1976 1977 /* Keep up to 2 standard commands on-chip at a time. */ 1978 do { 1979 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1980 ap->ap_active |= 1 << nextccb->ccb_slot; 1981 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1982 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 1983 if (last_was_ncq) 1984 ap->ap_active_cnt++; 1985 if (ap->ap_active_cnt == 2) 1986 break; 1987 KASSERT(ap->ap_active_cnt == 1); 1988 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1989 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 1990 } else if (!last_was_ncq) { 1991 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 1992 1993 /* Standard command finished, none waiting to start. */ 1994 ap->ap_active_cnt--; 1995 } else { 1996 KASSERT(ap->ap_active_cnt == 0); 1997 1998 /* NCQ command finished. */ 1999 } 2000 } 2001 2002 int 2003 ahci_intr(void *arg) 2004 { 2005 struct ahci_softc *sc = arg; 2006 u_int32_t is, ack = 0; 2007 int port; 2008 2009 /* Read global interrupt status */ 2010 is = ahci_read(sc, AHCI_REG_IS); 2011 if (is == 0 || is == 0xffffffff) 2012 return (0); 2013 ack = is; 2014 2015 #ifdef AHCI_COALESCE 2016 /* Check coalescing interrupt first */ 2017 if (is & sc->sc_ccc_mask) { 2018 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 2019 DEVNAME(sc)); 2020 is &= ~sc->sc_ccc_mask; 2021 is |= sc->sc_ccc_ports_cur; 2022 } 2023 #endif 2024 2025 /* Process interrupts for each port */ 2026 while (is) { 2027 port = ffs(is) - 1; 2028 if (sc->sc_ports[port]) 2029 ahci_port_intr(sc->sc_ports[port], 2030 AHCI_PREG_CI_ALL_SLOTS); 2031 is &= ~(1 << port); 2032 } 2033 2034 /* Finally, acknowledge global interrupt */ 2035 ahci_write(sc, AHCI_REG_IS, ack); 2036 2037 return (1); 2038 } 2039 2040 u_int32_t 2041 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 2042 { 2043 struct ahci_softc *sc = ap->ap_sc; 2044 u_int32_t is, ci_saved, ci_masked, processed = 0; 2045 int slot, need_restart = 0; 2046 int process_error = 0; 2047 struct ahci_ccb *ccb; 2048 volatile u_int32_t *active; 2049 #ifdef DIAGNOSTIC 2050 u_int32_t tmp; 2051 #endif 2052 2053 is = ahci_pread(ap, AHCI_PREG_IS); 2054 2055 /* Ack port interrupt only if checking all command slots. */ 2056 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 2057 ahci_pwrite(ap, AHCI_PREG_IS, is); 2058 2059 if (is) 2060 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 2061 is, AHCI_PFMT_IS); 2062 2063 if (ap->ap_sactive) { 2064 /* Active NCQ commands - use SActive instead of CI */ 2065 KASSERT(ap->ap_active == 0); 2066 KASSERT(ap->ap_active_cnt == 0); 2067 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2068 active = &ap->ap_sactive; 2069 } else { 2070 /* Save CI */ 2071 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2072 active = &ap->ap_active; 2073 } 2074 2075 if (is & AHCI_PREG_IS_TFES) { 2076 process_error = 1; 2077 } else if (is & AHCI_PREG_IS_DHRS) { 2078 u_int32_t tfd; 2079 u_int32_t cmd; 2080 u_int32_t serr; 2081 2082 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2083 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2084 serr = ahci_pread(ap, AHCI_PREG_SERR); 2085 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2086 (cmd & AHCI_PREG_CMD_CR) == 0) { 2087 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 2088 " %b, DIAG: %b\n", PORTNAME(ap), tfd, 2089 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2090 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2091 AHCI_PFMT_SERR_DIAG); 2092 process_error = 1; 2093 } else { 2094 /* rfis copy back is in the normal execution path */ 2095 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2096 } 2097 } 2098 2099 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 2100 if (process_error) { 2101 u_int32_t tfd, serr; 2102 int err_slot; 2103 2104 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2105 serr = ahci_pread(ap, AHCI_PREG_SERR); 2106 2107 if (ap->ap_sactive == 0) { 2108 /* Errored slot is easy to determine from CMD. */ 2109 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 2110 AHCI_PREG_CMD)); 2111 2112 if ((ci_saved & (1 << err_slot)) == 0) { 2113 /* 2114 * Hardware doesn't seem to report correct 2115 * slot number. If there's only one 2116 * outstanding command we can cope, 2117 * otherwise fail all active commands. 2118 */ 2119 if (ap->ap_active_cnt == 1) 2120 err_slot = ffs(ap->ap_active) - 1; 2121 else 2122 goto failall; 2123 } 2124 2125 ccb = &ap->ap_ccbs[err_slot]; 2126 2127 /* Preserve received taskfile data from the RFIS. */ 2128 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2129 sizeof(struct ata_fis_d2h)); 2130 } else 2131 err_slot = -1; /* Must extract error from log page */ 2132 2133 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 2134 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 2135 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2136 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2137 AHCI_PFMT_SERR_DIAG); 2138 2139 /* Turn off ST to clear CI and SACT. */ 2140 ahci_port_stop(ap, 0); 2141 need_restart = 1; 2142 2143 /* Clear SERR to enable capturing new errors. */ 2144 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2145 2146 /* Acknowledge the interrupts we can recover from. */ 2147 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2148 AHCI_PREG_IS_IFS); 2149 is = ahci_pread(ap, AHCI_PREG_IS); 2150 2151 /* If device hasn't cleared its busy status, try to idle it. */ 2152 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2153 2154 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 2155 (ap->ap_state == AP_S_ERROR_RECOVERY)) { 2156 /* can't reset the port here, just make sure 2157 * the operation fails and the port still works. 2158 */ 2159 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 2160 printf("%s: error on PMP port %d, idling " 2161 "device\n", PORTNAME(ap), 2162 ccb->ccb_xa.pmp_port); 2163 if (ahci_pmp_port_softreset(ap, 2164 ccb->ccb_xa.pmp_port) == 0) { 2165 printf("%s: unable to softreset port " 2166 "%d\n", PORTNAME(ap), 2167 ccb->ccb_xa.pmp_port); 2168 if (ahci_pmp_port_portreset(ap, 2169 ccb->ccb_xa.pmp_port)) { 2170 printf("%s: failed to port " 2171 " reset %d, giving up on " 2172 "it\n", PORTNAME(ap), 2173 ccb->ccb_xa.pmp_port); 2174 goto fatal; 2175 } 2176 } 2177 } else { 2178 printf("%s: attempting to idle device\n", 2179 PORTNAME(ap)); 2180 if (ahci_port_softreset(ap)) { 2181 printf("%s: failed to soft reset " 2182 "device\n", PORTNAME(ap)); 2183 if (ahci_port_portreset(ap, 0)) { 2184 printf("%s: failed to port " 2185 "reset device, give up on " 2186 "it\n", PORTNAME(ap)); 2187 goto fatal; 2188 } 2189 } 2190 } 2191 2192 /* Had to reset device, can't gather extended info. */ 2193 } else if (ap->ap_sactive) { 2194 /* Recover the NCQ error from log page 10h. 2195 * We can only have queued commands active for one port 2196 * at a time, so we know which device errored. 2197 */ 2198 ahci_port_read_ncq_error(ap, &err_slot, 2199 ap->ap_pmp_ncq_port); 2200 if (err_slot < 0) 2201 goto failall; 2202 2203 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 2204 PORTNAME(ap), err_slot); 2205 2206 ccb = &ap->ap_ccbs[err_slot]; 2207 if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2208 printf("%s: NCQ errored slot %d is idle" 2209 " (%08x active)\n", PORTNAME(ap), err_slot, 2210 ci_saved); 2211 goto failall; 2212 } 2213 } else { 2214 /* Didn't reset, could gather extended info from log. */ 2215 } 2216 2217 /* 2218 * If we couldn't determine the errored slot, reset the port 2219 * and fail all the active slots. 2220 */ 2221 if (err_slot == -1) { 2222 if (ahci_port_softreset(ap) != 0 && 2223 ahci_port_portreset(ap, 0) != 0) { 2224 printf("%s: couldn't reset after NCQ error, " 2225 "disabling device.\n", PORTNAME(ap)); 2226 goto fatal; 2227 } 2228 printf("%s: couldn't recover NCQ error, failing " 2229 "all outstanding commands.\n", PORTNAME(ap)); 2230 goto failall; 2231 } 2232 2233 /* Clear the failed command in saved CI so completion runs. */ 2234 ci_saved &= ~(1 << err_slot); 2235 2236 /* Note the error in the ata_xfer. */ 2237 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2238 ccb->ccb_xa.state = ATA_S_ERROR; 2239 2240 #ifdef DIAGNOSTIC 2241 /* There may only be one outstanding standard command now. */ 2242 if (ap->ap_sactive == 0) { 2243 tmp = ci_saved; 2244 if (tmp) { 2245 slot = ffs(tmp) - 1; 2246 tmp &= ~(1 << slot); 2247 KASSERT(tmp == 0); 2248 } 2249 } 2250 #endif 2251 } 2252 2253 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 2254 * when there's a normal SATA device attached. 2255 */ 2256 if ((ap->ap_state == AP_S_PMP_PROBE) && 2257 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 2258 (is & AHCI_PREG_IS_IPMS)) { 2259 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2260 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 2261 PORTNAME(ap), slot); 2262 2263 ccb = &ap->ap_ccbs[slot]; 2264 ccb->ccb_xa.state = ATA_S_ERROR; 2265 2266 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 2267 is &= ~AHCI_PREG_IS_IPMS; 2268 } 2269 2270 /* ignore IFS errors while resetting a PMP port */ 2271 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 2272 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 2273 "port\n", PORTNAME(ap)); 2274 2275 need_restart = 1; 2276 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2277 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2278 is &= ~AHCI_PREG_IS_IFS; 2279 goto failall; 2280 } 2281 2282 /* Check for remaining errors - they are fatal. */ 2283 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2284 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2285 printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 2286 PORTNAME(ap), is, AHCI_PFMT_IS); 2287 2288 /* XXX try recovery first */ 2289 goto fatal; 2290 } 2291 2292 /* Fail all outstanding commands if we know the port won't recover. */ 2293 if (ap->ap_state == AP_S_FATAL_ERROR) { 2294 fatal: 2295 ap->ap_state = AP_S_FATAL_ERROR; 2296 failall: 2297 2298 /* Ensure port is shut down. */ 2299 ahci_port_stop(ap, 1); 2300 2301 /* Error all the active slots. */ 2302 ci_masked = ci_saved & *active; 2303 while (ci_masked) { 2304 slot = ffs(ci_masked) - 1; 2305 ccb = &ap->ap_ccbs[slot]; 2306 ci_masked &= ~(1 << slot); 2307 ccb->ccb_xa.state = ATA_S_ERROR; 2308 } 2309 2310 /* Run completion for all active slots. */ 2311 ci_saved &= ~*active; 2312 2313 /* Don't restart the port if our problems were deemed fatal. */ 2314 if (ap->ap_state == AP_S_FATAL_ERROR) 2315 need_restart = 0; 2316 } 2317 2318 /* 2319 * CCB completion is detected by noticing its slot's bit in CI has 2320 * changed to zero some time after we activated it. 2321 * If we are polling, we may only be interested in particular slot(s). 2322 */ 2323 ci_masked = ~ci_saved & *active & ci_mask; 2324 while (ci_masked) { 2325 slot = ffs(ci_masked) - 1; 2326 ccb = &ap->ap_ccbs[slot]; 2327 ci_masked &= ~(1 << slot); 2328 2329 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2330 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2331 " (error)" : ""); 2332 2333 bus_dmamap_sync(sc->sc_dmat, 2334 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2335 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 2336 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 2337 2338 bus_dmamap_sync(sc->sc_dmat, 2339 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2340 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 2341 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 2342 2343 bus_dmamap_sync(sc->sc_dmat, 2344 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 2345 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 2346 2347 *active &= ~(1 << ccb->ccb_slot); 2348 /* Copy the rfis into the ccb if we were asked for it */ 2349 if (ccb->ccb_xa.state == ATA_S_ONCHIP && 2350 ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 2351 memcpy(&ccb->ccb_xa.rfis, 2352 ap->ap_rfis->rfis, 2353 sizeof(struct ata_fis_d2h)); 2354 } 2355 2356 processed |= 1 << ccb->ccb_slot; 2357 2358 ccb->ccb_done(ccb); 2359 } 2360 2361 if (need_restart) { 2362 /* Restart command DMA on the port */ 2363 ahci_port_start(ap, 0); 2364 2365 /* Re-enable outstanding commands on port. */ 2366 if (ci_saved) { 2367 #ifdef DIAGNOSTIC 2368 tmp = ci_saved; 2369 while (tmp) { 2370 slot = ffs(tmp) - 1; 2371 tmp &= ~(1 << slot); 2372 ccb = &ap->ap_ccbs[slot]; 2373 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2374 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 2375 (!!ap->ap_sactive)); 2376 } 2377 #endif 2378 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 2379 "re-enabling%s slots %08x\n", PORTNAME(ap), 2380 ap->ap_sactive ? " NCQ" : "", ci_saved); 2381 2382 if (ap->ap_sactive) 2383 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 2384 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 2385 } 2386 } 2387 2388 return (processed); 2389 } 2390 2391 struct ahci_ccb * 2392 ahci_get_ccb(struct ahci_port *ap) 2393 { 2394 struct ahci_ccb *ccb; 2395 2396 mtx_enter(&ap->ap_ccb_mtx); 2397 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2398 if (ccb != NULL) { 2399 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2400 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2401 ccb->ccb_xa.state = ATA_S_SETUP; 2402 } 2403 mtx_leave(&ap->ap_ccb_mtx); 2404 2405 return (ccb); 2406 } 2407 2408 void 2409 ahci_put_ccb(struct ahci_ccb *ccb) 2410 { 2411 struct ahci_port *ap = ccb->ccb_port; 2412 2413 #ifdef DIAGNOSTIC 2414 if (ccb->ccb_xa.state != ATA_S_COMPLETE && 2415 ccb->ccb_xa.state != ATA_S_TIMEOUT && 2416 ccb->ccb_xa.state != ATA_S_ERROR) { 2417 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 2418 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 2419 ccb->ccb_slot); 2420 } 2421 #endif 2422 2423 ccb->ccb_xa.state = ATA_S_PUT; 2424 mtx_enter(&ap->ap_ccb_mtx); 2425 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2426 mtx_leave(&ap->ap_ccb_mtx); 2427 } 2428 2429 struct ahci_ccb * 2430 ahci_get_err_ccb(struct ahci_port *ap) 2431 { 2432 struct ahci_ccb *err_ccb; 2433 u_int32_t sact; 2434 2435 splassert(IPL_BIO); 2436 2437 /* No commands may be active on the chip. */ 2438 sact = ahci_pread(ap, AHCI_PREG_SACT); 2439 if (sact != 0) 2440 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 2441 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2442 2443 #ifdef DIAGNOSTIC 2444 KASSERT(ap->ap_err_busy == 0); 2445 ap->ap_err_busy = 1; 2446 #endif 2447 /* Save outstanding command state. */ 2448 ap->ap_err_saved_active = ap->ap_active; 2449 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2450 ap->ap_err_saved_sactive = ap->ap_sactive; 2451 2452 /* 2453 * Pretend we have no commands outstanding, so that completions won't 2454 * run prematurely. 2455 */ 2456 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2457 2458 /* 2459 * Grab a CCB to use for error recovery. This should never fail, as 2460 * we ask atascsi to reserve one for us at init time. 2461 */ 2462 err_ccb = ap->ap_ccb_err; 2463 err_ccb->ccb_xa.flags = 0; 2464 err_ccb->ccb_xa.state = ATA_S_SETUP; 2465 err_ccb->ccb_done = ahci_empty_done; 2466 2467 return (err_ccb); 2468 } 2469 2470 void 2471 ahci_put_err_ccb(struct ahci_ccb *ccb) 2472 { 2473 struct ahci_port *ap = ccb->ccb_port; 2474 u_int32_t sact; 2475 2476 splassert(IPL_BIO); 2477 2478 #ifdef DIAGNOSTIC 2479 KASSERT(ap->ap_err_busy); 2480 #endif 2481 /* No commands may be active on the chip */ 2482 sact = ahci_pread(ap, AHCI_PREG_SACT); 2483 if (sact != 0) 2484 printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact); 2485 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2486 2487 /* Done with the CCB */ 2488 KASSERT(ccb == ap->ap_ccb_err); 2489 2490 /* Restore outstanding command state */ 2491 ap->ap_sactive = ap->ap_err_saved_sactive; 2492 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2493 ap->ap_active = ap->ap_err_saved_active; 2494 2495 #ifdef DIAGNOSTIC 2496 ap->ap_err_busy = 0; 2497 #endif 2498 } 2499 2500 struct ahci_ccb * 2501 ahci_get_pmp_ccb(struct ahci_port *ap) 2502 { 2503 struct ahci_ccb *ccb; 2504 u_int32_t sact; 2505 2506 /* some PMP commands need to be issued on slot 1, 2507 * particularly the command that clears SRST and 2508 * fetches the device signature. 2509 * 2510 * ensure the chip is idle and ccb 1 is available. 2511 */ 2512 splassert(IPL_BIO); 2513 2514 sact = ahci_pread(ap, AHCI_PREG_SACT); 2515 if (sact != 0) 2516 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 2517 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2518 2519 ccb = &ap->ap_ccbs[1]; 2520 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2521 ccb->ccb_xa.flags = 0; 2522 ccb->ccb_done = ahci_pmp_cmd_done; 2523 2524 mtx_enter(&ap->ap_ccb_mtx); 2525 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2526 mtx_leave(&ap->ap_ccb_mtx); 2527 2528 return ccb; 2529 } 2530 2531 void 2532 ahci_put_pmp_ccb(struct ahci_ccb *ccb) 2533 { 2534 struct ahci_port *ap = ccb->ccb_port; 2535 u_int32_t sact; 2536 2537 /* make sure this is the right ccb */ 2538 KASSERT(ccb == &ap->ap_ccbs[1]); 2539 2540 /* No commands may be active on the chip */ 2541 sact = ahci_pread(ap, AHCI_PREG_SACT); 2542 if (sact != 0) 2543 printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact); 2544 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2545 2546 ccb->ccb_xa.state = ATA_S_PUT; 2547 mtx_enter(&ap->ap_ccb_mtx); 2548 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2549 mtx_leave(&ap->ap_ccb_mtx); 2550 } 2551 2552 int 2553 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 2554 { 2555 struct ahci_ccb *ccb; 2556 struct ahci_cmd_hdr *cmd_slot; 2557 u_int32_t cmd; 2558 struct ata_fis_h2d *fis; 2559 int rc = EIO, oldstate; 2560 2561 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 2562 oldstate = ap->ap_state; 2563 ap->ap_state = AP_S_ERROR_RECOVERY; 2564 2565 /* Save command register state. */ 2566 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 2567 2568 /* Port should have been idled already. Start it. */ 2569 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 2570 ahci_port_start(ap, 0); 2571 2572 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 2573 ccb = ahci_get_err_ccb(ap); 2574 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 2575 ccb->ccb_xa.data = ap->ap_err_scratch; 2576 ccb->ccb_xa.datalen = 512; 2577 cmd_slot = ccb->ccb_cmd_hdr; 2578 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2579 2580 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2581 fis->type = ATA_FIS_TYPE_H2D; 2582 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 2583 fis->command = ATA_C_READ_LOG_EXT; 2584 fis->lba_low = 0x10; /* queued error log page (10h) */ 2585 fis->sector_count = 1; /* number of sectors (1) */ 2586 fis->sector_count_exp = 0; 2587 fis->lba_mid = 0; /* starting offset */ 2588 fis->lba_mid_exp = 0; 2589 fis->device = 0; 2590 2591 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 2592 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 2593 2594 if (ahci_load_prdt(ccb) != 0) { 2595 rc = ENOMEM; /* XXX caller must abort all commands */ 2596 goto err; 2597 } 2598 2599 ccb->ccb_xa.state = ATA_S_PENDING; 2600 if (ahci_poll(ccb, 1000, NULL) != 0 || 2601 ccb->ccb_xa.state == ATA_S_ERROR) 2602 goto err; 2603 2604 rc = 0; 2605 err: 2606 /* Abort our command, if it failed, by stopping command DMA. */ 2607 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 2608 printf("%s: log page read failed, slot %d was still active.\n", 2609 PORTNAME(ap), ccb->ccb_slot); 2610 ahci_port_stop(ap, 0); 2611 } 2612 2613 /* Done with the error CCB now. */ 2614 ahci_unload_prdt(ccb); 2615 ahci_put_err_ccb(ccb); 2616 2617 /* Extract failed register set and tags from the scratch space. */ 2618 if (rc == 0) { 2619 struct ata_log_page_10h *log; 2620 int err_slot; 2621 2622 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2623 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 2624 /* Not queued bit was set - wasn't an NCQ error? */ 2625 printf("%s: read NCQ error page, but not an NCQ " 2626 "error?\n", PORTNAME(ap)); 2627 rc = ESRCH; 2628 } else { 2629 /* Copy back the log record as a D2H register FIS. */ 2630 *err_slotp = err_slot = log->err_regs.type & 2631 ATA_LOG_10H_TYPE_TAG_MASK; 2632 2633 ccb = &ap->ap_ccbs[err_slot]; 2634 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2635 sizeof(struct ata_fis_d2h)); 2636 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2637 ccb->ccb_xa.rfis.flags = 0; 2638 } 2639 } 2640 2641 /* Restore saved CMD register state */ 2642 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 2643 ap->ap_state = oldstate; 2644 2645 return (rc); 2646 } 2647 2648 struct ahci_dmamem * 2649 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 2650 { 2651 struct ahci_dmamem *adm; 2652 int nsegs; 2653 2654 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 2655 if (adm == NULL) 2656 return (NULL); 2657 2658 adm->adm_size = size; 2659 2660 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2661 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2662 goto admfree; 2663 2664 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 2665 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 2666 goto destroy; 2667 2668 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2669 &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0) 2670 goto free; 2671 2672 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 2673 NULL, BUS_DMA_NOWAIT) != 0) 2674 goto unmap; 2675 2676 return (adm); 2677 2678 unmap: 2679 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 2680 free: 2681 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2682 destroy: 2683 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2684 admfree: 2685 free(adm, M_DEVBUF, sizeof(*adm)); 2686 2687 return (NULL); 2688 } 2689 2690 void 2691 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 2692 { 2693 bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 2694 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 2695 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2696 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2697 free(adm, M_DEVBUF, sizeof(*adm)); 2698 } 2699 2700 u_int32_t 2701 ahci_read(struct ahci_softc *sc, bus_size_t r) 2702 { 2703 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2704 BUS_SPACE_BARRIER_READ); 2705 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 2706 } 2707 2708 void 2709 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 2710 { 2711 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 2712 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2713 BUS_SPACE_BARRIER_WRITE); 2714 } 2715 2716 int 2717 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 2718 u_int32_t target) 2719 { 2720 int i; 2721 2722 for (i = 0; i < 1000; i++) { 2723 if ((ahci_read(sc, r) & mask) != target) 2724 return (0); 2725 delay(1000); 2726 } 2727 2728 return (1); 2729 } 2730 2731 u_int32_t 2732 ahci_pread(struct ahci_port *ap, bus_size_t r) 2733 { 2734 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2735 BUS_SPACE_BARRIER_READ); 2736 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 2737 } 2738 2739 void 2740 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 2741 { 2742 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 2743 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2744 BUS_SPACE_BARRIER_WRITE); 2745 } 2746 2747 int 2748 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 2749 u_int32_t target, int n) 2750 { 2751 int i; 2752 2753 for (i = 0; i < n * 1000; i++) { 2754 if ((ahci_pread(ap, r) & mask) == target) 2755 return (0); 2756 delay(1000); 2757 } 2758 2759 return (1); 2760 } 2761 2762 int 2763 ahci_ata_probe(void *xsc, int port, int lun) 2764 { 2765 struct ahci_softc *sc = xsc; 2766 struct ahci_port *ap = sc->sc_ports[port]; 2767 2768 if (ap == NULL) 2769 return (ATA_PORT_T_NONE); 2770 2771 if (lun != 0) { 2772 int pmp_port = lun - 1; 2773 if (pmp_port >= ap->ap_pmp_ports) { 2774 return (ATA_PORT_T_NONE); 2775 } 2776 return (ahci_pmp_port_probe(ap, pmp_port)); 2777 } else { 2778 return (ahci_port_signature(ap)); 2779 } 2780 } 2781 2782 void 2783 ahci_ata_free(void *xsc, int port, int lun) 2784 { 2785 2786 } 2787 2788 struct ata_xfer * 2789 ahci_ata_get_xfer(void *aaa_cookie, int port) 2790 { 2791 struct ahci_softc *sc = aaa_cookie; 2792 struct ahci_port *ap = sc->sc_ports[port]; 2793 struct ahci_ccb *ccb; 2794 2795 ccb = ahci_get_ccb(ap); 2796 if (ccb == NULL) { 2797 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 2798 PORTNAME(ap)); 2799 return (NULL); 2800 } 2801 2802 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 2803 PORTNAME(ap), ccb->ccb_slot); 2804 2805 return ((struct ata_xfer *)ccb); 2806 } 2807 2808 void 2809 ahci_ata_put_xfer(struct ata_xfer *xa) 2810 { 2811 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2812 2813 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 2814 2815 ahci_put_ccb(ccb); 2816 } 2817 2818 void 2819 ahci_ata_cmd(struct ata_xfer *xa) 2820 { 2821 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2822 struct ahci_cmd_hdr *cmd_slot; 2823 int s; 2824 u_int16_t flags; 2825 2826 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 2827 goto failcmd; 2828 2829 ccb->ccb_done = ahci_ata_cmd_done; 2830 2831 cmd_slot = ccb->ccb_cmd_hdr; 2832 flags = 5 /* FIS length (in DWORDs) */; 2833 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 2834 2835 if (xa->flags & ATA_F_WRITE) 2836 flags |= AHCI_CMD_LIST_FLAG_W; 2837 2838 if (xa->flags & ATA_F_PACKET) 2839 flags |= AHCI_CMD_LIST_FLAG_A; 2840 2841 htolem16(&cmd_slot->flags, flags); 2842 2843 if (ahci_load_prdt(ccb) != 0) 2844 goto failcmd; 2845 2846 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 2847 2848 xa->state = ATA_S_PENDING; 2849 2850 if (xa->flags & ATA_F_POLL) 2851 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 2852 else { 2853 s = splbio(); 2854 timeout_add_msec(&xa->stimeout, xa->timeout); 2855 ahci_start(ccb); 2856 splx(s); 2857 } 2858 2859 return; 2860 2861 failcmd: 2862 s = splbio(); 2863 xa->state = ATA_S_ERROR; 2864 ata_complete(xa); 2865 splx(s); 2866 } 2867 2868 void 2869 ahci_pmp_cmd_done(struct ahci_ccb *ccb) 2870 { 2871 struct ata_xfer *xa = &ccb->ccb_xa; 2872 2873 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2874 ahci_issue_pending_commands(ccb->ccb_port, 2875 xa->flags & ATA_F_NCQ); 2876 2877 xa->state = ATA_S_COMPLETE; 2878 } 2879 2880 2881 void 2882 ahci_ata_cmd_done(struct ahci_ccb *ccb) 2883 { 2884 struct ata_xfer *xa = &ccb->ccb_xa; 2885 2886 timeout_del(&xa->stimeout); 2887 2888 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2889 ahci_issue_pending_commands(ccb->ccb_port, 2890 xa->flags & ATA_F_NCQ); 2891 2892 ahci_unload_prdt(ccb); 2893 2894 if (xa->state == ATA_S_ONCHIP) 2895 xa->state = ATA_S_COMPLETE; 2896 #ifdef DIAGNOSTIC 2897 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 2898 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 2899 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 2900 ccb->ccb_slot); 2901 #endif 2902 if (xa->state != ATA_S_TIMEOUT) 2903 ata_complete(xa); 2904 } 2905 2906 void 2907 ahci_ata_cmd_timeout(void *arg) 2908 { 2909 struct ahci_ccb *ccb = arg; 2910 struct ata_xfer *xa = &ccb->ccb_xa; 2911 struct ahci_port *ap = ccb->ccb_port; 2912 int s, ccb_was_started, ncq_cmd; 2913 volatile u_int32_t *active; 2914 2915 s = splbio(); 2916 2917 ncq_cmd = (xa->flags & ATA_F_NCQ); 2918 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 2919 2920 if (ccb->ccb_xa.state == ATA_S_PENDING) { 2921 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 2922 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 2923 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2924 ccb_was_started = 0; 2925 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 2926 1 << ccb->ccb_slot)) { 2927 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 2928 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 2929 goto ret; 2930 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2931 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 2932 "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 2933 ISSET(*active, 1 << ccb->ccb_slot) ? 2934 " but slot is still active?" : "."); 2935 goto ret; 2936 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 2937 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 2938 1 << ccb->ccb_slot)) { 2939 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 2940 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 2941 ccb->ccb_slot); 2942 *active &= ~(1 << ccb->ccb_slot); 2943 ccb->ccb_done(ccb); 2944 goto ret; 2945 } else { 2946 ccb_was_started = 1; 2947 } 2948 2949 /* Complete the slot with a timeout error. */ 2950 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2951 *active &= ~(1 << ccb->ccb_slot); 2952 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 2953 ccb->ccb_done(ccb); /* This won't issue pending commands or run the 2954 atascsi completion. */ 2955 2956 /* Reset port to abort running command. */ 2957 if (ccb_was_started) { 2958 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 2959 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 2960 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 2961 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 2962 != 0) { 2963 printf("%s: failed to reset port during timeout " 2964 "handling, disabling it\n", PORTNAME(ap)); 2965 ap->ap_state = AP_S_FATAL_ERROR; 2966 } 2967 2968 /* Restart any other commands that were aborted by the reset. */ 2969 if (*active) { 2970 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 2971 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 2972 *active); 2973 if (ncq_cmd) 2974 ahci_pwrite(ap, AHCI_PREG_SACT, *active); 2975 ahci_pwrite(ap, AHCI_PREG_CI, *active); 2976 } 2977 } 2978 2979 /* Issue any pending commands now. */ 2980 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 2981 if (ccb_was_started) 2982 ahci_issue_pending_commands(ap, ncq_cmd); 2983 else if (ap->ap_active == 0) 2984 ahci_issue_pending_ncq_commands(ap); 2985 2986 /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 2987 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 2988 ata_complete(xa); 2989 2990 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 2991 ret: 2992 splx(s); 2993 } 2994 2995 void 2996 ahci_empty_done(struct ahci_ccb *ccb) 2997 { 2998 if (ccb->ccb_xa.state != ATA_S_ERROR) 2999 ccb->ccb_xa.state = ATA_S_COMPLETE; 3000 } 3001 3002 int 3003 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 3004 { 3005 struct ahci_ccb *ccb; 3006 struct ata_fis_h2d *fis; 3007 int error; 3008 3009 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 3010 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 3011 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 3012 ccb->ccb_xa.state = ATA_S_PENDING; 3013 3014 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3015 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3016 fis->type = ATA_FIS_TYPE_H2D; 3017 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3018 fis->command = ATA_C_READ_PM; 3019 fis->features = which; 3020 fis->device = target | ATA_H2D_DEVICE_LBA; 3021 fis->control = ATA_FIS_CONTROL_4BIT; 3022 3023 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 3024 error = 1; 3025 } else { 3026 *datap = ccb->ccb_xa.rfis.sector_count | 3027 (ccb->ccb_xa.rfis.lba_low << 8) | 3028 (ccb->ccb_xa.rfis.lba_mid << 16) | 3029 (ccb->ccb_xa.rfis.lba_high << 24); 3030 error = 0; 3031 } 3032 ahci_put_pmp_ccb(ccb); 3033 return (error); 3034 } 3035 3036 int 3037 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 3038 { 3039 struct ahci_ccb *ccb; 3040 struct ata_fis_h2d *fis; 3041 int error; 3042 3043 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ 3044 ccb->ccb_xa.flags = ATA_F_POLL; 3045 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 3046 ccb->ccb_xa.state = ATA_S_PENDING; 3047 3048 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3049 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3050 fis->type = ATA_FIS_TYPE_H2D; 3051 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3052 fis->command = ATA_C_WRITE_PM; 3053 fis->features = which; 3054 fis->device = target | ATA_H2D_DEVICE_LBA; 3055 fis->sector_count = (u_int8_t)data; 3056 fis->lba_low = (u_int8_t)(data >> 8); 3057 fis->lba_mid = (u_int8_t)(data >> 16); 3058 fis->lba_high = (u_int8_t)(data >> 24); 3059 fis->control = ATA_FIS_CONTROL_4BIT; 3060 3061 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 3062 ahci_put_pmp_ccb(ccb); 3063 return (error); 3064 } 3065 3066 int 3067 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 3068 { 3069 int error; 3070 3071 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 3072 if (error == 0) 3073 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 3074 if (error) 3075 *datap = 0; 3076 3077 return (error); 3078 } 3079 3080 int 3081 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 3082 { 3083 u_int32_t chipid; 3084 u_int32_t rev; 3085 u_int32_t nports; 3086 u_int32_t features; 3087 u_int32_t enabled; 3088 int s; 3089 3090 s = splbio(); 3091 3092 if (ahci_pmp_read(ap, 15, 0, &chipid) || 3093 ahci_pmp_read(ap, 15, 1, &rev) || 3094 ahci_pmp_read(ap, 15, 2, &nports) || 3095 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 3096 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 3097 printf("%s: port multiplier identification failed\n", 3098 PORTNAME(ap)); 3099 splx(s); 3100 return (1); 3101 } 3102 splx(s); 3103 3104 nports &= 0x0F; 3105 3106 /* ignore SEMB port on SiI3726 port multiplier chips */ 3107 if (chipid == 0x37261095) { 3108 nports--; 3109 } 3110 3111 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 3112 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 3113 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 3114 SATA_PFMT_PM_FEA); 3115 3116 *ret_nports = nports; 3117 return (0); 3118 } 3119 3120 3121 #ifdef HIBERNATE 3122 void 3123 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 3124 { 3125 ccb->ccb_cmd_hdr->prdbc = 0; 3126 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 3127 } 3128 3129 int 3130 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 3131 { 3132 u_int32_t is, ci_saved; 3133 int process_error = 0; 3134 3135 is = ahci_pread(ap, AHCI_PREG_IS); 3136 3137 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 3138 3139 if (is & AHCI_PREG_IS_DHRS) { 3140 u_int32_t tfd; 3141 u_int32_t cmd; 3142 3143 tfd = ahci_pread(ap, AHCI_PREG_TFD); 3144 cmd = ahci_pread(ap, AHCI_PREG_CMD); 3145 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 3146 (cmd & AHCI_PREG_CMD_CR) == 0) { 3147 process_error = 1; 3148 } else { 3149 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 3150 } 3151 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 3152 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 3153 process_error = 1; 3154 } 3155 3156 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 3157 if (process_error) { 3158 3159 /* Turn off ST to clear CI and SACT. */ 3160 ahci_port_stop(ap, 0); 3161 3162 /* just return an error indicator? we can't meaningfully 3163 * recover, and on the way back out we'll DVACT_RESUME which 3164 * resets and reinits the port. 3165 */ 3166 return (EIO); 3167 } 3168 3169 /* command is finished when the bit in CI for the slot goes to 0 */ 3170 if (ci_saved & (1 << ccb->ccb_slot)) { 3171 return (EAGAIN); 3172 } 3173 3174 return (0); 3175 } 3176 3177 void 3178 ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 3179 { 3180 struct ata_xfer *xa = &ccb->ccb_xa; 3181 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 3182 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 3183 int i; 3184 paddr_t data_phys; 3185 u_int64_t data_bus_phys; 3186 vaddr_t data_addr; 3187 size_t seglen; 3188 size_t buflen; 3189 3190 if (xa->datalen == 0) { 3191 ccb->ccb_cmd_hdr->prdtl = 0; 3192 return; 3193 } 3194 3195 /* derived from i386/amd64 _bus_dma_load_buffer; 3196 * for amd64 the buffer will always be dma safe. 3197 */ 3198 3199 buflen = xa->datalen; 3200 data_addr = (vaddr_t)xa->data; 3201 for (i = 0; buflen > 0; i++) { 3202 pmap_extract(pmap_kernel(), data_addr, &data_phys); 3203 data_bus_phys = data_phys; 3204 3205 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 3206 if (buflen < seglen) 3207 seglen = buflen; 3208 3209 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3210 3211 data_addr += seglen; 3212 buflen -= seglen; 3213 } 3214 3215 htolem16(&cmd_slot->prdtl, i); 3216 } 3217 3218 int 3219 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 3220 int op, void *page) 3221 { 3222 /* we use the 'real' ahci_port and ahci_softc here, but 3223 * never write to them 3224 */ 3225 struct { 3226 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 3227 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 3228 /* cmd table isn't actually used because of mysteries */ 3229 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 3230 struct ahci_port *ap; 3231 struct ahci_ccb ccb_buf; 3232 struct ahci_ccb *ccb; 3233 struct ahci_cmd_hdr *hdr_buf; 3234 int pmp_port; 3235 daddr_t poffset; 3236 size_t psize; 3237 } *my = page; 3238 struct ata_fis_h2d *fis; 3239 u_int32_t sector_count; 3240 struct ahci_cmd_hdr *cmd_slot; 3241 int rc; 3242 int timeout; 3243 u_int16_t flags; 3244 3245 if (op == HIB_INIT) { 3246 struct device *disk; 3247 struct device *scsibus; 3248 struct ahci_softc *sc; 3249 extern struct cfdriver sd_cd; 3250 struct scsi_link *link; 3251 struct scsibus_softc *bus_sc; 3252 int port; 3253 paddr_t page_phys; 3254 u_int64_t item_phys; 3255 u_int32_t cmd; 3256 3257 my->poffset = blkno; 3258 my->psize = size; 3259 3260 /* map dev to an ahci port */ 3261 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 3262 scsibus = disk->dv_parent; 3263 sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 3264 3265 /* find the scsi_link for the device, which has the port */ 3266 port = -1; 3267 bus_sc = (struct scsibus_softc *)scsibus; 3268 SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) { 3269 if (link->device_softc == disk) { 3270 port = link->target; 3271 if (link->lun > 0) 3272 my->pmp_port = link->lun - 1; 3273 else 3274 my->pmp_port = 0; 3275 3276 break; 3277 } 3278 } 3279 if (port == -1) { 3280 /* don't know where the disk is */ 3281 return (EIO); 3282 } 3283 3284 my->ap = sc->sc_ports[port]; 3285 3286 /* we're going to use the first command slot, 3287 * so ensure it's not already in use 3288 */ 3289 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 3290 /* this shouldn't happen, we should be idle */ 3291 return (EIO); 3292 } 3293 3294 /* stop the port so we can relocate to the hibernate page */ 3295 if (ahci_port_stop(my->ap, 1)) { 3296 return (EIO); 3297 } 3298 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 3299 3300 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 3301 3302 /* Setup RFIS base address */ 3303 item_phys = page_phys + ((void *)&my->rfis - page); 3304 ahci_pwrite(my->ap, AHCI_PREG_FBU, 3305 (u_int32_t)(item_phys >> 32)); 3306 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 3307 3308 /* Enable FIS reception and activate port. */ 3309 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3310 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 3311 AHCI_PREG_CMD_SUD; 3312 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 3313 AHCI_PREG_CMD_ICC_ACTIVE); 3314 3315 /* Check whether port activated. */ 3316 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3317 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 3318 return (EIO); 3319 } 3320 3321 /* Set up the single CCB */ 3322 my->ccb = &my->ccb_buf; 3323 my->ccb->ccb_slot = 0; 3324 my->ccb->ccb_port = my->ap; 3325 3326 /* Setup command list base address */ 3327 item_phys = page_phys + ((void *)&my->cmd_hdr - page); 3328 ahci_pwrite(my->ap, AHCI_PREG_CLBU, 3329 (u_int32_t)(item_phys >> 32)); 3330 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 3331 3332 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 3333 3334 /* use existing cmd table - moving to a new one fails */ 3335 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 3336 pmap_extract(pmap_kernel(), 3337 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 3338 &page_phys); 3339 item_phys = page_phys; 3340 #if 0 3341 /* use cmd table in hibernate page (doesn't work) */ 3342 my->ccb->ccb_cmd_table = &my->cmd_table; 3343 item_phys = page_phys + ((void *)&my->cmd_table - page); 3344 #endif 3345 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 3346 3347 my->ccb->ccb_xa.fis = 3348 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 3349 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 3350 my->ccb->ccb_xa.tag = 0; 3351 3352 /* Wait for ICC change to complete */ 3353 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 3354 3355 if (ahci_port_start(my->ap, 0)) { 3356 return (EIO); 3357 } 3358 3359 /* Flush interrupts for port */ 3360 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 3361 AHCI_PREG_IS)); 3362 ahci_write(sc, AHCI_REG_IS, 1 << port); 3363 3364 ahci_enable_interrupts(my->ap); 3365 return (0); 3366 } else if (op == HIB_DONE) { 3367 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 3368 return (0); 3369 } 3370 3371 if (blkno > my->psize) 3372 return (E2BIG); 3373 blkno += my->poffset; 3374 3375 /* build fis */ 3376 sector_count = size / 512; /* dlg promises this is okay */ 3377 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 3378 fis = my->ccb->ccb_xa.fis; 3379 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 3380 fis->lba_low = blkno & 0xff; 3381 fis->lba_mid = (blkno >> 8) & 0xff; 3382 fis->lba_high = (blkno >> 16) & 0xff; 3383 3384 if (sector_count > 0x100 || blkno > 0xfffffff) { 3385 /* Use LBA48 */ 3386 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 3387 ATA_C_READDMA_EXT; 3388 fis->device = ATA_H2D_DEVICE_LBA; 3389 fis->lba_low_exp = (blkno >> 24) & 0xff; 3390 fis->lba_mid_exp = (blkno >> 32) & 0xff; 3391 fis->lba_high_exp = (blkno >> 40) & 0xff; 3392 fis->sector_count = sector_count & 0xff; 3393 fis->sector_count_exp = (sector_count >> 8) & 0xff; 3394 } else { 3395 /* Use LBA */ 3396 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 3397 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 3398 fis->sector_count = sector_count & 0xff; 3399 } 3400 3401 my->ccb->ccb_xa.data = (void *)addr; 3402 my->ccb->ccb_xa.datalen = size; 3403 my->ccb->ccb_xa.pmp_port = my->pmp_port; 3404 my->ccb->ccb_xa.flags |= ATA_F_POLL; 3405 3406 cmd_slot = my->ccb->ccb_cmd_hdr; 3407 flags = 5; /* FIS length (in DWORDs) */ 3408 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 3409 3410 if (op == HIB_W) 3411 flags |= AHCI_CMD_LIST_FLAG_W; 3412 3413 htolem16(&cmd_slot->flags, flags); 3414 3415 ahci_hibernate_load_prdt(my->ccb); 3416 3417 ahci_hibernate_io_start(my->ap, my->ccb); 3418 timeout = 1000000; 3419 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 3420 delay(1); 3421 timeout--; 3422 if (timeout == 0) { 3423 return (EIO); 3424 } 3425 } 3426 3427 return (0); 3428 } 3429 3430 #endif 3431