1 /* $OpenBSD: ahci.c,v 1.32 2017/08/21 21:43:46 jmatthew Exp $ */ 2 3 /* 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/buf.h> 24 #include <sys/kernel.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/queue.h> 28 #include <sys/mutex.h> 29 #include <sys/pool.h> 30 31 #include <machine/bus.h> 32 33 #include <dev/ic/ahcireg.h> 34 #include <dev/ic/ahcivar.h> 35 36 #ifdef AHCI_DEBUG 37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 38 while (0) 39 #define AHCI_D_TIMEOUT 0x00 40 #define AHCI_D_VERBOSE 0x01 41 #define AHCI_D_INTR 0x02 42 #define AHCI_D_XFER 0x08 43 int ahcidebug = AHCI_D_VERBOSE; 44 #else 45 #define DPRINTF(m, f...) 46 #endif 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disk.h> 52 #include <sys/disklabel.h> 53 54 #include <scsi/scsi_all.h> 55 #include <scsi/scsiconf.h> 56 57 void ahci_hibernate_io_start(struct ahci_port *, 58 struct ahci_ccb *); 59 int ahci_hibernate_io_poll(struct ahci_port *, 60 struct ahci_ccb *); 61 void ahci_hibernate_load_prdt(struct ahci_ccb *); 62 63 int ahci_hibernate_io(dev_t dev, daddr_t blkno, 64 vaddr_t addr, size_t size, int wr, void *page); 65 #endif 66 67 struct cfdriver ahci_cd = { 68 NULL, "ahci", DV_DULL 69 }; 70 71 void ahci_enable_interrupts(struct ahci_port *); 72 73 int ahci_init(struct ahci_softc *); 74 int ahci_port_alloc(struct ahci_softc *, u_int); 75 void ahci_port_detect(struct ahci_softc *, u_int); 76 void ahci_port_free(struct ahci_softc *, u_int); 77 int ahci_port_init(struct ahci_softc *, u_int); 78 79 int ahci_default_port_start(struct ahci_port *, int); 80 int ahci_port_stop(struct ahci_port *, int); 81 int ahci_port_clo(struct ahci_port *); 82 int ahci_port_softreset(struct ahci_port *); 83 void ahci_port_comreset(struct ahci_port *); 84 int ahci_port_portreset(struct ahci_port *, int); 85 void ahci_port_portreset_start(struct ahci_port *); 86 int ahci_port_portreset_poll(struct ahci_port *); 87 void ahci_port_portreset_wait(struct ahci_port *); 88 int ahci_port_portreset_finish(struct ahci_port *, int); 89 int ahci_port_signature(struct ahci_port *); 90 int ahci_pmp_port_softreset(struct ahci_port *, int); 91 int ahci_pmp_port_portreset(struct ahci_port *, int); 92 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 93 94 int ahci_load_prdt(struct ahci_ccb *); 95 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 96 u_int32_t, u_int32_t); 97 void ahci_unload_prdt(struct ahci_ccb *); 98 99 int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 100 void ahci_start(struct ahci_ccb *); 101 102 void ahci_issue_pending_ncq_commands(struct ahci_port *); 103 void ahci_issue_pending_commands(struct ahci_port *, int); 104 105 int ahci_intr(void *); 106 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 107 108 struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 109 void ahci_put_ccb(struct ahci_ccb *); 110 111 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 112 void ahci_put_err_ccb(struct ahci_ccb *); 113 114 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 115 void ahci_put_pmp_ccb(struct ahci_ccb *); 116 117 int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 118 119 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 120 void ahci_dmamem_free(struct ahci_softc *, 121 struct ahci_dmamem *); 122 123 u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 124 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 125 int ahci_wait_ne(struct ahci_softc *, bus_size_t, 126 u_int32_t, u_int32_t); 127 128 u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 129 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 130 int ahci_pwait_eq(struct ahci_port *, bus_size_t, 131 u_int32_t, u_int32_t, int); 132 void ahci_flush_tfd(struct ahci_port *ap); 133 u_int32_t ahci_active_mask(struct ahci_port *); 134 int ahci_port_detect_pmp(struct ahci_port *); 135 void ahci_pmp_probe_timeout(void *); 136 137 /* pmp operations */ 138 int ahci_pmp_read(struct ahci_port *, int, int, 139 u_int32_t *); 140 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 141 int ahci_pmp_phy_status(struct ahci_port *, int, 142 u_int32_t *); 143 int ahci_pmp_identify(struct ahci_port *, int *); 144 145 146 /* Wait for all bits in _b to be cleared */ 147 #define ahci_pwait_clr(_ap, _r, _b, _n) \ 148 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 149 150 /* Wait for all bits in _b to be set */ 151 #define ahci_pwait_set(_ap, _r, _b, _n) \ 152 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 153 154 155 156 /* provide methods for atascsi to call */ 157 int ahci_ata_probe(void *, int, int); 158 void ahci_ata_free(void *, int, int); 159 struct ata_xfer * ahci_ata_get_xfer(void *, int); 160 void ahci_ata_put_xfer(struct ata_xfer *); 161 void ahci_ata_cmd(struct ata_xfer *); 162 163 struct atascsi_methods ahci_atascsi_methods = { 164 ahci_ata_probe, 165 ahci_ata_free, 166 ahci_ata_get_xfer, 167 ahci_ata_put_xfer, 168 ahci_ata_cmd 169 }; 170 171 /* ccb completions */ 172 void ahci_ata_cmd_done(struct ahci_ccb *); 173 void ahci_pmp_cmd_done(struct ahci_ccb *); 174 void ahci_ata_cmd_timeout(void *); 175 void ahci_empty_done(struct ahci_ccb *); 176 177 int 178 ahci_attach(struct ahci_softc *sc) 179 { 180 struct atascsi_attach_args aaa; 181 u_int32_t pi; 182 int i, j, done; 183 184 if (sc->sc_port_start == NULL) 185 sc->sc_port_start = ahci_default_port_start; 186 187 if (ahci_init(sc) != 0) { 188 /* error already printed by ahci_init */ 189 goto unmap; 190 } 191 192 printf("\n"); 193 194 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 195 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 196 #ifdef AHCI_DEBUG 197 if (ahcidebug & AHCI_D_VERBOSE) { 198 const char *gen; 199 200 switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 201 case AHCI_REG_CAP_ISS_G1: 202 gen = "1 (1.5Gbps)"; 203 break; 204 case AHCI_REG_CAP_ISS_G2: 205 gen = "2 (3.0Gb/s)"; 206 break; 207 case AHCI_REG_CAP_ISS_G3: 208 gen = "3 (6.0Gb/s)"; 209 break; 210 default: 211 gen = "unknown"; 212 break; 213 } 214 215 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 216 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 217 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 218 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc), 219 ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2); 220 } 221 #endif 222 223 pi = ahci_read(sc, AHCI_REG_PI); 224 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 225 DEVNAME(sc), pi); 226 227 #ifdef AHCI_COALESCE 228 /* Naive coalescing support - enable for all ports. */ 229 if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 230 u_int16_t ccc_timeout = 20; 231 u_int8_t ccc_numcomplete = 12; 232 u_int32_t ccc_ctl; 233 234 /* disable coalescing during reconfiguration. */ 235 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 236 ccc_ctl &= ~0x00000001; 237 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 238 239 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 240 if (pi & sc->sc_ccc_mask) { 241 /* A conflict with the implemented port list? */ 242 printf("%s: coalescing interrupt/implemented port list " 243 "conflict, PI: %08x, ccc_mask: %08x\n", 244 DEVNAME(sc), pi, sc->sc_ccc_mask); 245 sc->sc_ccc_mask = 0; 246 goto noccc; 247 } 248 249 /* ahci_port_start will enable each port when it starts. */ 250 sc->sc_ccc_ports = pi; 251 sc->sc_ccc_ports_cur = 0; 252 253 /* program thresholds and enable overall coalescing. */ 254 ccc_ctl &= ~0xffffff00; 255 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 256 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 257 ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 258 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 259 } 260 noccc: 261 #endif 262 /* 263 * Given that ahci_port_alloc() will grab one CCB for error recovery 264 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds 265 * pretend at least 2 command slots for devices without NCQ support. 266 * That way, also at least 1 slot is made available for atascsi(4). 267 */ 268 sc->sc_ncmds = max(2, sc->sc_ncmds); 269 for (i = 0; i < AHCI_MAX_PORTS; i++) { 270 if (!ISSET(pi, 1 << i)) { 271 /* dont allocate stuff if the port isnt implemented */ 272 continue; 273 } 274 275 if (ahci_port_alloc(sc, i) == ENOMEM) 276 goto freeports; 277 278 if (sc->sc_ports[i] != NULL) 279 ahci_port_portreset_start(sc->sc_ports[i]); 280 } 281 282 /* 283 * Poll for device detection until all ports report a device, or one 284 * second has elapsed. 285 */ 286 for (i = 0; i < 1000; i++) { 287 done = 1; 288 for (j = 0; j < AHCI_MAX_PORTS; j++) { 289 if (sc->sc_ports[j] == NULL) 290 continue; 291 292 if (ahci_port_portreset_poll(sc->sc_ports[j])) 293 done = 0; 294 } 295 296 if (done) 297 break; 298 299 delay(1000); 300 } 301 302 /* 303 * Finish device detection on all ports that initialized. 304 */ 305 for (i = 0; i < AHCI_MAX_PORTS; i++) { 306 if (sc->sc_ports[i] != NULL) 307 ahci_port_detect(sc, i); 308 } 309 310 memset(&aaa, 0, sizeof(aaa)); 311 aaa.aaa_cookie = sc; 312 aaa.aaa_methods = &ahci_atascsi_methods; 313 aaa.aaa_minphys = NULL; 314 aaa.aaa_nports = AHCI_MAX_PORTS; 315 aaa.aaa_ncmds = sc->sc_ncmds - 1; 316 if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 317 sc->sc_ncmds > 2 && 318 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 319 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 320 } 321 322 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 323 324 /* Enable interrupts */ 325 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 326 327 return 0; 328 329 freeports: 330 for (i = 0; i < AHCI_MAX_PORTS; i++) { 331 if (sc->sc_ports[i] != NULL) 332 ahci_port_free(sc, i); 333 } 334 unmap: 335 /* Disable controller */ 336 ahci_write(sc, AHCI_REG_GHC, 0); 337 return 1; 338 } 339 340 int 341 ahci_detach(struct ahci_softc *sc, int flags) 342 { 343 int rv, i; 344 345 if (sc->sc_atascsi != NULL) { 346 rv = atascsi_detach(sc->sc_atascsi, flags); 347 if (rv != 0) 348 return (rv); 349 } 350 351 for (i = 0; i < AHCI_MAX_PORTS; i++) { 352 if (sc->sc_ports[i] != NULL) 353 ahci_port_free(sc, i); 354 } 355 356 return (0); 357 } 358 359 int 360 ahci_activate(struct device *self, int act) 361 { 362 struct ahci_softc *sc = (struct ahci_softc *)self; 363 int i, rv = 0; 364 365 switch (act) { 366 case DVACT_RESUME: 367 /* enable ahci (global interrupts disabled) */ 368 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 369 370 /* restore BIOS initialised parameters */ 371 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 372 373 for (i = 0; i < AHCI_MAX_PORTS; i++) { 374 if (sc->sc_ports[i] != NULL) 375 ahci_port_init(sc, i); 376 } 377 378 /* Enable interrupts */ 379 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 380 381 rv = config_activate_children(self, act); 382 break; 383 case DVACT_POWERDOWN: 384 rv = config_activate_children(self, act); 385 for (i = 0; i < AHCI_MAX_PORTS; i++) { 386 if (sc->sc_ports[i] != NULL) 387 ahci_port_stop(sc->sc_ports[i], 1); 388 } 389 break; 390 default: 391 rv = config_activate_children(self, act); 392 break; 393 } 394 return (rv); 395 } 396 397 int 398 ahci_init(struct ahci_softc *sc) 399 { 400 u_int32_t reg, cap, pi; 401 const char *revision; 402 403 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 404 AHCI_FMT_GHC); 405 406 /* save BIOS initialised parameters, enable staggered spin up */ 407 cap = ahci_read(sc, AHCI_REG_CAP); 408 cap &= AHCI_REG_CAP_SMPS; 409 cap |= AHCI_REG_CAP_SSS; 410 pi = ahci_read(sc, AHCI_REG_PI); 411 412 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 413 /* reset the controller */ 414 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 415 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 416 AHCI_REG_GHC_HR) != 0) { 417 printf(" unable to reset controller\n"); 418 return (1); 419 } 420 } 421 422 /* enable ahci (global interrupts disabled) */ 423 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 424 425 /* restore parameters */ 426 ahci_write(sc, AHCI_REG_CAP, cap); 427 ahci_write(sc, AHCI_REG_PI, pi); 428 429 /* check the revision */ 430 reg = ahci_read(sc, AHCI_REG_VS); 431 switch (reg) { 432 case AHCI_REG_VS_0_95: 433 revision = "0.95"; 434 break; 435 case AHCI_REG_VS_1_0: 436 revision = "1.0"; 437 break; 438 case AHCI_REG_VS_1_1: 439 revision = "1.1"; 440 break; 441 case AHCI_REG_VS_1_2: 442 revision = "1.2"; 443 break; 444 case AHCI_REG_VS_1_3: 445 revision = "1.3"; 446 break; 447 case AHCI_REG_VS_1_3_1: 448 revision = "1.3.1"; 449 break; 450 451 default: 452 printf(" unsupported AHCI revision 0x%08x\n", reg); 453 return (1); 454 } 455 456 printf(" AHCI %s", revision); 457 458 return (0); 459 } 460 461 void 462 ahci_enable_interrupts(struct ahci_port *ap) 463 { 464 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 465 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 466 AHCI_PREG_IE_UFE | 467 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 468 #ifdef AHCI_COALESCE 469 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 470 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 471 #else 472 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 473 #endif 474 ); 475 } 476 477 int 478 ahci_port_alloc(struct ahci_softc *sc, u_int port) 479 { 480 struct ahci_port *ap; 481 struct ahci_ccb *ccb; 482 u_int64_t dva; 483 u_int32_t cmd; 484 struct ahci_cmd_hdr *hdr; 485 struct ahci_cmd_table *table; 486 int i, rc = ENOMEM; 487 488 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 489 if (ap == NULL) { 490 printf("%s: unable to allocate memory for port %d\n", 491 DEVNAME(sc), port); 492 goto reterr; 493 } 494 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 495 if (ap->ap_err_scratch == NULL) { 496 printf("%s: unable to allocate DMA scratch buf for port %d\n", 497 DEVNAME(sc), port); 498 free(ap, M_DEVBUF, sizeof(*ap)); 499 goto reterr; 500 } 501 502 #ifdef AHCI_DEBUG 503 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 504 DEVNAME(sc), port); 505 #endif 506 ap->ap_port = port; 507 sc->sc_ports[port] = ap; 508 509 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 510 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 511 printf("%s: unable to create register window for port %d\n", 512 DEVNAME(sc), port); 513 goto freeport; 514 } 515 516 ap->ap_sc = sc; 517 #ifdef AHCI_COALESCE 518 ap->ap_num = port; 519 #endif 520 TAILQ_INIT(&ap->ap_ccb_free); 521 TAILQ_INIT(&ap->ap_ccb_pending); 522 mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 523 524 /* Disable port interrupts */ 525 ahci_pwrite(ap, AHCI_PREG_IE, 0); 526 527 /* Sec 10.1.2 - deinitialise port if it is already running */ 528 cmd = ahci_pread(ap, AHCI_PREG_CMD); 529 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 530 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 531 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 532 int r; 533 534 r = ahci_port_stop(ap, 1); 535 if (r) { 536 printf("%s: unable to disable %s, ignoring port %d\n", 537 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 538 rc = ENXIO; 539 goto freeport; 540 } 541 542 /* Write DET to zero */ 543 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 544 } 545 546 /* Allocate RFIS */ 547 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 548 if (ap->ap_dmamem_rfis == NULL) 549 goto nomem; 550 551 /* Setup RFIS base address */ 552 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 553 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 554 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 555 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 556 557 /* Enable FIS reception and activate port. */ 558 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 559 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 560 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 561 562 /* Check whether port activated. Skip it if not. */ 563 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 564 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 565 rc = ENXIO; 566 goto freeport; 567 } 568 569 /* Allocate a CCB for each command slot */ 570 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 571 M_DEVBUF, M_NOWAIT | M_ZERO); 572 if (ap->ap_ccbs == NULL) { 573 printf("%s: unable to allocate command list for port %d\n", 574 DEVNAME(sc), port); 575 goto freeport; 576 } 577 578 /* Command List Structures and Command Tables */ 579 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 580 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 581 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 582 sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 583 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 584 nomem: 585 printf("%s: unable to allocate DMA memory for port %d\n", 586 DEVNAME(sc), port); 587 goto freeport; 588 } 589 590 /* Setup command list base address */ 591 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 592 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 593 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 594 595 /* Split CCB allocation into CCBs and assign to command header/table */ 596 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 597 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 598 for (i = 0; i < sc->sc_ncmds; i++) { 599 ccb = &ap->ap_ccbs[i]; 600 601 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 602 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 603 &ccb->ccb_dmamap) != 0) { 604 printf("%s: unable to create dmamap for port %d " 605 "ccb %d\n", DEVNAME(sc), port, i); 606 goto freeport; 607 } 608 609 ccb->ccb_slot = i; 610 ccb->ccb_port = ap; 611 ccb->ccb_cmd_hdr = &hdr[i]; 612 ccb->ccb_cmd_table = &table[i]; 613 htolem64(&ccb->ccb_cmd_hdr->ctba, 614 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 615 ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 616 617 ccb->ccb_xa.fis = 618 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 619 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 620 ccb->ccb_xa.tag = i; 621 622 ccb->ccb_xa.state = ATA_S_COMPLETE; 623 ahci_put_ccb(ccb); 624 } 625 626 /* grab a ccb for use during error recovery */ 627 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 628 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 629 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 630 631 /* Wait for ICC change to complete */ 632 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 633 rc = 0; 634 635 freeport: 636 if (rc != 0) 637 ahci_port_free(sc, port); 638 reterr: 639 return (rc); 640 } 641 642 void 643 ahci_port_detect(struct ahci_softc *sc, u_int port) 644 { 645 struct ahci_port *ap; 646 const char *speed; 647 int rc; 648 649 ap = sc->sc_ports[port]; 650 651 rc = ahci_port_portreset_finish(ap, 1); 652 switch (rc) { 653 case ENODEV: 654 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 655 case AHCI_PREG_SSTS_DET_DEV_NE: 656 printf("%s: device not communicating on port %d\n", 657 DEVNAME(sc), port); 658 break; 659 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 660 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 661 port); 662 break; 663 default: 664 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 665 "on port %d\n", DEVNAME(sc), port); 666 break; 667 } 668 goto freeport; 669 670 case EBUSY: 671 printf("%s: device on port %d didn't come ready, " 672 "TFD: 0x%b\n", DEVNAME(sc), port, 673 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 674 675 /* Try a soft reset to clear busy */ 676 rc = ahci_port_softreset(ap); 677 if (rc) { 678 printf("%s: unable to communicate " 679 "with device on port %d\n", DEVNAME(sc), port); 680 goto freeport; 681 } 682 break; 683 684 default: 685 break; 686 } 687 688 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 689 DEVNAME(sc), port, rc); 690 691 /* Read current link speed */ 692 switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { 693 case AHCI_PREG_SSTS_SPD_GEN1: 694 speed = "1.5Gb/s"; 695 break; 696 case AHCI_PREG_SSTS_SPD_GEN2: 697 speed = "3.0Gb/s"; 698 break; 699 case AHCI_PREG_SSTS_SPD_GEN3: 700 speed = "6.0Gb/s"; 701 break; 702 default: 703 speed = NULL; 704 break; 705 } 706 if (speed != NULL) 707 printf("%s: port %d: %s\n", PORTNAME(ap), port, speed); 708 709 /* Enable command transfers on port */ 710 if (ahci_port_start(ap, 0)) { 711 printf("%s: failed to start command DMA on port %d, " 712 "disabling\n", DEVNAME(sc), port); 713 rc = ENXIO; /* couldn't start port */ 714 } 715 716 /* Flush interrupts for port */ 717 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 718 ahci_write(sc, AHCI_REG_IS, 1 << port); 719 720 ahci_enable_interrupts(ap); 721 freeport: 722 if (rc != 0) 723 ahci_port_free(sc, port); 724 } 725 726 void 727 ahci_port_free(struct ahci_softc *sc, u_int port) 728 { 729 struct ahci_port *ap = sc->sc_ports[port]; 730 struct ahci_ccb *ccb; 731 732 /* Ensure port is disabled and its interrupts are flushed */ 733 if (ap->ap_sc) { 734 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 735 ahci_pwrite(ap, AHCI_PREG_IE, 0); 736 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 737 ahci_write(sc, AHCI_REG_IS, 1 << port); 738 } 739 740 if (ap->ap_ccb_err) 741 ahci_put_ccb(ap->ap_ccb_err); 742 743 if (ap->ap_ccbs) { 744 while ((ccb = ahci_get_ccb(ap)) != NULL) 745 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 746 free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb)); 747 } 748 749 if (ap->ap_dmamem_cmd_list) 750 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 751 if (ap->ap_dmamem_rfis) 752 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 753 if (ap->ap_dmamem_cmd_table) 754 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 755 if (ap->ap_err_scratch) 756 dma_free(ap->ap_err_scratch, DEV_BSIZE); 757 758 /* bus_space(9) says we dont free the subregions handle */ 759 760 free(ap, M_DEVBUF, sizeof(*ap)); 761 sc->sc_ports[port] = NULL; 762 } 763 764 int 765 ahci_port_init(struct ahci_softc *sc, u_int port) 766 { 767 struct ahci_port *ap; 768 u_int64_t dva; 769 u_int32_t cmd; 770 int rc = ENOMEM; 771 772 ap = sc->sc_ports[port]; 773 #ifdef AHCI_DEBUG 774 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 775 DEVNAME(sc), port); 776 #endif 777 778 /* Disable port interrupts */ 779 ahci_pwrite(ap, AHCI_PREG_IE, 0); 780 781 /* Sec 10.1.2 - deinitialise port if it is already running */ 782 cmd = ahci_pread(ap, AHCI_PREG_CMD); 783 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 784 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 785 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 786 int r; 787 788 r = ahci_port_stop(ap, 1); 789 if (r) { 790 printf("%s: unable to disable %s, ignoring port %d\n", 791 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 792 rc = ENXIO; 793 goto reterr; 794 } 795 796 /* Write DET to zero */ 797 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 798 } 799 800 /* Setup RFIS base address */ 801 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 802 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 803 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 804 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 805 806 /* Enable FIS reception and activate port. */ 807 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 808 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 809 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 810 811 /* Check whether port activated. Skip it if not. */ 812 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 813 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 814 rc = ENXIO; 815 goto reterr; 816 } 817 818 /* Setup command list base address */ 819 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 820 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 821 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 822 823 /* Wait for ICC change to complete */ 824 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 825 826 /* Reset port */ 827 rc = ahci_port_portreset(ap, 1); 828 switch (rc) { 829 case ENODEV: 830 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 831 case AHCI_PREG_SSTS_DET_DEV_NE: 832 printf("%s: device not communicating on port %d\n", 833 DEVNAME(sc), port); 834 break; 835 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 836 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 837 port); 838 break; 839 default: 840 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 841 "on port %d\n", DEVNAME(sc), port); 842 break; 843 } 844 goto reterr; 845 846 case EBUSY: 847 printf("%s: device on port %d didn't come ready, " 848 "TFD: 0x%b\n", DEVNAME(sc), port, 849 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 850 851 /* Try a soft reset to clear busy */ 852 rc = ahci_port_softreset(ap); 853 if (rc) { 854 printf("%s: unable to communicate " 855 "with device on port %d\n", DEVNAME(sc), port); 856 goto reterr; 857 } 858 break; 859 860 default: 861 break; 862 } 863 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 864 DEVNAME(sc), port); 865 866 if (ap->ap_pmp_ports > 0) { 867 int p; 868 869 for (p = 0; p < ap->ap_pmp_ports; p++) { 870 int sig; 871 872 /* might need to do a portreset first here? */ 873 874 /* softreset the port */ 875 if (ahci_pmp_port_softreset(ap, p)) { 876 printf("%s.%d: unable to probe PMP port due to" 877 " softreset failure\n", PORTNAME(ap), p); 878 continue; 879 } 880 881 sig = ahci_port_signature(ap); 882 printf("%s.%d: port signature returned %d\n", 883 PORTNAME(ap), p, sig); 884 } 885 } 886 887 /* Enable command transfers on port */ 888 if (ahci_port_start(ap, 0)) { 889 printf("%s: failed to start command DMA on port %d, " 890 "disabling\n", DEVNAME(sc), port); 891 rc = ENXIO; /* couldn't start port */ 892 } 893 894 /* Flush interrupts for port */ 895 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 896 ahci_write(sc, AHCI_REG_IS, 1 << port); 897 898 ahci_enable_interrupts(ap); 899 900 reterr: 901 return (rc); 902 } 903 904 int 905 ahci_default_port_start(struct ahci_port *ap, int fre_only) 906 { 907 u_int32_t r; 908 909 /* Turn on FRE (and ST) */ 910 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 911 r |= AHCI_PREG_CMD_FRE; 912 if (!fre_only) 913 r |= AHCI_PREG_CMD_ST; 914 ahci_pwrite(ap, AHCI_PREG_CMD, r); 915 916 #ifdef AHCI_COALESCE 917 /* (Re-)enable coalescing on the port. */ 918 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 919 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 920 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 921 ap->ap_sc->sc_ccc_ports_cur); 922 } 923 #endif 924 925 /* Wait for CR to come on */ 926 if (!fre_only && 927 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 928 return (1); 929 930 return (0); 931 } 932 933 int 934 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 935 { 936 u_int32_t r; 937 938 #ifdef AHCI_COALESCE 939 /* Disable coalescing on the port while it is stopped. */ 940 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 941 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 942 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 943 ap->ap_sc->sc_ccc_ports_cur); 944 } 945 #endif 946 947 /* Turn off ST (and FRE) */ 948 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 949 r &= ~AHCI_PREG_CMD_ST; 950 if (stop_fis_rx) 951 r &= ~AHCI_PREG_CMD_FRE; 952 ahci_pwrite(ap, AHCI_PREG_CMD, r); 953 954 /* Wait for CR to go off */ 955 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 956 return (1); 957 958 /* Wait for FR to go off */ 959 if (stop_fis_rx && 960 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 961 return (2); 962 963 return (0); 964 } 965 966 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 967 int 968 ahci_port_clo(struct ahci_port *ap) 969 { 970 struct ahci_softc *sc = ap->ap_sc; 971 u_int32_t cmd; 972 973 /* Only attempt CLO if supported by controller */ 974 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 975 return (1); 976 977 /* Issue CLO */ 978 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 979 #ifdef DIAGNOSTIC 980 if (ISSET(cmd, AHCI_PREG_CMD_ST)) 981 printf("%s: CLO requested while port running\n", PORTNAME(ap)); 982 #endif 983 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 984 985 /* Wait for completion */ 986 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 987 printf("%s: CLO did not complete\n", PORTNAME(ap)); 988 return (1); 989 } 990 991 return (0); 992 } 993 994 /* AHCI soft reset, Section 10.4.1 */ 995 int 996 ahci_port_softreset(struct ahci_port *ap) 997 { 998 struct ahci_ccb *ccb = NULL; 999 struct ahci_cmd_hdr *cmd_slot; 1000 u_int8_t *fis; 1001 int s, rc = EIO, oldstate; 1002 u_int32_t cmd; 1003 1004 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 1005 1006 s = splbio(); 1007 oldstate = ap->ap_state; 1008 ap->ap_state = AP_S_ERROR_RECOVERY; 1009 1010 /* Save previous command register state */ 1011 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1012 1013 /* Idle port */ 1014 if (ahci_port_stop(ap, 0)) { 1015 printf("%s: failed to stop port, cannot softreset\n", 1016 PORTNAME(ap)); 1017 goto err; 1018 } 1019 1020 /* Request CLO if device appears hung */ 1021 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 1022 AHCI_PREG_TFD_STS_DRQ)) 1023 ahci_port_clo(ap); 1024 1025 /* Clear port errors to permit TFD transfer */ 1026 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1027 1028 /* Restart port */ 1029 if (ahci_port_start(ap, 0)) { 1030 printf("%s: failed to start port, cannot softreset\n", 1031 PORTNAME(ap)); 1032 goto err; 1033 } 1034 1035 /* Check whether CLO worked */ 1036 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1037 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1038 printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 1039 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 1040 ? "failed" : "unsupported"); 1041 rc = EBUSY; 1042 goto err; 1043 } 1044 1045 /* Prep first D2H command with SRST feature & clear busy/reset flags */ 1046 ccb = ahci_get_err_ccb(ap); 1047 cmd_slot = ccb->ccb_cmd_hdr; 1048 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1049 1050 fis = ccb->ccb_cmd_table->cfis; 1051 fis[0] = ATA_FIS_TYPE_H2D; 1052 fis[15] = ATA_FIS_CONTROL_SRST; 1053 1054 cmd_slot->prdtl = 0; 1055 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1056 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1057 AHCI_CMD_LIST_FLAG_W); 1058 1059 ccb->ccb_xa.state = ATA_S_PENDING; 1060 if (ahci_poll(ccb, 1000, NULL) != 0) 1061 goto err; 1062 1063 /* Prep second D2H command to read status and complete reset sequence */ 1064 fis[0] = ATA_FIS_TYPE_H2D; 1065 fis[15] = 0; 1066 1067 cmd_slot->prdtl = 0; 1068 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 1069 1070 ccb->ccb_xa.state = ATA_S_PENDING; 1071 if (ahci_poll(ccb, 1000, NULL) != 0) 1072 goto err; 1073 1074 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1075 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 1076 printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1077 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 1078 AHCI_PFMT_TFD_STS); 1079 rc = EBUSY; 1080 goto err; 1081 } 1082 1083 rc = 0; 1084 err: 1085 if (ccb != NULL) { 1086 /* Abort our command, if it failed, by stopping command DMA. */ 1087 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 1088 printf("%s: stopping the port, softreset slot %d was " 1089 "still active.\n", PORTNAME(ap), ccb->ccb_slot); 1090 ahci_port_stop(ap, 0); 1091 } 1092 ccb->ccb_xa.state = ATA_S_ERROR; 1093 ahci_put_err_ccb(ccb); 1094 } 1095 1096 /* Restore saved CMD register state */ 1097 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1098 ap->ap_state = oldstate; 1099 1100 splx(s); 1101 1102 return (rc); 1103 } 1104 1105 int 1106 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 1107 { 1108 struct ahci_ccb *ccb = NULL; 1109 u_int32_t data; 1110 int count; 1111 int rc; 1112 int s; 1113 struct ahci_cmd_hdr *cmd_slot; 1114 u_int8_t *fis; 1115 1116 s = splbio(); 1117 /* ignore spurious IFS errors while resetting */ 1118 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 1119 ap->ap_pmp_ignore_ifs = 1; 1120 1121 count = 2; 1122 rc = 0; 1123 do { 1124 if (ccb != NULL) { 1125 ahci_put_pmp_ccb(ccb); 1126 ccb = NULL; 1127 } 1128 1129 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1130 printf("%s.%d: unable to clear PHY status\n", 1131 PORTNAME(ap), pmp_port); 1132 } 1133 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1134 /* maybe don't do this on the first loop: */ 1135 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1136 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1137 1138 /* send first softreset FIS */ 1139 ccb = ahci_get_pmp_ccb(ap); 1140 cmd_slot = ccb->ccb_cmd_hdr; 1141 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1142 1143 fis = ccb->ccb_cmd_table->cfis; 1144 fis[0] = ATA_FIS_TYPE_H2D; 1145 fis[1] = pmp_port; 1146 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1147 1148 cmd_slot->prdtl = 0; 1149 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1150 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1151 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1152 1153 ccb->ccb_xa.state = ATA_S_PENDING; 1154 1155 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 1156 PORTNAME(ap), pmp_port); 1157 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1158 printf("%s.%d: PMP port softreset cmd failed\n", 1159 PORTNAME(ap), pmp_port); 1160 rc = EBUSY; 1161 if (count > 0) { 1162 /* probably delay a while to allow 1163 * it to settle down? 1164 */ 1165 } 1166 continue; 1167 } 1168 1169 /* send signature FIS */ 1170 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1171 fis[0] = ATA_FIS_TYPE_H2D; 1172 fis[1] = pmp_port; 1173 fis[15] = ATA_FIS_CONTROL_4BIT; 1174 1175 cmd_slot->prdtl = 0; 1176 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1177 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1178 1179 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 1180 PORTNAME(ap), pmp_port); 1181 ccb->ccb_xa.state = ATA_S_PENDING; 1182 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1183 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 1184 "failed\n", PORTNAME(ap), pmp_port); 1185 rc = EBUSY; 1186 if (count > 0) { 1187 /* sleep a while? */ 1188 } 1189 continue; 1190 } 1191 1192 fis[15] = 0; 1193 break; 1194 } while (count--); 1195 1196 if (ccb != NULL) { 1197 ahci_put_pmp_ccb(ccb); 1198 ccb = NULL; 1199 } 1200 1201 /* clean up a bit */ 1202 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1203 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1204 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1205 ap->ap_pmp_ignore_ifs = 0; 1206 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 1207 splx(s); 1208 1209 return (rc); 1210 } 1211 1212 int 1213 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 1214 { 1215 int sig; 1216 1217 ap->ap_state = AP_S_PMP_PORT_PROBE; 1218 1219 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 1220 pmp_port); 1221 if (ahci_pmp_port_portreset(ap, pmp_port)) { 1222 printf("%s.%d: unable to probe PMP port; portreset failed\n", 1223 PORTNAME(ap), pmp_port); 1224 ap->ap_state = AP_S_NORMAL; 1225 return (ATA_PORT_T_NONE); 1226 } 1227 1228 if (ahci_pmp_port_softreset(ap, pmp_port)) { 1229 printf("%s.%d: unable to probe PMP port due to softreset " 1230 "failure\n", PORTNAME(ap), pmp_port); 1231 ap->ap_state = AP_S_NORMAL; 1232 return (ATA_PORT_T_NONE); 1233 } 1234 1235 sig = ahci_port_signature(ap); 1236 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 1237 PORTNAME(ap), pmp_port, sig); 1238 ap->ap_state = AP_S_NORMAL; 1239 return (sig); 1240 } 1241 1242 1243 void 1244 ahci_flush_tfd(struct ahci_port *ap) 1245 { 1246 u_int32_t r; 1247 1248 r = ahci_pread(ap, AHCI_PREG_SERR); 1249 if (r & AHCI_PREG_SERR_DIAG_X) 1250 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1251 } 1252 1253 u_int32_t 1254 ahci_active_mask(struct ahci_port *ap) 1255 { 1256 u_int32_t mask; 1257 1258 mask = ahci_pread(ap, AHCI_PREG_CI); 1259 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 1260 mask |= ahci_pread(ap, AHCI_PREG_SACT); 1261 return mask; 1262 } 1263 1264 void 1265 ahci_pmp_probe_timeout(void *cookie) 1266 { 1267 struct ahci_ccb *ccb = cookie; 1268 struct ahci_port *ap = ccb->ccb_port; 1269 u_int32_t mask; 1270 1271 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 1272 switch (ccb->ccb_xa.state) { 1273 case ATA_S_PENDING: 1274 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1275 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1276 break; 1277 1278 case ATA_S_ONCHIP: 1279 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 1280 /* clear the command on-chip */ 1281 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 1282 ap->ap_sactive == 0); 1283 ahci_port_stop(ap, 0); 1284 ahci_port_start(ap, 0); 1285 1286 if (ahci_active_mask(ap) != 0) { 1287 ahci_port_stop(ap, 0); 1288 ahci_port_start(ap, 0); 1289 mask = ahci_active_mask(ap); 1290 if (mask != 0) { 1291 printf("%s: ahci_pmp_probe_timeout: failed to " 1292 "clear active cmds: %08x\n", PORTNAME(ap), 1293 mask); 1294 } 1295 } 1296 1297 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1298 ap->ap_active &= ~(1 << ccb->ccb_slot); 1299 KASSERT(ap->ap_active_cnt > 0); 1300 --ap->ap_active_cnt; 1301 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n", 1302 PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt); 1303 break; 1304 1305 default: 1306 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 1307 PORTNAME(ap), ccb->ccb_xa.state); 1308 } 1309 } 1310 1311 int 1312 ahci_port_signature(struct ahci_port *ap) 1313 { 1314 u_int32_t sig; 1315 1316 sig = ahci_pread(ap, AHCI_PREG_SIG); 1317 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 1318 return (ATA_PORT_T_ATAPI); 1319 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 1320 0xffff0000)) 1321 return (ATA_PORT_T_PM); 1322 else 1323 return (ATA_PORT_T_DISK); 1324 } 1325 1326 int 1327 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 1328 { 1329 u_int32_t cmd, data; 1330 int loop; 1331 int rc = 1; 1332 int s; 1333 1334 s = splbio(); 1335 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 1336 pmp_port); 1337 1338 /* Save previous command register state */ 1339 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1340 1341 /* turn off power management and disable the PHY */ 1342 data = AHCI_PREG_SCTL_IPM_DISABLED; 1343 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 1344 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 1345 goto err; 1346 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1347 goto err; 1348 delay(10000); 1349 1350 /* start COMRESET */ 1351 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1352 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1353 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 1354 pmp_port); 1355 data |= AHCI_PREG_SCTL_SPD_GEN1; 1356 } else 1357 data |= AHCI_PREG_SCTL_SPD_ANY; 1358 1359 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1360 goto err; 1361 1362 /* give it a while to settle down */ 1363 delay(100000); 1364 1365 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1366 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 1367 pmp_port); 1368 } 1369 1370 /* start trying to negotiate */ 1371 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1372 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 1373 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1374 goto err; 1375 1376 /* give it a while to detect */ 1377 for (loop = 3; loop; --loop) { 1378 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1379 goto err; 1380 if (data & AHCI_PREG_SSTS_DET) 1381 break; 1382 delay(100000); 1383 } 1384 if (loop == 0) { 1385 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 1386 goto err; 1387 } 1388 1389 /* give it even longer to fully negotiate */ 1390 for (loop = 30; loop; --loop) { 1391 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1392 goto err; 1393 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 1394 break; 1395 delay(100000); 1396 } 1397 1398 if (loop == 0) { 1399 printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 1400 pmp_port); 1401 goto err; 1402 } 1403 1404 /* device detected */ 1405 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 1406 pmp_port); 1407 1408 /* clean up a bit */ 1409 delay(100000); 1410 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1411 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1412 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1413 1414 rc = 0; 1415 err: 1416 /* Restore preserved port state */ 1417 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1418 splx(s); 1419 return (rc); 1420 } 1421 1422 /* AHCI port reset, Section 10.4.2 */ 1423 1424 void 1425 ahci_port_comreset(struct ahci_port *ap) 1426 { 1427 u_int32_t r; 1428 1429 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1430 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1431 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 1432 r |= AHCI_PREG_SCTL_SPD_GEN1; 1433 } else 1434 r |= AHCI_PREG_SCTL_SPD_ANY; 1435 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1436 delay(10000); /* wait at least 1ms for COMRESET to be sent */ 1437 r &= ~AHCI_PREG_SCTL_DET_INIT; 1438 r |= AHCI_PREG_SCTL_DET_NONE; 1439 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1440 delay(10000); 1441 } 1442 1443 void 1444 ahci_port_portreset_start(struct ahci_port *ap) 1445 { 1446 int s; 1447 1448 s = splbio(); 1449 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 1450 1451 /* Save previous command register state */ 1452 ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1453 1454 /* Clear ST, ignoring failure */ 1455 ahci_port_stop(ap, 0); 1456 1457 /* Perform device detection */ 1458 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 1459 delay(10000); 1460 ahci_port_comreset(ap); 1461 splx(s); 1462 } 1463 1464 int 1465 ahci_port_portreset_poll(struct ahci_port *ap) 1466 { 1467 if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) != 1468 AHCI_PREG_SSTS_DET_DEV) 1469 return (EAGAIN); 1470 return (0); 1471 } 1472 1473 void 1474 ahci_port_portreset_wait(struct ahci_port *ap) 1475 { 1476 int i; 1477 1478 for (i = 0; i < 1000; i++) { 1479 if (ahci_port_portreset_poll(ap) == 0) 1480 break; 1481 delay(1000); 1482 } 1483 } 1484 1485 int 1486 ahci_port_portreset_finish(struct ahci_port *ap, int pmp) 1487 { 1488 int rc, s, retries = 0; 1489 1490 s = splbio(); 1491 retry: 1492 if (ahci_port_portreset_poll(ap)) { 1493 rc = ENODEV; 1494 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 1495 /* this may be a port multiplier with no device 1496 * on port 0, so still do the pmp check if requested. 1497 */ 1498 } else { 1499 goto err; 1500 } 1501 } else { 1502 /* Clear SERR (incl X bit), so TFD can update */ 1503 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1504 1505 /* Wait for device to become ready */ 1506 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1507 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 1508 /* even if the device doesn't wake up, check if there's 1509 * a port multiplier there 1510 */ 1511 if (retries == 0) { 1512 retries = 1; 1513 ahci_port_comreset(ap); 1514 ahci_port_portreset_wait(ap); 1515 goto retry; 1516 } 1517 rc = EBUSY; 1518 } else { 1519 rc = 0; 1520 } 1521 } 1522 1523 if (pmp != 0) { 1524 if (ahci_port_detect_pmp(ap)) { 1525 /* reset again without pmp support */ 1526 pmp = 0; 1527 retries = 0; 1528 ahci_port_comreset(ap); 1529 ahci_port_portreset_wait(ap); 1530 goto retry; 1531 } 1532 } 1533 1534 err: 1535 /* Restore preserved port state */ 1536 ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd); 1537 ap->ap_saved_cmd = 0; 1538 splx(s); 1539 1540 return (rc); 1541 } 1542 1543 int 1544 ahci_port_portreset(struct ahci_port *ap, int pmp) 1545 { 1546 ahci_port_portreset_start(ap); 1547 ahci_port_portreset_wait(ap); 1548 return (ahci_port_portreset_finish(ap, pmp)); 1549 } 1550 1551 int 1552 ahci_port_detect_pmp(struct ahci_port *ap) 1553 { 1554 int count, pmp_rc, rc; 1555 u_int32_t r, cmd; 1556 struct ahci_cmd_hdr *cmd_slot; 1557 struct ahci_ccb *ccb = NULL; 1558 u_int8_t *fis = NULL; 1559 1560 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 1561 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 1562 return 0; 1563 } 1564 1565 rc = 0; 1566 pmp_rc = 0; 1567 count = 2; 1568 do { 1569 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 1570 count); 1571 if (ccb != NULL) { 1572 ahci_put_pmp_ccb(ccb); 1573 ccb = NULL; 1574 } 1575 ahci_port_stop(ap, 0); 1576 ap->ap_state = AP_S_PMP_PROBE; 1577 1578 /* set PMA in cmd reg */ 1579 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1580 if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 1581 cmd |= AHCI_PREG_CMD_PMA; 1582 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1583 } 1584 1585 /* Flush errors and request CLO unconditionally, 1586 * then start the port 1587 */ 1588 r = ahci_pread(ap, AHCI_PREG_SERR); 1589 if (r & AHCI_PREG_SERR_DIAG_X) 1590 ahci_pwrite(ap, AHCI_PREG_SERR, 1591 AHCI_PREG_SERR_DIAG_X); 1592 1593 /* Request CLO */ 1594 ahci_port_clo(ap); 1595 1596 /* Clear port errors to permit TFD transfer */ 1597 r = ahci_pread(ap, AHCI_PREG_SERR); 1598 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1599 1600 /* Restart port */ 1601 if (ahci_port_start(ap, 0)) { 1602 rc = EBUSY; 1603 printf("%s: failed to start port, cannot probe PMP\n", 1604 PORTNAME(ap)); 1605 break; 1606 } 1607 1608 /* Check whether CLO worked */ 1609 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1610 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1611 u_int32_t cap; 1612 1613 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 1614 printf("%s: CLO %s, need port reset\n", 1615 PORTNAME(ap), 1616 ISSET(cap, AHCI_REG_CAP_SCLO) 1617 ? "failed" : "unsupported"); 1618 pmp_rc = EBUSY; 1619 break; 1620 } 1621 1622 /* Prep first command with SRST feature & 1623 * clear busy/reset flags 1624 */ 1625 ccb = ahci_get_pmp_ccb(ap); 1626 cmd_slot = ccb->ccb_cmd_hdr; 1627 memset(ccb->ccb_cmd_table, 0, 1628 sizeof(struct ahci_cmd_table)); 1629 1630 fis = ccb->ccb_cmd_table->cfis; 1631 fis[0] = ATA_FIS_TYPE_H2D; 1632 fis[1] = SATA_PMP_CONTROL_PORT; 1633 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1634 1635 cmd_slot->prdtl = 0; 1636 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1637 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1638 AHCI_CMD_LIST_FLAG_PMP); 1639 1640 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 1641 PORTNAME(ap)); 1642 ccb->ccb_xa.state = ATA_S_PENDING; 1643 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1644 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 1645 PORTNAME(ap)); 1646 pmp_rc = EBUSY; 1647 continue; 1648 } 1649 1650 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1651 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1652 printf("%s: port busy after first PMP probe FIS\n", 1653 PORTNAME(ap)); 1654 } 1655 1656 /* clear errors in case the device 1657 * didn't reset cleanly 1658 */ 1659 ahci_flush_tfd(ap); 1660 r = ahci_pread(ap, AHCI_PREG_SERR); 1661 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1662 1663 /* Prep second command to read status and 1664 * complete reset sequence 1665 */ 1666 memset(ccb->ccb_cmd_table, 0, 1667 sizeof(struct ahci_cmd_table)); 1668 fis[0] = ATA_FIS_TYPE_H2D; 1669 fis[1] = SATA_PMP_CONTROL_PORT; 1670 fis[15] = ATA_FIS_CONTROL_4BIT; 1671 1672 cmd_slot->prdtl = 0; 1673 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1674 AHCI_CMD_LIST_FLAG_PMP); 1675 1676 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 1677 PORTNAME(ap)); 1678 ccb->ccb_xa.state = ATA_S_PENDING; 1679 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1680 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 1681 "cmd failed\n", PORTNAME(ap)); 1682 pmp_rc = EBUSY; 1683 continue; 1684 } 1685 1686 /* apparently we need to retry at least once 1687 * to get the right signature 1688 */ 1689 fis[15] = 0; 1690 pmp_rc = 0; 1691 } while (--count); 1692 1693 if (ccb != NULL) { 1694 ahci_put_pmp_ccb(ccb); 1695 ccb = NULL; 1696 } 1697 1698 if (ap->ap_state == AP_S_PMP_PROBE) { 1699 ap->ap_state = AP_S_NORMAL; 1700 } 1701 1702 if (pmp_rc == 0) { 1703 if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 1704 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 1705 PORTNAME(ap)); 1706 pmp_rc = EBUSY; 1707 } else { 1708 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 1709 PORTNAME(ap)); 1710 } 1711 } 1712 1713 if (pmp_rc == 0) { 1714 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 1715 pmp_rc = EBUSY; 1716 } else { 1717 rc = 0; 1718 } 1719 } 1720 1721 /* if PMP detection failed, so turn off the PMA bit and 1722 * reset the port again 1723 */ 1724 if (pmp_rc != 0) { 1725 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 1726 "the port\n", PORTNAME(ap)); 1727 ahci_port_stop(ap, 0); 1728 ahci_port_clo(ap); 1729 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1730 cmd &= ~AHCI_PREG_CMD_PMA; 1731 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1732 1733 ahci_pwrite(ap, AHCI_PREG_IE, 0); 1734 ahci_port_stop(ap, 0); 1735 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 1736 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 1737 ahci_flush_tfd(ap); 1738 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1739 1740 ahci_pwrite(ap, AHCI_PREG_IS, -1); 1741 1742 ahci_enable_interrupts(ap); 1743 1744 rc = pmp_rc; 1745 } 1746 1747 return (rc); 1748 } 1749 1750 void 1751 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1752 u_int32_t flags) 1753 { 1754 flags |= len - 1; 1755 1756 htolem64(&prd->dba, addr); 1757 htolem32(&prd->flags, flags); 1758 } 1759 1760 int 1761 ahci_load_prdt(struct ahci_ccb *ccb) 1762 { 1763 struct ahci_port *ap = ccb->ccb_port; 1764 struct ahci_softc *sc = ap->ap_sc; 1765 struct ata_xfer *xa = &ccb->ccb_xa; 1766 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1767 bus_dmamap_t dmap = ccb->ccb_dmamap; 1768 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1769 int i, error; 1770 1771 if (xa->datalen == 0) { 1772 ccb->ccb_cmd_hdr->prdtl = 0; 1773 return (0); 1774 } 1775 1776 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 1777 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1778 if (error != 0) { 1779 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1780 return (1); 1781 } 1782 1783 for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1784 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1785 dmap->dm_segs[i].ds_len, 0); 1786 } 1787 1788 ahci_load_prdt_seg(&prdt[i], 1789 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1790 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 1791 1792 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 1793 1794 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1795 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 1796 BUS_DMASYNC_PREWRITE); 1797 1798 return (0); 1799 } 1800 1801 void 1802 ahci_unload_prdt(struct ahci_ccb *ccb) 1803 { 1804 struct ahci_port *ap = ccb->ccb_port; 1805 struct ahci_softc *sc = ap->ap_sc; 1806 struct ata_xfer *xa = &ccb->ccb_xa; 1807 bus_dmamap_t dmap = ccb->ccb_dmamap; 1808 1809 if (xa->datalen != 0) { 1810 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1811 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 1812 BUS_DMASYNC_POSTWRITE); 1813 1814 bus_dmamap_unload(sc->sc_dmat, dmap); 1815 1816 if (ccb->ccb_xa.flags & ATA_F_NCQ) 1817 xa->resid = 0; 1818 else 1819 xa->resid = xa->datalen - 1820 lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 1821 } 1822 } 1823 1824 int 1825 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 1826 { 1827 struct ahci_port *ap = ccb->ccb_port; 1828 int s; 1829 1830 s = splbio(); 1831 ahci_start(ccb); 1832 do { 1833 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 1834 1 << ccb->ccb_slot)) { 1835 splx(s); 1836 return (0); 1837 } 1838 if (ccb->ccb_xa.state == ATA_S_ERROR) { 1839 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 1840 PORTNAME(ap), ccb->ccb_slot); 1841 /* pretend it timed out? */ 1842 if (timeout_fn != NULL) { 1843 timeout_fn(ccb); 1844 } 1845 splx(s); 1846 return (1); 1847 } 1848 1849 delay(1000); 1850 } while (--timeout > 0); 1851 1852 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 1853 if (timeout_fn != NULL) 1854 timeout_fn(ccb); 1855 1856 splx(s); 1857 1858 return (1); 1859 } 1860 1861 void 1862 ahci_start(struct ahci_ccb *ccb) 1863 { 1864 struct ahci_port *ap = ccb->ccb_port; 1865 struct ahci_softc *sc = ap->ap_sc; 1866 1867 /* Zero transferred byte count before transfer */ 1868 ccb->ccb_cmd_hdr->prdbc = 0; 1869 1870 /* Sync command list entry and corresponding command table entry */ 1871 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1872 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 1873 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 1874 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1875 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 1876 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 1877 1878 /* Prepare RFIS area for write by controller */ 1879 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 1880 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 1881 1882 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1883 /* Issue NCQ commands only when there are no outstanding 1884 * standard commands. */ 1885 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 1886 (ap->ap_sactive != 0 && 1887 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 1888 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1889 } else { 1890 KASSERT(ap->ap_active_cnt == 0); 1891 ap->ap_sactive |= (1 << ccb->ccb_slot); 1892 ccb->ccb_xa.state = ATA_S_ONCHIP; 1893 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 1894 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1895 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 1896 } 1897 } else { 1898 /* Wait for all NCQ commands to finish before issuing standard 1899 * command. */ 1900 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 1901 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1902 else if (ap->ap_active_cnt < 2) { 1903 ap->ap_active |= 1 << ccb->ccb_slot; 1904 ccb->ccb_xa.state = ATA_S_ONCHIP; 1905 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1906 ap->ap_active_cnt++; 1907 } 1908 } 1909 } 1910 1911 void 1912 ahci_issue_pending_ncq_commands(struct ahci_port *ap) 1913 { 1914 struct ahci_ccb *nextccb; 1915 u_int32_t sact_change = 0; 1916 1917 KASSERT(ap->ap_active_cnt == 0); 1918 1919 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1920 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 1921 return; 1922 1923 /* Start all the NCQ commands at the head of the pending list. 1924 * If a port multiplier is attached to the port, we can only 1925 * issue commands for one of its ports at a time. 1926 */ 1927 if (ap->ap_sactive != 0 && 1928 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 1929 return; 1930 } 1931 1932 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 1933 do { 1934 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1935 sact_change |= 1 << nextccb->ccb_slot; 1936 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1937 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1938 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 1939 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 1940 1941 ap->ap_sactive |= sact_change; 1942 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 1943 ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 1944 } 1945 1946 void 1947 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 1948 { 1949 struct ahci_ccb *nextccb; 1950 1951 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1952 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 1953 if (last_was_ncq) { 1954 KASSERT(nextccb->ccb_xa.pmp_port != 1955 ap->ap_pmp_ncq_port); 1956 /* otherwise it should have been started already */ 1957 } else { 1958 ap->ap_active_cnt--; 1959 } 1960 1961 /* Issue NCQ commands only when there are no outstanding 1962 * standard commands, and previous NCQ commands for other 1963 * PMP ports have finished. 1964 */ 1965 if (ap->ap_active == 0) 1966 ahci_issue_pending_ncq_commands(ap); 1967 else 1968 KASSERT(ap->ap_active_cnt == 1); 1969 } else if (nextccb) { 1970 if (ap->ap_sactive != 0 || last_was_ncq) 1971 KASSERT(ap->ap_active_cnt == 0); 1972 1973 /* Wait for all NCQ commands to finish before issuing standard 1974 * command. */ 1975 if (ap->ap_sactive != 0) 1976 return; 1977 1978 /* Keep up to 2 standard commands on-chip at a time. */ 1979 do { 1980 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1981 ap->ap_active |= 1 << nextccb->ccb_slot; 1982 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1983 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 1984 if (last_was_ncq) 1985 ap->ap_active_cnt++; 1986 if (ap->ap_active_cnt == 2) 1987 break; 1988 KASSERT(ap->ap_active_cnt == 1); 1989 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1990 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 1991 } else if (!last_was_ncq) { 1992 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 1993 1994 /* Standard command finished, none waiting to start. */ 1995 ap->ap_active_cnt--; 1996 } else { 1997 KASSERT(ap->ap_active_cnt == 0); 1998 1999 /* NCQ command finished. */ 2000 } 2001 } 2002 2003 int 2004 ahci_intr(void *arg) 2005 { 2006 struct ahci_softc *sc = arg; 2007 u_int32_t is, ack = 0; 2008 int port; 2009 2010 /* Read global interrupt status */ 2011 is = ahci_read(sc, AHCI_REG_IS); 2012 if (is == 0 || is == 0xffffffff) 2013 return (0); 2014 ack = is; 2015 2016 #ifdef AHCI_COALESCE 2017 /* Check coalescing interrupt first */ 2018 if (is & sc->sc_ccc_mask) { 2019 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 2020 DEVNAME(sc)); 2021 is &= ~sc->sc_ccc_mask; 2022 is |= sc->sc_ccc_ports_cur; 2023 } 2024 #endif 2025 2026 /* Process interrupts for each port */ 2027 while (is) { 2028 port = ffs(is) - 1; 2029 if (sc->sc_ports[port]) 2030 ahci_port_intr(sc->sc_ports[port], 2031 AHCI_PREG_CI_ALL_SLOTS); 2032 is &= ~(1 << port); 2033 } 2034 2035 /* Finally, acknowledge global interrupt */ 2036 ahci_write(sc, AHCI_REG_IS, ack); 2037 2038 return (1); 2039 } 2040 2041 u_int32_t 2042 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 2043 { 2044 struct ahci_softc *sc = ap->ap_sc; 2045 u_int32_t is, ci_saved, ci_masked, processed = 0; 2046 int slot, need_restart = 0; 2047 int process_error = 0; 2048 struct ahci_ccb *ccb; 2049 volatile u_int32_t *active; 2050 #ifdef DIAGNOSTIC 2051 u_int32_t tmp; 2052 #endif 2053 2054 is = ahci_pread(ap, AHCI_PREG_IS); 2055 2056 /* Ack port interrupt only if checking all command slots. */ 2057 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 2058 ahci_pwrite(ap, AHCI_PREG_IS, is); 2059 2060 if (is) 2061 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 2062 is, AHCI_PFMT_IS); 2063 2064 if (ap->ap_sactive) { 2065 /* Active NCQ commands - use SActive instead of CI */ 2066 KASSERT(ap->ap_active == 0); 2067 KASSERT(ap->ap_active_cnt == 0); 2068 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2069 active = &ap->ap_sactive; 2070 } else { 2071 /* Save CI */ 2072 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2073 active = &ap->ap_active; 2074 } 2075 2076 if (is & AHCI_PREG_IS_TFES) { 2077 process_error = 1; 2078 } else if (is & AHCI_PREG_IS_DHRS) { 2079 u_int32_t tfd; 2080 u_int32_t cmd; 2081 u_int32_t serr; 2082 2083 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2084 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2085 serr = ahci_pread(ap, AHCI_PREG_SERR); 2086 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2087 (cmd & AHCI_PREG_CMD_CR) == 0) { 2088 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 2089 " %b, DIAG: %b\n", PORTNAME(ap), tfd, 2090 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2091 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2092 AHCI_PFMT_SERR_DIAG); 2093 process_error = 1; 2094 } else { 2095 /* rfis copy back is in the normal execution path */ 2096 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2097 } 2098 } 2099 2100 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 2101 if (process_error) { 2102 u_int32_t tfd, serr; 2103 int err_slot; 2104 2105 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2106 serr = ahci_pread(ap, AHCI_PREG_SERR); 2107 2108 if (ap->ap_sactive == 0) { 2109 /* Errored slot is easy to determine from CMD. */ 2110 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 2111 AHCI_PREG_CMD)); 2112 2113 if ((ci_saved & (1 << err_slot)) == 0) { 2114 /* 2115 * Hardware doesn't seem to report correct 2116 * slot number. If there's only one 2117 * outstanding command we can cope, 2118 * otherwise fail all active commands. 2119 */ 2120 if (ap->ap_active_cnt == 1) 2121 err_slot = ffs(ap->ap_active) - 1; 2122 else 2123 goto failall; 2124 } 2125 2126 ccb = &ap->ap_ccbs[err_slot]; 2127 2128 /* Preserve received taskfile data from the RFIS. */ 2129 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2130 sizeof(struct ata_fis_d2h)); 2131 } else 2132 err_slot = -1; /* Must extract error from log page */ 2133 2134 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 2135 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 2136 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2137 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2138 AHCI_PFMT_SERR_DIAG); 2139 2140 /* Turn off ST to clear CI and SACT. */ 2141 ahci_port_stop(ap, 0); 2142 need_restart = 1; 2143 2144 /* Clear SERR to enable capturing new errors. */ 2145 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2146 2147 /* Acknowledge the interrupts we can recover from. */ 2148 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2149 AHCI_PREG_IS_IFS); 2150 is = ahci_pread(ap, AHCI_PREG_IS); 2151 2152 /* If device hasn't cleared its busy status, try to idle it. */ 2153 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2154 2155 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 2156 (ap->ap_state == AP_S_ERROR_RECOVERY)) { 2157 /* can't reset the port here, just make sure 2158 * the operation fails and the port still works. 2159 */ 2160 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 2161 printf("%s: error on PMP port %d, idling " 2162 "device\n", PORTNAME(ap), 2163 ccb->ccb_xa.pmp_port); 2164 if (ahci_pmp_port_softreset(ap, 2165 ccb->ccb_xa.pmp_port) == 0) { 2166 printf("%s: unable to softreset port " 2167 "%d\n", PORTNAME(ap), 2168 ccb->ccb_xa.pmp_port); 2169 if (ahci_pmp_port_portreset(ap, 2170 ccb->ccb_xa.pmp_port)) { 2171 printf("%s: failed to port " 2172 " reset %d, giving up on " 2173 "it\n", PORTNAME(ap), 2174 ccb->ccb_xa.pmp_port); 2175 goto fatal; 2176 } 2177 } 2178 } else { 2179 printf("%s: attempting to idle device\n", 2180 PORTNAME(ap)); 2181 if (ahci_port_softreset(ap)) { 2182 printf("%s: failed to soft reset " 2183 "device\n", PORTNAME(ap)); 2184 if (ahci_port_portreset(ap, 0)) { 2185 printf("%s: failed to port " 2186 "reset device, give up on " 2187 "it\n", PORTNAME(ap)); 2188 goto fatal; 2189 } 2190 } 2191 } 2192 2193 /* Had to reset device, can't gather extended info. */ 2194 } else if (ap->ap_sactive) { 2195 /* Recover the NCQ error from log page 10h. 2196 * We can only have queued commands active for one port 2197 * at a time, so we know which device errored. 2198 */ 2199 ahci_port_read_ncq_error(ap, &err_slot, 2200 ap->ap_pmp_ncq_port); 2201 if (err_slot < 0) 2202 goto failall; 2203 2204 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 2205 PORTNAME(ap), err_slot); 2206 2207 ccb = &ap->ap_ccbs[err_slot]; 2208 if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2209 printf("%s: NCQ errored slot %d is idle" 2210 " (%08x active)\n", PORTNAME(ap), err_slot, 2211 ci_saved); 2212 goto failall; 2213 } 2214 } else { 2215 /* Didn't reset, could gather extended info from log. */ 2216 } 2217 2218 /* 2219 * If we couldn't determine the errored slot, reset the port 2220 * and fail all the active slots. 2221 */ 2222 if (err_slot == -1) { 2223 if (ahci_port_softreset(ap) != 0 && 2224 ahci_port_portreset(ap, 0) != 0) { 2225 printf("%s: couldn't reset after NCQ error, " 2226 "disabling device.\n", PORTNAME(ap)); 2227 goto fatal; 2228 } 2229 printf("%s: couldn't recover NCQ error, failing " 2230 "all outstanding commands.\n", PORTNAME(ap)); 2231 goto failall; 2232 } 2233 2234 /* Clear the failed command in saved CI so completion runs. */ 2235 ci_saved &= ~(1 << err_slot); 2236 2237 /* Note the error in the ata_xfer. */ 2238 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2239 ccb->ccb_xa.state = ATA_S_ERROR; 2240 2241 #ifdef DIAGNOSTIC 2242 /* There may only be one outstanding standard command now. */ 2243 if (ap->ap_sactive == 0) { 2244 tmp = ci_saved; 2245 if (tmp) { 2246 slot = ffs(tmp) - 1; 2247 tmp &= ~(1 << slot); 2248 KASSERT(tmp == 0); 2249 } 2250 } 2251 #endif 2252 } 2253 2254 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 2255 * when there's a normal SATA device attached. 2256 */ 2257 if ((ap->ap_state == AP_S_PMP_PROBE) && 2258 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 2259 (is & AHCI_PREG_IS_IPMS)) { 2260 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2261 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 2262 PORTNAME(ap), slot); 2263 2264 ccb = &ap->ap_ccbs[slot]; 2265 ccb->ccb_xa.state = ATA_S_ERROR; 2266 2267 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 2268 is &= ~AHCI_PREG_IS_IPMS; 2269 } 2270 2271 /* ignore IFS errors while resetting a PMP port */ 2272 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 2273 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 2274 "port\n", PORTNAME(ap)); 2275 2276 need_restart = 1; 2277 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2278 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2279 is &= ~AHCI_PREG_IS_IFS; 2280 goto failall; 2281 } 2282 2283 /* Check for remaining errors - they are fatal. */ 2284 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2285 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2286 printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 2287 PORTNAME(ap), is, AHCI_PFMT_IS); 2288 2289 /* XXX try recovery first */ 2290 goto fatal; 2291 } 2292 2293 /* Fail all outstanding commands if we know the port won't recover. */ 2294 if (ap->ap_state == AP_S_FATAL_ERROR) { 2295 fatal: 2296 ap->ap_state = AP_S_FATAL_ERROR; 2297 failall: 2298 2299 /* Ensure port is shut down. */ 2300 ahci_port_stop(ap, 1); 2301 2302 /* Error all the active slots. */ 2303 ci_masked = ci_saved & *active; 2304 while (ci_masked) { 2305 slot = ffs(ci_masked) - 1; 2306 ccb = &ap->ap_ccbs[slot]; 2307 ci_masked &= ~(1 << slot); 2308 ccb->ccb_xa.state = ATA_S_ERROR; 2309 } 2310 2311 /* Run completion for all active slots. */ 2312 ci_saved &= ~*active; 2313 2314 /* Don't restart the port if our problems were deemed fatal. */ 2315 if (ap->ap_state == AP_S_FATAL_ERROR) 2316 need_restart = 0; 2317 } 2318 2319 /* 2320 * CCB completion is detected by noticing its slot's bit in CI has 2321 * changed to zero some time after we activated it. 2322 * If we are polling, we may only be interested in particular slot(s). 2323 */ 2324 ci_masked = ~ci_saved & *active & ci_mask; 2325 while (ci_masked) { 2326 slot = ffs(ci_masked) - 1; 2327 ccb = &ap->ap_ccbs[slot]; 2328 ci_masked &= ~(1 << slot); 2329 2330 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2331 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2332 " (error)" : ""); 2333 2334 bus_dmamap_sync(sc->sc_dmat, 2335 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2336 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 2337 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 2338 2339 bus_dmamap_sync(sc->sc_dmat, 2340 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2341 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 2342 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 2343 2344 bus_dmamap_sync(sc->sc_dmat, 2345 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 2346 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 2347 2348 *active &= ~(1 << ccb->ccb_slot); 2349 /* Copy the rfis into the ccb if we were asked for it */ 2350 if (ccb->ccb_xa.state == ATA_S_ONCHIP && 2351 ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 2352 memcpy(&ccb->ccb_xa.rfis, 2353 ap->ap_rfis->rfis, 2354 sizeof(struct ata_fis_d2h)); 2355 } 2356 2357 ccb->ccb_done(ccb); 2358 2359 processed |= 1 << ccb->ccb_slot; 2360 } 2361 2362 if (need_restart) { 2363 /* Restart command DMA on the port */ 2364 ahci_port_start(ap, 0); 2365 2366 /* Re-enable outstanding commands on port. */ 2367 if (ci_saved) { 2368 #ifdef DIAGNOSTIC 2369 tmp = ci_saved; 2370 while (tmp) { 2371 slot = ffs(tmp) - 1; 2372 tmp &= ~(1 << slot); 2373 ccb = &ap->ap_ccbs[slot]; 2374 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2375 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 2376 (!!ap->ap_sactive)); 2377 } 2378 #endif 2379 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 2380 "re-enabling%s slots %08x\n", PORTNAME(ap), 2381 ap->ap_sactive ? " NCQ" : "", ci_saved); 2382 2383 if (ap->ap_sactive) 2384 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 2385 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 2386 } 2387 } 2388 2389 return (processed); 2390 } 2391 2392 struct ahci_ccb * 2393 ahci_get_ccb(struct ahci_port *ap) 2394 { 2395 struct ahci_ccb *ccb; 2396 2397 mtx_enter(&ap->ap_ccb_mtx); 2398 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2399 if (ccb != NULL) { 2400 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2401 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2402 ccb->ccb_xa.state = ATA_S_SETUP; 2403 } 2404 mtx_leave(&ap->ap_ccb_mtx); 2405 2406 return (ccb); 2407 } 2408 2409 void 2410 ahci_put_ccb(struct ahci_ccb *ccb) 2411 { 2412 struct ahci_port *ap = ccb->ccb_port; 2413 2414 #ifdef DIAGNOSTIC 2415 if (ccb->ccb_xa.state != ATA_S_COMPLETE && 2416 ccb->ccb_xa.state != ATA_S_TIMEOUT && 2417 ccb->ccb_xa.state != ATA_S_ERROR) { 2418 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 2419 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 2420 ccb->ccb_slot); 2421 } 2422 #endif 2423 2424 ccb->ccb_xa.state = ATA_S_PUT; 2425 mtx_enter(&ap->ap_ccb_mtx); 2426 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2427 mtx_leave(&ap->ap_ccb_mtx); 2428 } 2429 2430 struct ahci_ccb * 2431 ahci_get_err_ccb(struct ahci_port *ap) 2432 { 2433 struct ahci_ccb *err_ccb; 2434 u_int32_t sact; 2435 2436 splassert(IPL_BIO); 2437 2438 /* No commands may be active on the chip. */ 2439 sact = ahci_pread(ap, AHCI_PREG_SACT); 2440 if (sact != 0) 2441 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 2442 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2443 2444 #ifdef DIAGNOSTIC 2445 KASSERT(ap->ap_err_busy == 0); 2446 ap->ap_err_busy = 1; 2447 #endif 2448 /* Save outstanding command state. */ 2449 ap->ap_err_saved_active = ap->ap_active; 2450 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2451 ap->ap_err_saved_sactive = ap->ap_sactive; 2452 2453 /* 2454 * Pretend we have no commands outstanding, so that completions won't 2455 * run prematurely. 2456 */ 2457 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2458 2459 /* 2460 * Grab a CCB to use for error recovery. This should never fail, as 2461 * we ask atascsi to reserve one for us at init time. 2462 */ 2463 err_ccb = ap->ap_ccb_err; 2464 err_ccb->ccb_xa.flags = 0; 2465 err_ccb->ccb_xa.state = ATA_S_SETUP; 2466 err_ccb->ccb_done = ahci_empty_done; 2467 2468 return (err_ccb); 2469 } 2470 2471 void 2472 ahci_put_err_ccb(struct ahci_ccb *ccb) 2473 { 2474 struct ahci_port *ap = ccb->ccb_port; 2475 u_int32_t sact; 2476 2477 splassert(IPL_BIO); 2478 2479 #ifdef DIAGNOSTIC 2480 KASSERT(ap->ap_err_busy); 2481 #endif 2482 /* No commands may be active on the chip */ 2483 sact = ahci_pread(ap, AHCI_PREG_SACT); 2484 if (sact != 0) 2485 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2486 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2487 2488 #ifdef DIAGNOSTIC 2489 /* Done with the CCB */ 2490 KASSERT(ccb == ap->ap_ccb_err); 2491 #endif 2492 2493 /* Restore outstanding command state */ 2494 ap->ap_sactive = ap->ap_err_saved_sactive; 2495 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2496 ap->ap_active = ap->ap_err_saved_active; 2497 2498 #ifdef DIAGNOSTIC 2499 ap->ap_err_busy = 0; 2500 #endif 2501 } 2502 2503 struct ahci_ccb * 2504 ahci_get_pmp_ccb(struct ahci_port *ap) 2505 { 2506 struct ahci_ccb *ccb; 2507 u_int32_t sact; 2508 2509 /* some PMP commands need to be issued on slot 1, 2510 * particularly the command that clears SRST and 2511 * fetches the device signature. 2512 * 2513 * ensure the chip is idle and ccb 1 is available. 2514 */ 2515 splassert(IPL_BIO); 2516 2517 sact = ahci_pread(ap, AHCI_PREG_SACT); 2518 if (sact != 0) 2519 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 2520 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2521 2522 ccb = &ap->ap_ccbs[1]; 2523 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2524 ccb->ccb_xa.flags = 0; 2525 ccb->ccb_done = ahci_pmp_cmd_done; 2526 2527 mtx_enter(&ap->ap_ccb_mtx); 2528 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2529 mtx_leave(&ap->ap_ccb_mtx); 2530 2531 return ccb; 2532 } 2533 2534 void 2535 ahci_put_pmp_ccb(struct ahci_ccb *ccb) 2536 { 2537 struct ahci_port *ap = ccb->ccb_port; 2538 u_int32_t sact; 2539 2540 /* make sure this is the right ccb */ 2541 KASSERT(ccb == &ap->ap_ccbs[1]); 2542 2543 /* No commands may be active on the chip */ 2544 sact = ahci_pread(ap, AHCI_PREG_SACT); 2545 if (sact != 0) 2546 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2547 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2548 2549 ccb->ccb_xa.state = ATA_S_PUT; 2550 mtx_enter(&ap->ap_ccb_mtx); 2551 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2552 mtx_leave(&ap->ap_ccb_mtx); 2553 } 2554 2555 int 2556 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 2557 { 2558 struct ahci_ccb *ccb; 2559 struct ahci_cmd_hdr *cmd_slot; 2560 u_int32_t cmd; 2561 struct ata_fis_h2d *fis; 2562 int rc = EIO, oldstate; 2563 2564 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 2565 oldstate = ap->ap_state; 2566 ap->ap_state = AP_S_ERROR_RECOVERY; 2567 2568 /* Save command register state. */ 2569 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 2570 2571 /* Port should have been idled already. Start it. */ 2572 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 2573 ahci_port_start(ap, 0); 2574 2575 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 2576 ccb = ahci_get_err_ccb(ap); 2577 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 2578 ccb->ccb_xa.data = ap->ap_err_scratch; 2579 ccb->ccb_xa.datalen = 512; 2580 cmd_slot = ccb->ccb_cmd_hdr; 2581 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2582 2583 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2584 fis->type = ATA_FIS_TYPE_H2D; 2585 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 2586 fis->command = ATA_C_READ_LOG_EXT; 2587 fis->lba_low = 0x10; /* queued error log page (10h) */ 2588 fis->sector_count = 1; /* number of sectors (1) */ 2589 fis->sector_count_exp = 0; 2590 fis->lba_mid = 0; /* starting offset */ 2591 fis->lba_mid_exp = 0; 2592 fis->device = 0; 2593 2594 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 2595 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 2596 2597 if (ahci_load_prdt(ccb) != 0) { 2598 rc = ENOMEM; /* XXX caller must abort all commands */ 2599 goto err; 2600 } 2601 2602 ccb->ccb_xa.state = ATA_S_PENDING; 2603 if (ahci_poll(ccb, 1000, NULL) != 0 || 2604 ccb->ccb_xa.state == ATA_S_ERROR) 2605 goto err; 2606 2607 rc = 0; 2608 err: 2609 /* Abort our command, if it failed, by stopping command DMA. */ 2610 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 2611 printf("%s: log page read failed, slot %d was still active.\n", 2612 PORTNAME(ap), ccb->ccb_slot); 2613 ahci_port_stop(ap, 0); 2614 } 2615 2616 /* Done with the error CCB now. */ 2617 ahci_unload_prdt(ccb); 2618 ahci_put_err_ccb(ccb); 2619 2620 /* Extract failed register set and tags from the scratch space. */ 2621 if (rc == 0) { 2622 struct ata_log_page_10h *log; 2623 int err_slot; 2624 2625 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2626 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 2627 /* Not queued bit was set - wasn't an NCQ error? */ 2628 printf("%s: read NCQ error page, but not an NCQ " 2629 "error?\n", PORTNAME(ap)); 2630 rc = ESRCH; 2631 } else { 2632 /* Copy back the log record as a D2H register FIS. */ 2633 *err_slotp = err_slot = log->err_regs.type & 2634 ATA_LOG_10H_TYPE_TAG_MASK; 2635 2636 ccb = &ap->ap_ccbs[err_slot]; 2637 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2638 sizeof(struct ata_fis_d2h)); 2639 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2640 ccb->ccb_xa.rfis.flags = 0; 2641 } 2642 } 2643 2644 /* Restore saved CMD register state */ 2645 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 2646 ap->ap_state = oldstate; 2647 2648 return (rc); 2649 } 2650 2651 struct ahci_dmamem * 2652 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 2653 { 2654 struct ahci_dmamem *adm; 2655 int nsegs; 2656 2657 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 2658 if (adm == NULL) 2659 return (NULL); 2660 2661 adm->adm_size = size; 2662 2663 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2664 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2665 goto admfree; 2666 2667 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 2668 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 2669 goto destroy; 2670 2671 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2672 &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0) 2673 goto free; 2674 2675 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 2676 NULL, BUS_DMA_NOWAIT) != 0) 2677 goto unmap; 2678 2679 return (adm); 2680 2681 unmap: 2682 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 2683 free: 2684 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2685 destroy: 2686 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2687 admfree: 2688 free(adm, M_DEVBUF, sizeof(*adm)); 2689 2690 return (NULL); 2691 } 2692 2693 void 2694 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 2695 { 2696 bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 2697 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 2698 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2699 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2700 free(adm, M_DEVBUF, sizeof(*adm)); 2701 } 2702 2703 u_int32_t 2704 ahci_read(struct ahci_softc *sc, bus_size_t r) 2705 { 2706 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2707 BUS_SPACE_BARRIER_READ); 2708 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 2709 } 2710 2711 void 2712 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 2713 { 2714 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 2715 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2716 BUS_SPACE_BARRIER_WRITE); 2717 } 2718 2719 int 2720 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 2721 u_int32_t target) 2722 { 2723 int i; 2724 2725 for (i = 0; i < 1000; i++) { 2726 if ((ahci_read(sc, r) & mask) != target) 2727 return (0); 2728 delay(1000); 2729 } 2730 2731 return (1); 2732 } 2733 2734 u_int32_t 2735 ahci_pread(struct ahci_port *ap, bus_size_t r) 2736 { 2737 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2738 BUS_SPACE_BARRIER_READ); 2739 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 2740 } 2741 2742 void 2743 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 2744 { 2745 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 2746 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2747 BUS_SPACE_BARRIER_WRITE); 2748 } 2749 2750 int 2751 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 2752 u_int32_t target, int n) 2753 { 2754 int i; 2755 2756 for (i = 0; i < n * 1000; i++) { 2757 if ((ahci_pread(ap, r) & mask) == target) 2758 return (0); 2759 delay(1000); 2760 } 2761 2762 return (1); 2763 } 2764 2765 int 2766 ahci_ata_probe(void *xsc, int port, int lun) 2767 { 2768 struct ahci_softc *sc = xsc; 2769 struct ahci_port *ap = sc->sc_ports[port]; 2770 2771 if (ap == NULL) 2772 return (ATA_PORT_T_NONE); 2773 2774 if (lun != 0) { 2775 int pmp_port = lun - 1; 2776 if (pmp_port >= ap->ap_pmp_ports) { 2777 return (ATA_PORT_T_NONE); 2778 } 2779 return (ahci_pmp_port_probe(ap, pmp_port)); 2780 } else { 2781 return (ahci_port_signature(ap)); 2782 } 2783 } 2784 2785 void 2786 ahci_ata_free(void *xsc, int port, int lun) 2787 { 2788 2789 } 2790 2791 struct ata_xfer * 2792 ahci_ata_get_xfer(void *aaa_cookie, int port) 2793 { 2794 struct ahci_softc *sc = aaa_cookie; 2795 struct ahci_port *ap = sc->sc_ports[port]; 2796 struct ahci_ccb *ccb; 2797 2798 ccb = ahci_get_ccb(ap); 2799 if (ccb == NULL) { 2800 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 2801 PORTNAME(ap)); 2802 return (NULL); 2803 } 2804 2805 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 2806 PORTNAME(ap), ccb->ccb_slot); 2807 2808 return ((struct ata_xfer *)ccb); 2809 } 2810 2811 void 2812 ahci_ata_put_xfer(struct ata_xfer *xa) 2813 { 2814 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2815 2816 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 2817 2818 ahci_put_ccb(ccb); 2819 } 2820 2821 void 2822 ahci_ata_cmd(struct ata_xfer *xa) 2823 { 2824 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2825 struct ahci_cmd_hdr *cmd_slot; 2826 int s; 2827 u_int16_t flags; 2828 2829 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 2830 goto failcmd; 2831 2832 ccb->ccb_done = ahci_ata_cmd_done; 2833 2834 cmd_slot = ccb->ccb_cmd_hdr; 2835 flags = 5 /* FIS length (in DWORDs) */; 2836 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 2837 2838 if (xa->flags & ATA_F_WRITE) 2839 flags |= AHCI_CMD_LIST_FLAG_W; 2840 2841 if (xa->flags & ATA_F_PACKET) 2842 flags |= AHCI_CMD_LIST_FLAG_A; 2843 2844 htolem16(&cmd_slot->flags, flags); 2845 2846 if (ahci_load_prdt(ccb) != 0) 2847 goto failcmd; 2848 2849 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 2850 2851 xa->state = ATA_S_PENDING; 2852 2853 if (xa->flags & ATA_F_POLL) 2854 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 2855 else { 2856 s = splbio(); 2857 timeout_add_msec(&xa->stimeout, xa->timeout); 2858 ahci_start(ccb); 2859 splx(s); 2860 } 2861 2862 return; 2863 2864 failcmd: 2865 s = splbio(); 2866 xa->state = ATA_S_ERROR; 2867 ata_complete(xa); 2868 splx(s); 2869 } 2870 2871 void 2872 ahci_pmp_cmd_done(struct ahci_ccb *ccb) 2873 { 2874 struct ata_xfer *xa = &ccb->ccb_xa; 2875 2876 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2877 ahci_issue_pending_commands(ccb->ccb_port, 2878 xa->flags & ATA_F_NCQ); 2879 2880 xa->state = ATA_S_COMPLETE; 2881 } 2882 2883 2884 void 2885 ahci_ata_cmd_done(struct ahci_ccb *ccb) 2886 { 2887 struct ata_xfer *xa = &ccb->ccb_xa; 2888 2889 timeout_del(&xa->stimeout); 2890 2891 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2892 ahci_issue_pending_commands(ccb->ccb_port, 2893 xa->flags & ATA_F_NCQ); 2894 2895 ahci_unload_prdt(ccb); 2896 2897 if (xa->state == ATA_S_ONCHIP) 2898 xa->state = ATA_S_COMPLETE; 2899 #ifdef DIAGNOSTIC 2900 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 2901 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 2902 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 2903 ccb->ccb_slot); 2904 #endif 2905 if (xa->state != ATA_S_TIMEOUT) 2906 ata_complete(xa); 2907 } 2908 2909 void 2910 ahci_ata_cmd_timeout(void *arg) 2911 { 2912 struct ahci_ccb *ccb = arg; 2913 struct ata_xfer *xa = &ccb->ccb_xa; 2914 struct ahci_port *ap = ccb->ccb_port; 2915 int s, ccb_was_started, ncq_cmd; 2916 volatile u_int32_t *active; 2917 2918 s = splbio(); 2919 2920 ncq_cmd = (xa->flags & ATA_F_NCQ); 2921 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 2922 2923 if (ccb->ccb_xa.state == ATA_S_PENDING) { 2924 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 2925 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 2926 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2927 ccb_was_started = 0; 2928 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 2929 1 << ccb->ccb_slot)) { 2930 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 2931 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 2932 goto ret; 2933 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2934 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 2935 "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 2936 ISSET(*active, 1 << ccb->ccb_slot) ? 2937 " but slot is still active?" : "."); 2938 goto ret; 2939 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 2940 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 2941 1 << ccb->ccb_slot)) { 2942 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 2943 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 2944 ccb->ccb_slot); 2945 *active &= ~(1 << ccb->ccb_slot); 2946 ccb->ccb_done(ccb); 2947 goto ret; 2948 } else { 2949 ccb_was_started = 1; 2950 } 2951 2952 /* Complete the slot with a timeout error. */ 2953 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2954 *active &= ~(1 << ccb->ccb_slot); 2955 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 2956 ccb->ccb_done(ccb); /* This won't issue pending commands or run the 2957 atascsi completion. */ 2958 2959 /* Reset port to abort running command. */ 2960 if (ccb_was_started) { 2961 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 2962 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 2963 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 2964 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 2965 != 0) { 2966 printf("%s: failed to reset port during timeout " 2967 "handling, disabling it\n", PORTNAME(ap)); 2968 ap->ap_state = AP_S_FATAL_ERROR; 2969 } 2970 2971 /* Restart any other commands that were aborted by the reset. */ 2972 if (*active) { 2973 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 2974 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 2975 *active); 2976 if (ncq_cmd) 2977 ahci_pwrite(ap, AHCI_PREG_SACT, *active); 2978 ahci_pwrite(ap, AHCI_PREG_CI, *active); 2979 } 2980 } 2981 2982 /* Issue any pending commands now. */ 2983 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 2984 if (ccb_was_started) 2985 ahci_issue_pending_commands(ap, ncq_cmd); 2986 else if (ap->ap_active == 0) 2987 ahci_issue_pending_ncq_commands(ap); 2988 2989 /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 2990 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 2991 ata_complete(xa); 2992 2993 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 2994 ret: 2995 splx(s); 2996 } 2997 2998 void 2999 ahci_empty_done(struct ahci_ccb *ccb) 3000 { 3001 if (ccb->ccb_xa.state != ATA_S_ERROR) 3002 ccb->ccb_xa.state = ATA_S_COMPLETE; 3003 } 3004 3005 int 3006 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 3007 { 3008 struct ahci_ccb *ccb; 3009 struct ata_fis_h2d *fis; 3010 int error; 3011 3012 ccb = ahci_get_pmp_ccb(ap); 3013 if (ccb == NULL) { 3014 printf("%s: NULL ccb!\n", PORTNAME(ap)); 3015 return (1); 3016 } 3017 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 3018 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 3019 ccb->ccb_xa.state = ATA_S_PENDING; 3020 3021 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3022 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3023 fis->type = ATA_FIS_TYPE_H2D; 3024 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3025 fis->command = ATA_C_READ_PM; 3026 fis->features = which; 3027 fis->device = target | ATA_H2D_DEVICE_LBA; 3028 fis->control = ATA_FIS_CONTROL_4BIT; 3029 3030 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 3031 error = 1; 3032 } else { 3033 *datap = ccb->ccb_xa.rfis.sector_count | 3034 (ccb->ccb_xa.rfis.lba_low << 8) | 3035 (ccb->ccb_xa.rfis.lba_mid << 16) | 3036 (ccb->ccb_xa.rfis.lba_high << 24); 3037 error = 0; 3038 } 3039 ahci_put_pmp_ccb(ccb); 3040 return (error); 3041 } 3042 3043 int 3044 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 3045 { 3046 struct ahci_ccb *ccb; 3047 struct ata_fis_h2d *fis; 3048 int error; 3049 3050 ccb = ahci_get_pmp_ccb(ap); 3051 if (ccb == NULL) { 3052 printf("%s: NULL ccb!\n", PORTNAME(ap)); 3053 return (1); 3054 } 3055 ccb->ccb_xa.flags = ATA_F_POLL; 3056 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 3057 ccb->ccb_xa.state = ATA_S_PENDING; 3058 3059 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3060 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3061 fis->type = ATA_FIS_TYPE_H2D; 3062 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3063 fis->command = ATA_C_WRITE_PM; 3064 fis->features = which; 3065 fis->device = target | ATA_H2D_DEVICE_LBA; 3066 fis->sector_count = (u_int8_t)data; 3067 fis->lba_low = (u_int8_t)(data >> 8); 3068 fis->lba_mid = (u_int8_t)(data >> 16); 3069 fis->lba_high = (u_int8_t)(data >> 24); 3070 fis->control = ATA_FIS_CONTROL_4BIT; 3071 3072 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 3073 ahci_put_pmp_ccb(ccb); 3074 return (error); 3075 } 3076 3077 int 3078 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 3079 { 3080 int error; 3081 3082 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 3083 if (error == 0) 3084 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 3085 if (error) 3086 *datap = 0; 3087 3088 return (error); 3089 } 3090 3091 int 3092 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 3093 { 3094 u_int32_t chipid; 3095 u_int32_t rev; 3096 u_int32_t nports; 3097 u_int32_t features; 3098 u_int32_t enabled; 3099 int s; 3100 3101 s = splbio(); 3102 3103 if (ahci_pmp_read(ap, 15, 0, &chipid) || 3104 ahci_pmp_read(ap, 15, 1, &rev) || 3105 ahci_pmp_read(ap, 15, 2, &nports) || 3106 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 3107 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 3108 printf("%s: port multiplier identification failed\n", 3109 PORTNAME(ap)); 3110 splx(s); 3111 return (1); 3112 } 3113 splx(s); 3114 3115 nports &= 0x0F; 3116 3117 /* ignore SEMB port on SiI3726 port multiplier chips */ 3118 if (chipid == 0x37261095) { 3119 nports--; 3120 } 3121 3122 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 3123 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 3124 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 3125 SATA_PFMT_PM_FEA); 3126 3127 *ret_nports = nports; 3128 return (0); 3129 } 3130 3131 3132 #ifdef HIBERNATE 3133 void 3134 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 3135 { 3136 ccb->ccb_cmd_hdr->prdbc = 0; 3137 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 3138 } 3139 3140 int 3141 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 3142 { 3143 u_int32_t is, ci_saved; 3144 int process_error = 0; 3145 3146 is = ahci_pread(ap, AHCI_PREG_IS); 3147 3148 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 3149 3150 if (is & AHCI_PREG_IS_DHRS) { 3151 u_int32_t tfd; 3152 u_int32_t cmd; 3153 u_int32_t serr; 3154 3155 tfd = ahci_pread(ap, AHCI_PREG_TFD); 3156 cmd = ahci_pread(ap, AHCI_PREG_CMD); 3157 serr = ahci_pread(ap, AHCI_PREG_SERR); 3158 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 3159 (cmd & AHCI_PREG_CMD_CR) == 0) { 3160 process_error = 1; 3161 } else { 3162 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 3163 } 3164 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 3165 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 3166 process_error = 1; 3167 } 3168 3169 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 3170 if (process_error) { 3171 3172 /* Turn off ST to clear CI and SACT. */ 3173 ahci_port_stop(ap, 0); 3174 3175 /* just return an error indicator? we can't meaningfully 3176 * recover, and on the way back out we'll DVACT_RESUME which 3177 * resets and reinits the port. 3178 */ 3179 return (EIO); 3180 } 3181 3182 /* command is finished when the bit in CI for the slot goes to 0 */ 3183 if (ci_saved & (1 << ccb->ccb_slot)) { 3184 return (EAGAIN); 3185 } 3186 3187 return (0); 3188 } 3189 3190 void 3191 ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 3192 { 3193 struct ata_xfer *xa = &ccb->ccb_xa; 3194 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd; 3195 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 3196 int i; 3197 paddr_t data_phys; 3198 u_int64_t data_bus_phys; 3199 vaddr_t data_addr; 3200 size_t seglen; 3201 size_t buflen; 3202 3203 if (xa->datalen == 0) { 3204 ccb->ccb_cmd_hdr->prdtl = 0; 3205 return; 3206 } 3207 3208 /* derived from i386/amd64 _bus_dma_load_buffer; 3209 * for amd64 the buffer will always be dma safe. 3210 */ 3211 3212 buflen = xa->datalen; 3213 data_addr = (vaddr_t)xa->data; 3214 for (i = 0; buflen > 0; i++) { 3215 prd = &prdt[i]; 3216 3217 pmap_extract(pmap_kernel(), data_addr, &data_phys); 3218 data_bus_phys = data_phys; 3219 3220 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 3221 if (buflen < seglen) 3222 seglen = buflen; 3223 3224 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3225 3226 data_addr += seglen; 3227 buflen -= seglen; 3228 } 3229 3230 htolem16(&cmd_slot->prdtl, i); 3231 } 3232 3233 int 3234 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 3235 int op, void *page) 3236 { 3237 /* we use the 'real' ahci_port and ahci_softc here, but 3238 * never write to them 3239 */ 3240 struct { 3241 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 3242 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 3243 /* cmd table isn't actually used because of mysteries */ 3244 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 3245 struct ahci_port *ap; 3246 struct ahci_ccb ccb_buf; 3247 struct ahci_ccb *ccb; 3248 struct ahci_cmd_hdr *hdr_buf; 3249 int pmp_port; 3250 daddr_t poffset; 3251 size_t psize; 3252 } *my = page; 3253 struct ata_fis_h2d *fis; 3254 u_int32_t sector_count; 3255 struct ahci_cmd_hdr *cmd_slot; 3256 int rc; 3257 int timeout; 3258 u_int16_t flags; 3259 3260 if (op == HIB_INIT) { 3261 struct device *disk; 3262 struct device *scsibus; 3263 struct ahci_softc *sc; 3264 extern struct cfdriver sd_cd; 3265 struct scsi_link *link; 3266 struct scsibus_softc *bus_sc; 3267 int port; 3268 paddr_t page_phys; 3269 u_int64_t item_phys; 3270 u_int32_t cmd; 3271 3272 my->poffset = blkno; 3273 my->psize = size; 3274 3275 /* map dev to an ahci port */ 3276 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 3277 scsibus = disk->dv_parent; 3278 sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 3279 3280 /* find the scsi_link for the device, which has the port */ 3281 port = -1; 3282 bus_sc = (struct scsibus_softc *)scsibus; 3283 SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) { 3284 if (link->device_softc == disk) { 3285 /* link->adapter_softc == sc->sc_atascsi */ 3286 port = link->target; 3287 if (link->lun > 0) 3288 my->pmp_port = link->lun - 1; 3289 else 3290 my->pmp_port = 0; 3291 3292 break; 3293 } 3294 } 3295 if (port == -1) { 3296 /* don't know where the disk is */ 3297 return (EIO); 3298 } 3299 3300 my->ap = sc->sc_ports[port]; 3301 3302 /* we're going to use the first command slot, 3303 * so ensure it's not already in use 3304 */ 3305 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 3306 /* this shouldn't happen, we should be idle */ 3307 return (EIO); 3308 } 3309 3310 /* stop the port so we can relocate to the hibernate page */ 3311 if (ahci_port_stop(my->ap, 1)) { 3312 return (EIO); 3313 } 3314 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 3315 3316 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 3317 3318 /* Setup RFIS base address */ 3319 item_phys = page_phys + ((void *)&my->rfis - page); 3320 ahci_pwrite(my->ap, AHCI_PREG_FBU, 3321 (u_int32_t)(item_phys >> 32)); 3322 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 3323 3324 /* Enable FIS reception and activate port. */ 3325 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3326 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 3327 AHCI_PREG_CMD_SUD; 3328 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 3329 AHCI_PREG_CMD_ICC_ACTIVE); 3330 3331 /* Check whether port activated. */ 3332 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3333 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 3334 return (EIO); 3335 } 3336 3337 /* Set up the single CCB */ 3338 my->ccb = &my->ccb_buf; 3339 my->ccb->ccb_slot = 0; 3340 my->ccb->ccb_port = my->ap; 3341 3342 /* Setup command list base address */ 3343 item_phys = page_phys + ((void *)&my->cmd_hdr - page); 3344 ahci_pwrite(my->ap, AHCI_PREG_CLBU, 3345 (u_int32_t)(item_phys >> 32)); 3346 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 3347 3348 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 3349 3350 /* use existing cmd table - don't know why moving to a new one fails */ 3351 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 3352 pmap_extract(pmap_kernel(), 3353 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 3354 &page_phys); 3355 item_phys = page_phys; 3356 #if 0 3357 /* use cmd table in hibernate page (doesn't work) */ 3358 my->ccb->ccb_cmd_table = &my->cmd_table; 3359 item_phys = page_phys + ((void *)&my->cmd_table - page); 3360 #endif 3361 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 3362 3363 my->ccb->ccb_xa.fis = 3364 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 3365 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 3366 my->ccb->ccb_xa.tag = 0; 3367 3368 /* Wait for ICC change to complete */ 3369 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 3370 3371 if (ahci_port_start(my->ap, 0)) { 3372 return (EIO); 3373 } 3374 3375 /* Flush interrupts for port */ 3376 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 3377 AHCI_PREG_IS)); 3378 ahci_write(sc, AHCI_REG_IS, 1 << port); 3379 3380 ahci_enable_interrupts(my->ap); 3381 return (0); 3382 } else if (op == HIB_DONE) { 3383 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 3384 return (0); 3385 } 3386 3387 if (blkno > my->psize) 3388 return (E2BIG); 3389 blkno += my->poffset; 3390 3391 /* build fis */ 3392 sector_count = size / 512; /* dlg promises this is okay */ 3393 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 3394 fis = my->ccb->ccb_xa.fis; 3395 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 3396 fis->lba_low = blkno & 0xff; 3397 fis->lba_mid = (blkno >> 8) & 0xff; 3398 fis->lba_high = (blkno >> 16) & 0xff; 3399 3400 if (sector_count > 0x100 || blkno > 0xfffffff) { 3401 /* Use LBA48 */ 3402 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 3403 ATA_C_READDMA_EXT; 3404 fis->device = ATA_H2D_DEVICE_LBA; 3405 fis->lba_low_exp = (blkno >> 24) & 0xff; 3406 fis->lba_mid_exp = (blkno >> 32) & 0xff; 3407 fis->lba_high_exp = (blkno >> 40) & 0xff; 3408 fis->sector_count = sector_count & 0xff; 3409 fis->sector_count_exp = (sector_count >> 8) & 0xff; 3410 } else { 3411 /* Use LBA */ 3412 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 3413 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 3414 fis->sector_count = sector_count & 0xff; 3415 } 3416 3417 my->ccb->ccb_xa.data = (void *)addr; 3418 my->ccb->ccb_xa.datalen = size; 3419 my->ccb->ccb_xa.pmp_port = my->pmp_port; 3420 my->ccb->ccb_xa.flags |= ATA_F_POLL; 3421 3422 cmd_slot = my->ccb->ccb_cmd_hdr; 3423 flags = 5; /* FIS length (in DWORDs) */ 3424 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 3425 3426 if (op == HIB_W) 3427 flags |= AHCI_CMD_LIST_FLAG_W; 3428 3429 htolem16(&cmd_slot->flags, flags); 3430 3431 ahci_hibernate_load_prdt(my->ccb); 3432 3433 ahci_hibernate_io_start(my->ap, my->ccb); 3434 timeout = 1000000; 3435 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 3436 delay(1); 3437 timeout--; 3438 if (timeout == 0) { 3439 return (EIO); 3440 } 3441 } 3442 3443 return (0); 3444 } 3445 3446 #endif 3447