1 /* $OpenBSD: ahci.c,v 1.24 2016/03/10 13:56:14 krw Exp $ */ 2 3 /* 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/buf.h> 24 #include <sys/kernel.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/queue.h> 28 #include <sys/mutex.h> 29 #include <sys/pool.h> 30 31 #include <machine/bus.h> 32 33 #include <dev/ic/ahcireg.h> 34 #include <dev/ic/ahcivar.h> 35 36 #ifdef AHCI_DEBUG 37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 38 while (0) 39 #define AHCI_D_TIMEOUT 0x00 40 #define AHCI_D_VERBOSE 0x01 41 #define AHCI_D_INTR 0x02 42 #define AHCI_D_XFER 0x08 43 int ahcidebug = AHCI_D_VERBOSE; 44 #else 45 #define DPRINTF(m, f...) 46 #endif 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disk.h> 52 #include <sys/disklabel.h> 53 54 #include <scsi/scsi_all.h> 55 #include <scsi/scsiconf.h> 56 57 void ahci_hibernate_io_start(struct ahci_port *, 58 struct ahci_ccb *); 59 int ahci_hibernate_io_poll(struct ahci_port *, 60 struct ahci_ccb *); 61 void ahci_hibernate_load_prdt(struct ahci_ccb *); 62 63 int ahci_hibernate_io(dev_t dev, daddr_t blkno, 64 vaddr_t addr, size_t size, int wr, void *page); 65 #endif 66 67 struct cfdriver ahci_cd = { 68 NULL, "ahci", DV_DULL 69 }; 70 71 void ahci_enable_interrupts(struct ahci_port *); 72 73 int ahci_init(struct ahci_softc *); 74 int ahci_port_alloc(struct ahci_softc *, u_int); 75 void ahci_port_free(struct ahci_softc *, u_int); 76 int ahci_port_init(struct ahci_softc *, u_int); 77 78 int ahci_default_port_start(struct ahci_port *, int); 79 int ahci_port_stop(struct ahci_port *, int); 80 int ahci_port_clo(struct ahci_port *); 81 int ahci_port_softreset(struct ahci_port *); 82 int ahci_port_portreset(struct ahci_port *, int); 83 int ahci_port_signature(struct ahci_port *); 84 int ahci_pmp_port_softreset(struct ahci_port *, int); 85 int ahci_pmp_port_portreset(struct ahci_port *, int); 86 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 87 88 int ahci_load_prdt(struct ahci_ccb *); 89 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 90 u_int32_t, u_int32_t); 91 void ahci_unload_prdt(struct ahci_ccb *); 92 93 int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 94 void ahci_start(struct ahci_ccb *); 95 96 void ahci_issue_pending_ncq_commands(struct ahci_port *); 97 void ahci_issue_pending_commands(struct ahci_port *, int); 98 99 int ahci_intr(void *); 100 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 101 102 struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 103 void ahci_put_ccb(struct ahci_ccb *); 104 105 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 106 void ahci_put_err_ccb(struct ahci_ccb *); 107 108 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 109 void ahci_put_pmp_ccb(struct ahci_ccb *); 110 111 int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 112 113 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 114 void ahci_dmamem_free(struct ahci_softc *, 115 struct ahci_dmamem *); 116 117 u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 118 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 119 int ahci_wait_ne(struct ahci_softc *, bus_size_t, 120 u_int32_t, u_int32_t); 121 122 u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 123 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 124 int ahci_pwait_eq(struct ahci_port *, bus_size_t, 125 u_int32_t, u_int32_t, int); 126 void ahci_flush_tfd(struct ahci_port *ap); 127 u_int32_t ahci_active_mask(struct ahci_port *); 128 int ahci_port_detect_pmp(struct ahci_port *); 129 void ahci_pmp_probe_timeout(void *); 130 131 /* pmp operations */ 132 int ahci_pmp_read(struct ahci_port *, int, int, 133 u_int32_t *); 134 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 135 int ahci_pmp_phy_status(struct ahci_port *, int, 136 u_int32_t *); 137 int ahci_pmp_identify(struct ahci_port *, int *); 138 139 140 /* Wait for all bits in _b to be cleared */ 141 #define ahci_pwait_clr(_ap, _r, _b, _n) \ 142 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 143 144 /* Wait for all bits in _b to be set */ 145 #define ahci_pwait_set(_ap, _r, _b, _n) \ 146 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 147 148 149 150 /* provide methods for atascsi to call */ 151 int ahci_ata_probe(void *, int, int); 152 void ahci_ata_free(void *, int, int); 153 struct ata_xfer * ahci_ata_get_xfer(void *, int); 154 void ahci_ata_put_xfer(struct ata_xfer *); 155 void ahci_ata_cmd(struct ata_xfer *); 156 157 struct atascsi_methods ahci_atascsi_methods = { 158 ahci_ata_probe, 159 ahci_ata_free, 160 ahci_ata_get_xfer, 161 ahci_ata_put_xfer, 162 ahci_ata_cmd 163 }; 164 165 /* ccb completions */ 166 void ahci_ata_cmd_done(struct ahci_ccb *); 167 void ahci_pmp_cmd_done(struct ahci_ccb *); 168 void ahci_ata_cmd_timeout(void *); 169 void ahci_empty_done(struct ahci_ccb *); 170 171 int 172 ahci_attach(struct ahci_softc *sc) 173 { 174 struct atascsi_attach_args aaa; 175 u_int32_t pi; 176 int i; 177 178 if (sc->sc_port_start == NULL) 179 sc->sc_port_start = ahci_default_port_start; 180 181 if (ahci_init(sc) != 0) { 182 /* error already printed by ahci_init */ 183 goto unmap; 184 } 185 186 printf("\n"); 187 188 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 189 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 190 #ifdef AHCI_DEBUG 191 if (ahcidebug & AHCI_D_VERBOSE) { 192 const char *gen; 193 194 switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 195 case AHCI_REG_CAP_ISS_G1: 196 gen = "1 (1.5Gbps)"; 197 break; 198 case AHCI_REG_CAP_ISS_G2: 199 gen = "2 (3.0Gb/s)"; 200 break; 201 case AHCI_REG_CAP_ISS_G3: 202 gen = "3 (6.0Gb/s)"; 203 break; 204 default: 205 gen = "unknown"; 206 break; 207 } 208 209 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 210 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 211 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 212 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc), 213 ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2); 214 } 215 #endif 216 217 pi = ahci_read(sc, AHCI_REG_PI); 218 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 219 DEVNAME(sc), pi); 220 221 #ifdef AHCI_COALESCE 222 /* Naive coalescing support - enable for all ports. */ 223 if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 224 u_int16_t ccc_timeout = 20; 225 u_int8_t ccc_numcomplete = 12; 226 u_int32_t ccc_ctl; 227 228 /* disable coalescing during reconfiguration. */ 229 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 230 ccc_ctl &= ~0x00000001; 231 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 232 233 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 234 if (pi & sc->sc_ccc_mask) { 235 /* A conflict with the implemented port list? */ 236 printf("%s: coalescing interrupt/implemented port list " 237 "conflict, PI: %08x, ccc_mask: %08x\n", 238 DEVNAME(sc), pi, sc->sc_ccc_mask); 239 sc->sc_ccc_mask = 0; 240 goto noccc; 241 } 242 243 /* ahci_port_start will enable each port when it starts. */ 244 sc->sc_ccc_ports = pi; 245 sc->sc_ccc_ports_cur = 0; 246 247 /* program thresholds and enable overall coalescing. */ 248 ccc_ctl &= ~0xffffff00; 249 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 250 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 251 ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 252 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 253 } 254 noccc: 255 #endif 256 for (i = 0; i < AHCI_MAX_PORTS; i++) { 257 if (!ISSET(pi, 1 << i)) { 258 /* dont allocate stuff if the port isnt implemented */ 259 continue; 260 } 261 262 if (ahci_port_alloc(sc, i) == ENOMEM) 263 goto freeports; 264 } 265 266 memset(&aaa, 0, sizeof(aaa)); 267 aaa.aaa_cookie = sc; 268 aaa.aaa_methods = &ahci_atascsi_methods; 269 aaa.aaa_minphys = NULL; 270 aaa.aaa_nports = AHCI_MAX_PORTS; 271 aaa.aaa_ncmds = sc->sc_ncmds - 1; 272 if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 273 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 274 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 275 /* XXX enabling ASAA_CAP_PMP_NCQ with FBS: 276 * - some error recovery work required (single device vs port 277 * errors) 278 * - probably need to look at storing our active ccb queue 279 * differently so we can group ncq and non-ncq commands 280 * for different ports. as long as we preserve the order for 281 * each port, we can reorder commands to get more ncq 282 * commands to run in parallel. 283 */ 284 } 285 286 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 287 288 /* Enable interrupts */ 289 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 290 291 return 0; 292 293 freeports: 294 for (i = 0; i < AHCI_MAX_PORTS; i++) { 295 if (sc->sc_ports[i] != NULL) 296 ahci_port_free(sc, i); 297 } 298 unmap: 299 /* Disable controller */ 300 ahci_write(sc, AHCI_REG_GHC, 0); 301 return 1; 302 } 303 304 int 305 ahci_detach(struct ahci_softc *sc, int flags) 306 { 307 int rv, i; 308 309 if (sc->sc_atascsi != NULL) { 310 rv = atascsi_detach(sc->sc_atascsi, flags); 311 if (rv != 0) 312 return (rv); 313 } 314 315 for (i = 0; i < AHCI_MAX_PORTS; i++) { 316 if (sc->sc_ports[i] != NULL) 317 ahci_port_free(sc, i); 318 } 319 320 return (0); 321 } 322 323 int 324 ahci_activate(struct device *self, int act) 325 { 326 struct ahci_softc *sc = (struct ahci_softc *)self; 327 int i, rv = 0; 328 329 switch (act) { 330 case DVACT_RESUME: 331 /* enable ahci (global interrupts disabled) */ 332 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 333 334 /* restore BIOS initialised parameters */ 335 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 336 337 for (i = 0; i < AHCI_MAX_PORTS; i++) { 338 if (sc->sc_ports[i] != NULL) 339 ahci_port_init(sc, i); 340 } 341 342 /* Enable interrupts */ 343 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 344 345 rv = config_activate_children(self, act); 346 break; 347 case DVACT_POWERDOWN: 348 rv = config_activate_children(self, act); 349 for (i = 0; i < AHCI_MAX_PORTS; i++) { 350 if (sc->sc_ports[i] != NULL) 351 ahci_port_stop(sc->sc_ports[i], 1); 352 } 353 break; 354 default: 355 rv = config_activate_children(self, act); 356 break; 357 } 358 return (rv); 359 } 360 361 int 362 ahci_init(struct ahci_softc *sc) 363 { 364 u_int32_t reg, cap, pi; 365 const char *revision; 366 367 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 368 AHCI_FMT_GHC); 369 370 /* save BIOS initialised parameters, enable staggered spin up */ 371 cap = ahci_read(sc, AHCI_REG_CAP); 372 cap &= AHCI_REG_CAP_SMPS; 373 cap |= AHCI_REG_CAP_SSS; 374 pi = ahci_read(sc, AHCI_REG_PI); 375 376 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 377 /* reset the controller */ 378 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 379 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 380 AHCI_REG_GHC_HR) != 0) { 381 printf(" unable to reset controller\n"); 382 return (1); 383 } 384 } 385 386 /* enable ahci (global interrupts disabled) */ 387 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 388 389 /* restore parameters */ 390 ahci_write(sc, AHCI_REG_CAP, cap); 391 ahci_write(sc, AHCI_REG_PI, pi); 392 393 /* check the revision */ 394 reg = ahci_read(sc, AHCI_REG_VS); 395 switch (reg) { 396 case AHCI_REG_VS_0_95: 397 revision = "0.95"; 398 break; 399 case AHCI_REG_VS_1_0: 400 revision = "1.0"; 401 break; 402 case AHCI_REG_VS_1_1: 403 revision = "1.1"; 404 break; 405 case AHCI_REG_VS_1_2: 406 revision = "1.2"; 407 break; 408 case AHCI_REG_VS_1_3: 409 revision = "1.3"; 410 break; 411 case AHCI_REG_VS_1_3_1: 412 revision = "1.3.1"; 413 break; 414 415 default: 416 printf(" unsupported AHCI revision 0x%08x\n", reg); 417 return (1); 418 } 419 420 printf(" AHCI %s", revision); 421 422 return (0); 423 } 424 425 void 426 ahci_enable_interrupts(struct ahci_port *ap) 427 { 428 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 429 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 430 AHCI_PREG_IE_UFE | 431 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 432 #ifdef AHCI_COALESCE 433 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 434 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 435 #else 436 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 437 #endif 438 ); 439 } 440 441 int 442 ahci_port_alloc(struct ahci_softc *sc, u_int port) 443 { 444 struct ahci_port *ap; 445 struct ahci_ccb *ccb; 446 u_int64_t dva; 447 u_int32_t cmd; 448 struct ahci_cmd_hdr *hdr; 449 struct ahci_cmd_table *table; 450 const char *speed; 451 int i, rc = ENOMEM; 452 453 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 454 if (ap == NULL) { 455 printf("%s: unable to allocate memory for port %d\n", 456 DEVNAME(sc), port); 457 goto reterr; 458 } 459 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 460 if (ap->ap_err_scratch == NULL) { 461 printf("%s: unable to allocate DMA scratch buf for port %d\n", 462 DEVNAME(sc), port); 463 free(ap, M_DEVBUF, sizeof(*ap)); 464 goto reterr; 465 } 466 467 #ifdef AHCI_DEBUG 468 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 469 DEVNAME(sc), port); 470 #endif 471 ap->ap_port = port; 472 sc->sc_ports[port] = ap; 473 474 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 475 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 476 printf("%s: unable to create register window for port %d\n", 477 DEVNAME(sc), port); 478 goto freeport; 479 } 480 481 ap->ap_sc = sc; 482 #ifdef AHCI_COALESCE 483 ap->ap_num = port; 484 #endif 485 TAILQ_INIT(&ap->ap_ccb_free); 486 TAILQ_INIT(&ap->ap_ccb_pending); 487 mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 488 489 /* Disable port interrupts */ 490 ahci_pwrite(ap, AHCI_PREG_IE, 0); 491 492 /* Sec 10.1.2 - deinitialise port if it is already running */ 493 cmd = ahci_pread(ap, AHCI_PREG_CMD); 494 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 495 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 496 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 497 int r; 498 499 r = ahci_port_stop(ap, 1); 500 if (r) { 501 printf("%s: unable to disable %s, ignoring port %d\n", 502 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 503 rc = ENXIO; 504 goto freeport; 505 } 506 507 /* Write DET to zero */ 508 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 509 } 510 511 /* XXX FBS - need to allocate 16x ahci_rfis struct? - but we don't 512 * know if there's a PMP attached or if the HBA supports FBS yet.. 513 * reallocate when we enable FBS? 514 */ 515 516 /* Allocate RFIS */ 517 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 518 if (ap->ap_dmamem_rfis == NULL) 519 goto nomem; 520 521 /* Setup RFIS base address */ 522 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 523 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 524 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 525 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 526 527 /* Enable FIS reception and activate port. */ 528 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 529 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 530 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 531 532 /* Check whether port activated. Skip it if not. */ 533 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 534 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 535 rc = ENXIO; 536 goto freeport; 537 } 538 539 /* Allocate a CCB for each command slot */ 540 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 541 M_DEVBUF, M_NOWAIT | M_ZERO); 542 if (ap->ap_ccbs == NULL) { 543 printf("%s: unable to allocate command list for port %d\n", 544 DEVNAME(sc), port); 545 goto freeport; 546 } 547 548 /* Command List Structures and Command Tables */ 549 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 550 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 551 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 552 sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 553 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 554 nomem: 555 printf("%s: unable to allocate DMA memory for port %d\n", 556 DEVNAME(sc), port); 557 goto freeport; 558 } 559 560 /* Setup command list base address */ 561 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 562 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 563 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 564 565 /* Split CCB allocation into CCBs and assign to command header/table */ 566 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 567 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 568 for (i = 0; i < sc->sc_ncmds; i++) { 569 ccb = &ap->ap_ccbs[i]; 570 571 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 572 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 573 &ccb->ccb_dmamap) != 0) { 574 printf("%s: unable to create dmamap for port %d " 575 "ccb %d\n", DEVNAME(sc), port, i); 576 goto freeport; 577 } 578 579 ccb->ccb_slot = i; 580 ccb->ccb_port = ap; 581 ccb->ccb_cmd_hdr = &hdr[i]; 582 ccb->ccb_cmd_table = &table[i]; 583 htolem64(&ccb->ccb_cmd_hdr->ctba, 584 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 585 ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 586 587 ccb->ccb_xa.fis = 588 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 589 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 590 ccb->ccb_xa.tag = i; 591 592 ccb->ccb_xa.state = ATA_S_COMPLETE; 593 ahci_put_ccb(ccb); 594 } 595 596 /* grab a ccb for use during error recovery */ 597 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 598 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 599 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 600 601 /* Wait for ICC change to complete */ 602 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 603 604 /* Reset port */ 605 rc = ahci_port_portreset(ap, 1); 606 607 switch (rc) { 608 case ENODEV: 609 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 610 case AHCI_PREG_SSTS_DET_DEV_NE: 611 printf("%s: device not communicating on port %d\n", 612 DEVNAME(sc), port); 613 break; 614 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 615 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 616 port); 617 break; 618 default: 619 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 620 "on port %d\n", DEVNAME(sc), port); 621 break; 622 } 623 goto freeport; 624 625 case EBUSY: 626 printf("%s: device on port %d didn't come ready, " 627 "TFD: 0x%b\n", DEVNAME(sc), port, 628 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 629 630 /* Try a soft reset to clear busy */ 631 rc = ahci_port_softreset(ap); 632 if (rc) { 633 printf("%s: unable to communicate " 634 "with device on port %d\n", DEVNAME(sc), port); 635 goto freeport; 636 } 637 break; 638 639 default: 640 break; 641 } 642 643 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 644 DEVNAME(sc), port, rc); 645 646 /* Read current link speed */ 647 switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { 648 case AHCI_PREG_SSTS_SPD_GEN1: 649 speed = "1.5Gb/s"; 650 break; 651 case AHCI_PREG_SSTS_SPD_GEN2: 652 speed = "3.0Gb/s"; 653 break; 654 case AHCI_PREG_SSTS_SPD_GEN3: 655 speed = "6.0Gb/s"; 656 break; 657 default: 658 speed = NULL; 659 break; 660 } 661 if (speed != NULL) 662 printf("%s: port %d: %s\n", PORTNAME(ap), port, speed); 663 664 /* Enable command transfers on port */ 665 if (ahci_port_start(ap, 0)) { 666 printf("%s: failed to start command DMA on port %d, " 667 "disabling\n", DEVNAME(sc), port); 668 rc = ENXIO; /* couldn't start port */ 669 } 670 671 /* Flush interrupts for port */ 672 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 673 ahci_write(sc, AHCI_REG_IS, 1 << port); 674 675 ahci_enable_interrupts(ap); 676 677 freeport: 678 if (rc != 0) 679 ahci_port_free(sc, port); 680 reterr: 681 return (rc); 682 } 683 684 void 685 ahci_port_free(struct ahci_softc *sc, u_int port) 686 { 687 struct ahci_port *ap = sc->sc_ports[port]; 688 struct ahci_ccb *ccb; 689 690 /* Ensure port is disabled and its interrupts are flushed */ 691 if (ap->ap_sc) { 692 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 693 ahci_pwrite(ap, AHCI_PREG_IE, 0); 694 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 695 ahci_write(sc, AHCI_REG_IS, 1 << port); 696 } 697 698 if (ap->ap_ccb_err) 699 ahci_put_ccb(ap->ap_ccb_err); 700 701 if (ap->ap_ccbs) { 702 while ((ccb = ahci_get_ccb(ap)) != NULL) 703 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 704 free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb)); 705 } 706 707 if (ap->ap_dmamem_cmd_list) 708 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 709 if (ap->ap_dmamem_rfis) 710 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 711 if (ap->ap_dmamem_cmd_table) 712 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 713 if (ap->ap_err_scratch) 714 dma_free(ap->ap_err_scratch, DEV_BSIZE); 715 716 /* bus_space(9) says we dont free the subregions handle */ 717 718 free(ap, M_DEVBUF, sizeof(*ap)); 719 sc->sc_ports[port] = NULL; 720 } 721 722 int 723 ahci_port_init(struct ahci_softc *sc, u_int port) 724 { 725 struct ahci_port *ap; 726 u_int64_t dva; 727 u_int32_t cmd; 728 int rc = ENOMEM; 729 730 ap = sc->sc_ports[port]; 731 #ifdef AHCI_DEBUG 732 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 733 DEVNAME(sc), port); 734 #endif 735 736 /* Disable port interrupts */ 737 ahci_pwrite(ap, AHCI_PREG_IE, 0); 738 739 /* Sec 10.1.2 - deinitialise port if it is already running */ 740 cmd = ahci_pread(ap, AHCI_PREG_CMD); 741 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 742 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 743 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 744 int r; 745 746 r = ahci_port_stop(ap, 1); 747 if (r) { 748 printf("%s: unable to disable %s, ignoring port %d\n", 749 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 750 rc = ENXIO; 751 goto reterr; 752 } 753 754 /* Write DET to zero */ 755 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 756 } 757 758 /* Setup RFIS base address */ 759 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 760 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 761 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 762 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 763 764 /* Enable FIS reception and activate port. */ 765 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 766 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 767 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 768 769 /* Check whether port activated. Skip it if not. */ 770 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 771 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 772 rc = ENXIO; 773 goto reterr; 774 } 775 776 /* Setup command list base address */ 777 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 778 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 779 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 780 781 /* Wait for ICC change to complete */ 782 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 783 784 /* Reset port */ 785 rc = ahci_port_portreset(ap, 1); 786 switch (rc) { 787 case ENODEV: 788 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 789 case AHCI_PREG_SSTS_DET_DEV_NE: 790 printf("%s: device not communicating on port %d\n", 791 DEVNAME(sc), port); 792 break; 793 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 794 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 795 port); 796 break; 797 default: 798 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 799 "on port %d\n", DEVNAME(sc), port); 800 break; 801 } 802 goto reterr; 803 804 case EBUSY: 805 printf("%s: device on port %d didn't come ready, " 806 "TFD: 0x%b\n", DEVNAME(sc), port, 807 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 808 809 /* Try a soft reset to clear busy */ 810 rc = ahci_port_softreset(ap); 811 if (rc) { 812 printf("%s: unable to communicate " 813 "with device on port %d\n", DEVNAME(sc), port); 814 goto reterr; 815 } 816 break; 817 818 default: 819 break; 820 } 821 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 822 DEVNAME(sc), port); 823 824 if (ap->ap_pmp_ports > 0) { 825 int p; 826 827 for (p = 0; p < ap->ap_pmp_ports; p++) { 828 int sig; 829 830 /* might need to do a portreset first here? */ 831 832 /* softreset the port */ 833 if (ahci_pmp_port_softreset(ap, p)) { 834 printf("%s.%d: unable to probe PMP port due to" 835 " softreset failure\n", PORTNAME(ap), p); 836 continue; 837 } 838 839 sig = ahci_port_signature(ap); 840 printf("%s.%d: port signature returned %d\n", 841 PORTNAME(ap), p, sig); 842 } 843 } 844 845 /* Enable command transfers on port */ 846 if (ahci_port_start(ap, 0)) { 847 printf("%s: failed to start command DMA on port %d, " 848 "disabling\n", DEVNAME(sc), port); 849 rc = ENXIO; /* couldn't start port */ 850 } 851 852 /* Flush interrupts for port */ 853 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 854 ahci_write(sc, AHCI_REG_IS, 1 << port); 855 856 ahci_enable_interrupts(ap); 857 858 reterr: 859 return (rc); 860 } 861 862 int 863 ahci_default_port_start(struct ahci_port *ap, int fre_only) 864 { 865 u_int32_t r; 866 867 /* XXX FBS: possibly turn FBS on here */ 868 869 /* Turn on FRE (and ST) */ 870 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 871 r |= AHCI_PREG_CMD_FRE; 872 if (!fre_only) 873 r |= AHCI_PREG_CMD_ST; 874 ahci_pwrite(ap, AHCI_PREG_CMD, r); 875 876 #ifdef AHCI_COALESCE 877 /* (Re-)enable coalescing on the port. */ 878 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 879 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 880 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 881 ap->ap_sc->sc_ccc_ports_cur); 882 } 883 #endif 884 885 /* Wait for CR to come on */ 886 if (!fre_only && 887 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 888 return (1); 889 890 return (0); 891 } 892 893 int 894 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 895 { 896 u_int32_t r; 897 898 #ifdef AHCI_COALESCE 899 /* Disable coalescing on the port while it is stopped. */ 900 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 901 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 902 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 903 ap->ap_sc->sc_ccc_ports_cur); 904 } 905 #endif 906 907 /* Turn off ST (and FRE) */ 908 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 909 r &= ~AHCI_PREG_CMD_ST; 910 if (stop_fis_rx) 911 r &= ~AHCI_PREG_CMD_FRE; 912 ahci_pwrite(ap, AHCI_PREG_CMD, r); 913 914 /* Wait for CR to go off */ 915 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 916 return (1); 917 918 /* Wait for FR to go off */ 919 if (stop_fis_rx && 920 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 921 return (2); 922 923 /* XXX FBS: possibly disable FBS here? */ 924 925 return (0); 926 } 927 928 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 929 int 930 ahci_port_clo(struct ahci_port *ap) 931 { 932 struct ahci_softc *sc = ap->ap_sc; 933 u_int32_t cmd; 934 935 /* Only attempt CLO if supported by controller */ 936 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 937 return (1); 938 939 /* Issue CLO */ 940 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 941 #ifdef DIAGNOSTIC 942 if (ISSET(cmd, AHCI_PREG_CMD_ST)) 943 printf("%s: CLO requested while port running\n", PORTNAME(ap)); 944 #endif 945 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 946 947 /* Wait for completion */ 948 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 949 printf("%s: CLO did not complete\n", PORTNAME(ap)); 950 return (1); 951 } 952 953 return (0); 954 } 955 956 /* AHCI soft reset, Section 10.4.1 */ 957 int 958 ahci_port_softreset(struct ahci_port *ap) 959 { 960 struct ahci_ccb *ccb = NULL; 961 struct ahci_cmd_hdr *cmd_slot; 962 u_int8_t *fis; 963 int s, rc = EIO, oldstate; 964 u_int32_t cmd; 965 966 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 967 968 s = splbio(); 969 oldstate = ap->ap_state; 970 ap->ap_state = AP_S_ERROR_RECOVERY; 971 972 /* Save previous command register state */ 973 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 974 975 /* Idle port */ 976 if (ahci_port_stop(ap, 0)) { 977 printf("%s: failed to stop port, cannot softreset\n", 978 PORTNAME(ap)); 979 goto err; 980 } 981 982 /* Request CLO if device appears hung */ 983 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 984 AHCI_PREG_TFD_STS_DRQ)) 985 ahci_port_clo(ap); 986 987 /* Clear port errors to permit TFD transfer */ 988 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 989 990 /* XXX FBS - need to ensure we don't enable FBS here, since we're 991 * resetting stuff 992 * (AHCI spec 9.3.8) 993 */ 994 /* Restart port */ 995 if (ahci_port_start(ap, 0)) { 996 printf("%s: failed to start port, cannot softreset\n", 997 PORTNAME(ap)); 998 goto err; 999 } 1000 1001 /* Check whether CLO worked */ 1002 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1003 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1004 printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 1005 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 1006 ? "failed" : "unsupported"); 1007 rc = EBUSY; 1008 goto err; 1009 } 1010 1011 /* Prep first D2H command with SRST feature & clear busy/reset flags */ 1012 ccb = ahci_get_err_ccb(ap); 1013 cmd_slot = ccb->ccb_cmd_hdr; 1014 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1015 1016 fis = ccb->ccb_cmd_table->cfis; 1017 fis[0] = ATA_FIS_TYPE_H2D; 1018 fis[15] = ATA_FIS_CONTROL_SRST; 1019 1020 cmd_slot->prdtl = 0; 1021 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1022 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1023 AHCI_CMD_LIST_FLAG_W); 1024 1025 ccb->ccb_xa.state = ATA_S_PENDING; 1026 if (ahci_poll(ccb, 1000, NULL) != 0) 1027 goto err; 1028 1029 /* Prep second D2H command to read status and complete reset sequence */ 1030 fis[0] = ATA_FIS_TYPE_H2D; 1031 fis[15] = 0; 1032 1033 cmd_slot->prdtl = 0; 1034 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 1035 1036 ccb->ccb_xa.state = ATA_S_PENDING; 1037 if (ahci_poll(ccb, 1000, NULL) != 0) 1038 goto err; 1039 1040 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1041 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 1042 printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1043 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 1044 AHCI_PFMT_TFD_STS); 1045 rc = EBUSY; 1046 goto err; 1047 } 1048 1049 rc = 0; 1050 err: 1051 if (ccb != NULL) { 1052 /* Abort our command, if it failed, by stopping command DMA. */ 1053 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 1054 printf("%s: stopping the port, softreset slot %d was " 1055 "still active.\n", PORTNAME(ap), ccb->ccb_slot); 1056 ahci_port_stop(ap, 0); 1057 } 1058 ccb->ccb_xa.state = ATA_S_ERROR; 1059 ahci_put_err_ccb(ccb); 1060 } 1061 1062 /* Restore saved CMD register state */ 1063 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1064 ap->ap_state = oldstate; 1065 1066 splx(s); 1067 1068 return (rc); 1069 } 1070 1071 int 1072 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 1073 { 1074 struct ahci_ccb *ccb = NULL; 1075 u_int32_t data; 1076 int count; 1077 int rc; 1078 int s; 1079 struct ahci_cmd_hdr *cmd_slot; 1080 u_int8_t *fis; 1081 1082 /* XXX FBS: ensure fbs is disabled on ap, since we're resetting 1083 * devices (AHCI section 9.3.8) 1084 */ 1085 1086 s = splbio(); 1087 /* ignore spurious IFS errors while resetting */ 1088 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 1089 ap->ap_pmp_ignore_ifs = 1; 1090 1091 count = 2; 1092 rc = 0; 1093 do { 1094 if (ccb != NULL) { 1095 ahci_put_pmp_ccb(ccb); 1096 ccb = NULL; 1097 } 1098 1099 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1100 printf("%s.%d: unable to clear PHY status\n", 1101 PORTNAME(ap), pmp_port); 1102 } 1103 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1104 /* maybe don't do this on the first loop: */ 1105 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1106 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1107 1108 /* send first softreset FIS */ 1109 ccb = ahci_get_pmp_ccb(ap); 1110 cmd_slot = ccb->ccb_cmd_hdr; 1111 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1112 1113 fis = ccb->ccb_cmd_table->cfis; 1114 fis[0] = ATA_FIS_TYPE_H2D; 1115 fis[1] = pmp_port; 1116 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1117 1118 cmd_slot->prdtl = 0; 1119 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1120 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1121 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1122 1123 ccb->ccb_xa.state = ATA_S_PENDING; 1124 1125 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 1126 PORTNAME(ap), pmp_port); 1127 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1128 printf("%s.%d: PMP port softreset cmd failed\n", 1129 PORTNAME(ap), pmp_port); 1130 rc = EBUSY; 1131 if (count > 0) { 1132 /* probably delay a while to allow 1133 * it to settle down? 1134 */ 1135 } 1136 continue; 1137 } 1138 1139 /* send signature FIS */ 1140 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1141 fis[0] = ATA_FIS_TYPE_H2D; 1142 fis[1] = pmp_port; 1143 fis[15] = ATA_FIS_CONTROL_4BIT; 1144 1145 cmd_slot->prdtl = 0; 1146 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1147 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1148 1149 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 1150 PORTNAME(ap), pmp_port); 1151 ccb->ccb_xa.state = ATA_S_PENDING; 1152 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1153 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 1154 "failed\n", PORTNAME(ap), pmp_port); 1155 rc = EBUSY; 1156 if (count > 0) { 1157 /* sleep a while? */ 1158 } 1159 continue; 1160 } 1161 1162 fis[15] = 0; 1163 break; 1164 } while (count--); 1165 1166 if (ccb != NULL) { 1167 ahci_put_pmp_ccb(ccb); 1168 ccb = NULL; 1169 } 1170 1171 /* clean up a bit */ 1172 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1173 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1174 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1175 ap->ap_pmp_ignore_ifs = 0; 1176 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 1177 splx(s); 1178 1179 return (rc); 1180 } 1181 1182 int 1183 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 1184 { 1185 int sig; 1186 1187 ap->ap_state = AP_S_PMP_PORT_PROBE; 1188 1189 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 1190 pmp_port); 1191 if (ahci_pmp_port_portreset(ap, pmp_port)) { 1192 printf("%s.%d: unable to probe PMP port; portreset failed\n", 1193 PORTNAME(ap), pmp_port); 1194 ap->ap_state = AP_S_NORMAL; 1195 return (ATA_PORT_T_NONE); 1196 } 1197 1198 if (ahci_pmp_port_softreset(ap, pmp_port)) { 1199 printf("%s.%d: unable to probe PMP port due to softreset " 1200 "failure\n", PORTNAME(ap), pmp_port); 1201 ap->ap_state = AP_S_NORMAL; 1202 return (ATA_PORT_T_NONE); 1203 } 1204 1205 sig = ahci_port_signature(ap); 1206 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 1207 PORTNAME(ap), pmp_port, sig); 1208 ap->ap_state = AP_S_NORMAL; 1209 return (sig); 1210 } 1211 1212 1213 void 1214 ahci_flush_tfd(struct ahci_port *ap) 1215 { 1216 u_int32_t r; 1217 1218 r = ahci_pread(ap, AHCI_PREG_SERR); 1219 if (r & AHCI_PREG_SERR_DIAG_X) 1220 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1221 } 1222 1223 u_int32_t 1224 ahci_active_mask(struct ahci_port *ap) 1225 { 1226 u_int32_t mask; 1227 1228 mask = ahci_pread(ap, AHCI_PREG_CI); 1229 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 1230 mask |= ahci_pread(ap, AHCI_PREG_SACT); 1231 return mask; 1232 } 1233 1234 void 1235 ahci_pmp_probe_timeout(void *cookie) 1236 { 1237 struct ahci_ccb *ccb = cookie; 1238 struct ahci_port *ap = ccb->ccb_port; 1239 u_int32_t mask; 1240 1241 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 1242 switch (ccb->ccb_xa.state) { 1243 case ATA_S_PENDING: 1244 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1245 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1246 break; 1247 1248 case ATA_S_ONCHIP: 1249 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 1250 /* clear the command on-chip */ 1251 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 1252 ap->ap_sactive == 0); 1253 ahci_port_stop(ap, 0); 1254 ahci_port_start(ap, 0); 1255 1256 if (ahci_active_mask(ap) != 0) { 1257 ahci_port_stop(ap, 0); 1258 ahci_port_start(ap, 0); 1259 mask = ahci_active_mask(ap); 1260 if (mask != 0) { 1261 printf("%s: ahci_pmp_probe_timeout: failed to " 1262 "clear active cmds: %08x\n", PORTNAME(ap), 1263 mask); 1264 } 1265 } 1266 1267 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1268 ap->ap_active &= ~(1 << ccb->ccb_slot); 1269 KASSERT(ap->ap_active_cnt > 0); 1270 --ap->ap_active_cnt; 1271 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n", 1272 PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt); 1273 break; 1274 1275 default: 1276 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 1277 PORTNAME(ap), ccb->ccb_xa.state); 1278 } 1279 } 1280 1281 int 1282 ahci_port_signature(struct ahci_port *ap) 1283 { 1284 u_int32_t sig; 1285 1286 sig = ahci_pread(ap, AHCI_PREG_SIG); 1287 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 1288 return (ATA_PORT_T_ATAPI); 1289 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 1290 0xffff0000)) 1291 return (ATA_PORT_T_PM); 1292 else 1293 return (ATA_PORT_T_DISK); 1294 } 1295 1296 int 1297 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 1298 { 1299 u_int32_t cmd, data; 1300 int loop; 1301 int rc = 1; 1302 int s; 1303 1304 s = splbio(); 1305 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 1306 pmp_port); 1307 1308 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1309 1310 /* turn off power management and disable the PHY */ 1311 data = AHCI_PREG_SCTL_IPM_DISABLED; 1312 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 1313 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 1314 goto err; 1315 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1316 goto err; 1317 delay(10000); 1318 1319 /* start COMRESET */ 1320 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1321 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1322 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 1323 pmp_port); 1324 data |= AHCI_PREG_SCTL_SPD_GEN1; 1325 } else 1326 data |= AHCI_PREG_SCTL_SPD_ANY; 1327 1328 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1329 goto err; 1330 1331 /* give it a while to settle down */ 1332 delay(100000); 1333 1334 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1335 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 1336 pmp_port); 1337 } 1338 1339 /* start trying to negotiate */ 1340 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1341 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 1342 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1343 goto err; 1344 1345 /* give it a while to detect */ 1346 for (loop = 3; loop; --loop) { 1347 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1348 goto err; 1349 if (data & AHCI_PREG_SSTS_DET) 1350 break; 1351 delay(100000); 1352 } 1353 if (loop == 0) { 1354 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 1355 goto err; 1356 } 1357 1358 /* give it even longer to fully negotiate */ 1359 for (loop = 30; loop; --loop) { 1360 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1361 goto err; 1362 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 1363 break; 1364 delay(100000); 1365 } 1366 1367 if (loop == 0) { 1368 printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 1369 pmp_port); 1370 goto err; 1371 } 1372 1373 /* device detected */ 1374 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 1375 pmp_port); 1376 1377 /* clean up a bit */ 1378 delay(100000); 1379 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1380 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1381 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1382 1383 rc = 0; 1384 err: 1385 splx(s); 1386 return (rc); 1387 } 1388 1389 /* AHCI port reset, Section 10.4.2 */ 1390 int 1391 ahci_port_portreset(struct ahci_port *ap, int pmp) 1392 { 1393 u_int32_t cmd, r; 1394 int rc, s, retries = 0; 1395 1396 s = splbio(); 1397 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 1398 1399 /* Save previous command register state */ 1400 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1401 1402 /* Clear ST, ignoring failure */ 1403 ahci_port_stop(ap, 0); 1404 1405 /* Perform device detection */ 1406 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 1407 retry: 1408 delay(10000); 1409 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1410 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1411 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 1412 r |= AHCI_PREG_SCTL_SPD_GEN1; 1413 } else 1414 r |= AHCI_PREG_SCTL_SPD_ANY; 1415 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1416 delay(10000); /* wait at least 1ms for COMRESET to be sent */ 1417 r &= ~AHCI_PREG_SCTL_DET_INIT; 1418 r |= AHCI_PREG_SCTL_DET_NONE; 1419 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1420 delay(10000); 1421 1422 /* Wait for device to be detected and communications established */ 1423 if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET, 1424 AHCI_PREG_SSTS_DET_DEV, 1)) { 1425 rc = ENODEV; 1426 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 1427 /* this may be a port multiplier with no device 1428 * on port 0, so still do the pmp check if requested. 1429 */ 1430 } else { 1431 goto err; 1432 } 1433 } else { 1434 /* Clear SERR (incl X bit), so TFD can update */ 1435 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1436 1437 /* Wait for device to become ready */ 1438 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1439 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 1440 /* even if the device doesn't wake up, check if there's 1441 * a port multiplier there 1442 */ 1443 if (retries == 0) { 1444 retries = 1; 1445 goto retry; 1446 } 1447 rc = EBUSY; 1448 } else { 1449 rc = 0; 1450 } 1451 } 1452 1453 if (pmp != 0) { 1454 if (ahci_port_detect_pmp(ap) != 0) { 1455 rc = EBUSY; 1456 } 1457 } 1458 1459 err: 1460 /* Restore preserved port state */ 1461 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1462 splx(s); 1463 1464 return (rc); 1465 } 1466 1467 int 1468 ahci_port_detect_pmp(struct ahci_port *ap) 1469 { 1470 int count, pmp_rc, rc; 1471 u_int32_t r, cmd; 1472 struct ahci_cmd_hdr *cmd_slot; 1473 struct ahci_ccb *ccb = NULL; 1474 u_int8_t *fis = NULL; 1475 1476 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 1477 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 1478 return 0; 1479 } 1480 1481 rc = 0; 1482 pmp_rc = 0; 1483 count = 2; 1484 do { 1485 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 1486 count); 1487 if (ccb != NULL) { 1488 ahci_put_pmp_ccb(ccb); 1489 ccb = NULL; 1490 } 1491 ahci_port_stop(ap, 0); 1492 ap->ap_state = AP_S_PMP_PROBE; 1493 1494 /* set PMA in cmd reg */ 1495 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1496 if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 1497 cmd |= AHCI_PREG_CMD_PMA; 1498 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1499 } 1500 1501 /* Flush errors and request CLO unconditionally, 1502 * then start the port 1503 */ 1504 r = ahci_pread(ap, AHCI_PREG_SERR); 1505 if (r & AHCI_PREG_SERR_DIAG_X) 1506 ahci_pwrite(ap, AHCI_PREG_SERR, 1507 AHCI_PREG_SERR_DIAG_X); 1508 1509 /* Request CLO */ 1510 ahci_port_clo(ap); 1511 1512 /* Clear port errors to permit TFD transfer */ 1513 r = ahci_pread(ap, AHCI_PREG_SERR); 1514 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1515 1516 /* XXX FBS: ensure we don't enable FBS here, since 1517 * we're resetting the port 1518 * (AHCI section 9.3.8) 1519 */ 1520 /* Restart port */ 1521 if (ahci_port_start(ap, 0)) { 1522 rc = EBUSY; 1523 printf("%s: failed to start port, cannot probe PMP\n", 1524 PORTNAME(ap)); 1525 break; 1526 } 1527 1528 /* Check whether CLO worked */ 1529 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1530 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1531 u_int32_t cap; 1532 1533 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 1534 printf("%s: CLO %s, need port reset\n", 1535 PORTNAME(ap), 1536 ISSET(cap, AHCI_REG_CAP_SCLO) 1537 ? "failed" : "unsupported"); 1538 pmp_rc = EBUSY; 1539 break; 1540 } 1541 1542 /* Prep first command with SRST feature & 1543 * clear busy/reset flags 1544 */ 1545 ccb = ahci_get_pmp_ccb(ap); 1546 cmd_slot = ccb->ccb_cmd_hdr; 1547 memset(ccb->ccb_cmd_table, 0, 1548 sizeof(struct ahci_cmd_table)); 1549 1550 fis = ccb->ccb_cmd_table->cfis; 1551 fis[0] = ATA_FIS_TYPE_H2D; 1552 fis[1] = SATA_PMP_CONTROL_PORT; 1553 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1554 1555 cmd_slot->prdtl = 0; 1556 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1557 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1558 AHCI_CMD_LIST_FLAG_PMP); 1559 1560 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 1561 PORTNAME(ap)); 1562 ccb->ccb_xa.state = ATA_S_PENDING; 1563 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1564 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 1565 PORTNAME(ap)); 1566 pmp_rc = EBUSY; 1567 continue; 1568 } 1569 1570 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1571 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1572 printf("%s: port busy after first PMP probe FIS\n", 1573 PORTNAME(ap)); 1574 } 1575 1576 /* clear errors in case the device 1577 * didn't reset cleanly 1578 */ 1579 ahci_flush_tfd(ap); 1580 r = ahci_pread(ap, AHCI_PREG_SERR); 1581 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1582 1583 /* Prep second command to read status and 1584 * complete reset sequence 1585 */ 1586 memset(ccb->ccb_cmd_table, 0, 1587 sizeof(struct ahci_cmd_table)); 1588 fis[0] = ATA_FIS_TYPE_H2D; 1589 fis[1] = SATA_PMP_CONTROL_PORT; 1590 fis[15] = ATA_FIS_CONTROL_4BIT; 1591 1592 cmd_slot->prdtl = 0; 1593 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1594 AHCI_CMD_LIST_FLAG_PMP); 1595 1596 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 1597 PORTNAME(ap)); 1598 ccb->ccb_xa.state = ATA_S_PENDING; 1599 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1600 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 1601 "cmd failed\n", PORTNAME(ap)); 1602 pmp_rc = EBUSY; 1603 continue; 1604 } 1605 1606 /* apparently we need to retry at least once 1607 * to get the right signature 1608 */ 1609 fis[15] = 0; 1610 pmp_rc = 0; 1611 } while (--count); 1612 1613 if (ccb != NULL) { 1614 ahci_put_pmp_ccb(ccb); 1615 ccb = NULL; 1616 } 1617 1618 if (ap->ap_state == AP_S_PMP_PROBE) { 1619 ap->ap_state = AP_S_NORMAL; 1620 } 1621 1622 if (pmp_rc == 0) { 1623 if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 1624 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 1625 PORTNAME(ap)); 1626 pmp_rc = EBUSY; 1627 } else { 1628 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 1629 PORTNAME(ap)); 1630 } 1631 } 1632 1633 if (pmp_rc == 0) { 1634 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 1635 pmp_rc = EBUSY; 1636 } else { 1637 /* XXX enable FBS if available */ 1638 rc = 0; 1639 } 1640 } 1641 1642 /* if PMP detection failed, so turn off the PMA bit and 1643 * reset the port again 1644 */ 1645 if (pmp_rc != 0) { 1646 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 1647 "the port\n", PORTNAME(ap)); 1648 ahci_port_stop(ap, 0); 1649 ahci_port_clo(ap); 1650 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1651 cmd &= ~AHCI_PREG_CMD_PMA; 1652 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1653 1654 ahci_pwrite(ap, AHCI_PREG_IE, 0); 1655 ahci_port_stop(ap, 0); 1656 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 1657 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 1658 ahci_flush_tfd(ap); 1659 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1660 1661 ahci_pwrite(ap, AHCI_PREG_IS, -1); 1662 1663 ahci_enable_interrupts(ap); 1664 1665 ahci_port_portreset(ap, 0); 1666 } 1667 1668 return (rc); 1669 } 1670 1671 void 1672 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1673 u_int32_t flags) 1674 { 1675 flags |= len - 1; 1676 1677 htolem64(&prd->dba, addr); 1678 htolem32(&prd->flags, flags); 1679 } 1680 1681 int 1682 ahci_load_prdt(struct ahci_ccb *ccb) 1683 { 1684 struct ahci_port *ap = ccb->ccb_port; 1685 struct ahci_softc *sc = ap->ap_sc; 1686 struct ata_xfer *xa = &ccb->ccb_xa; 1687 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1688 bus_dmamap_t dmap = ccb->ccb_dmamap; 1689 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1690 int i, error; 1691 1692 if (xa->datalen == 0) { 1693 ccb->ccb_cmd_hdr->prdtl = 0; 1694 return (0); 1695 } 1696 1697 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 1698 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1699 if (error != 0) { 1700 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1701 return (1); 1702 } 1703 1704 for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1705 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1706 dmap->dm_segs[i].ds_len, 0); 1707 } 1708 1709 ahci_load_prdt_seg(&prdt[i], 1710 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1711 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 1712 1713 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 1714 1715 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1716 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 1717 BUS_DMASYNC_PREWRITE); 1718 1719 return (0); 1720 } 1721 1722 void 1723 ahci_unload_prdt(struct ahci_ccb *ccb) 1724 { 1725 struct ahci_port *ap = ccb->ccb_port; 1726 struct ahci_softc *sc = ap->ap_sc; 1727 struct ata_xfer *xa = &ccb->ccb_xa; 1728 bus_dmamap_t dmap = ccb->ccb_dmamap; 1729 1730 if (xa->datalen != 0) { 1731 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1732 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 1733 BUS_DMASYNC_POSTWRITE); 1734 1735 bus_dmamap_unload(sc->sc_dmat, dmap); 1736 1737 if (ccb->ccb_xa.flags & ATA_F_NCQ) 1738 xa->resid = 0; 1739 else 1740 xa->resid = xa->datalen - 1741 lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 1742 } 1743 } 1744 1745 int 1746 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 1747 { 1748 struct ahci_port *ap = ccb->ccb_port; 1749 int s; 1750 1751 s = splbio(); 1752 ahci_start(ccb); 1753 do { 1754 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 1755 1 << ccb->ccb_slot)) { 1756 splx(s); 1757 return (0); 1758 } 1759 if (ccb->ccb_xa.state == ATA_S_ERROR) { 1760 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 1761 PORTNAME(ap), ccb->ccb_slot); 1762 /* pretend it timed out? */ 1763 if (timeout_fn != NULL) { 1764 timeout_fn(ccb); 1765 } 1766 splx(s); 1767 return (1); 1768 } 1769 1770 delay(1000); 1771 } while (--timeout > 0); 1772 1773 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 1774 if (timeout_fn != NULL) 1775 timeout_fn(ccb); 1776 1777 splx(s); 1778 1779 return (1); 1780 } 1781 1782 void 1783 ahci_start(struct ahci_ccb *ccb) 1784 { 1785 struct ahci_port *ap = ccb->ccb_port; 1786 struct ahci_softc *sc = ap->ap_sc; 1787 1788 /* Zero transferred byte count before transfer */ 1789 ccb->ccb_cmd_hdr->prdbc = 0; 1790 1791 /* Sync command list entry and corresponding command table entry */ 1792 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1793 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 1794 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 1795 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1796 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 1797 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 1798 1799 /* Prepare RFIS area for write by controller */ 1800 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 1801 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 1802 1803 /* XXX FBS: need to figure out whether we still need to keep NCQ and 1804 * non-queued commands separate when FBS is in use. I guess probably 1805 * not? it's not particularly clear from the spec.. 1806 */ 1807 1808 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1809 /* Issue NCQ commands only when there are no outstanding 1810 * standard commands. */ 1811 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 1812 (ap->ap_sactive != 0 && 1813 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 1814 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1815 } else { 1816 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV 1817 * to the port number 1818 */ 1819 1820 KASSERT(ap->ap_active_cnt == 0); 1821 ap->ap_sactive |= (1 << ccb->ccb_slot); 1822 ccb->ccb_xa.state = ATA_S_ONCHIP; 1823 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 1824 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1825 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 1826 } 1827 } else { 1828 /* Wait for all NCQ commands to finish before issuing standard 1829 * command. */ 1830 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 1831 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1832 else if (ap->ap_active_cnt < 2) { 1833 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV to the 1834 * port number 1835 */ 1836 1837 ap->ap_active |= 1 << ccb->ccb_slot; 1838 ccb->ccb_xa.state = ATA_S_ONCHIP; 1839 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1840 ap->ap_active_cnt++; 1841 } 1842 } 1843 } 1844 1845 void 1846 ahci_issue_pending_ncq_commands(struct ahci_port *ap) 1847 { 1848 struct ahci_ccb *nextccb; 1849 u_int32_t sact_change = 0; 1850 1851 KASSERT(ap->ap_active_cnt == 0); 1852 1853 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1854 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 1855 return; 1856 1857 /* XXX FBS: 1858 * - set AHCI_PREG_FBS_DEV for each command 1859 * - one write to AHCI_PREG_CI per command 1860 */ 1861 1862 /* Start all the NCQ commands at the head of the pending list. 1863 * If a port multiplier is attached to the port, we can only 1864 * issue commands for one of its ports at a time. 1865 */ 1866 if (ap->ap_sactive != 0 && 1867 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 1868 return; 1869 } 1870 1871 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 1872 do { 1873 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1874 sact_change |= 1 << nextccb->ccb_slot; 1875 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1876 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1877 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 1878 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 1879 1880 ap->ap_sactive |= sact_change; 1881 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 1882 ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 1883 } 1884 1885 void 1886 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 1887 { 1888 struct ahci_ccb *nextccb; 1889 1890 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1891 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 1892 if (last_was_ncq) { 1893 KASSERT(nextccb->ccb_xa.pmp_port != 1894 ap->ap_pmp_ncq_port); 1895 /* otherwise it should have been started already */ 1896 } else { 1897 ap->ap_active_cnt--; 1898 } 1899 1900 /* Issue NCQ commands only when there are no outstanding 1901 * standard commands, and previous NCQ commands for other 1902 * PMP ports have finished. 1903 */ 1904 if (ap->ap_active == 0) 1905 ahci_issue_pending_ncq_commands(ap); 1906 else 1907 KASSERT(ap->ap_active_cnt == 1); 1908 } else if (nextccb) { 1909 if (ap->ap_sactive != 0 || last_was_ncq) 1910 KASSERT(ap->ap_active_cnt == 0); 1911 1912 /* Wait for all NCQ commands to finish before issuing standard 1913 * command. */ 1914 if (ap->ap_sactive != 0) 1915 return; 1916 1917 /* Keep up to 2 standard commands on-chip at a time. */ 1918 do { 1919 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1920 ap->ap_active |= 1 << nextccb->ccb_slot; 1921 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1922 /* XXX FBS: set AHCI_PREG_FBS_DEV here */ 1923 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 1924 if (last_was_ncq) 1925 ap->ap_active_cnt++; 1926 if (ap->ap_active_cnt == 2) 1927 break; 1928 KASSERT(ap->ap_active_cnt == 1); 1929 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1930 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 1931 } else if (!last_was_ncq) { 1932 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 1933 1934 /* Standard command finished, none waiting to start. */ 1935 ap->ap_active_cnt--; 1936 } else { 1937 KASSERT(ap->ap_active_cnt == 0); 1938 1939 /* NCQ command finished. */ 1940 } 1941 } 1942 1943 int 1944 ahci_intr(void *arg) 1945 { 1946 struct ahci_softc *sc = arg; 1947 u_int32_t is, ack = 0; 1948 int port; 1949 1950 /* Read global interrupt status */ 1951 is = ahci_read(sc, AHCI_REG_IS); 1952 if (is == 0 || is == 0xffffffff) 1953 return (0); 1954 ack = is; 1955 1956 #ifdef AHCI_COALESCE 1957 /* Check coalescing interrupt first */ 1958 if (is & sc->sc_ccc_mask) { 1959 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 1960 DEVNAME(sc)); 1961 is &= ~sc->sc_ccc_mask; 1962 is |= sc->sc_ccc_ports_cur; 1963 } 1964 #endif 1965 1966 /* Process interrupts for each port */ 1967 while (is) { 1968 port = ffs(is) - 1; 1969 if (sc->sc_ports[port]) 1970 ahci_port_intr(sc->sc_ports[port], 1971 AHCI_PREG_CI_ALL_SLOTS); 1972 is &= ~(1 << port); 1973 } 1974 1975 /* Finally, acknowledge global interrupt */ 1976 ahci_write(sc, AHCI_REG_IS, ack); 1977 1978 return (1); 1979 } 1980 1981 u_int32_t 1982 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 1983 { 1984 struct ahci_softc *sc = ap->ap_sc; 1985 u_int32_t is, ci_saved, ci_masked, processed = 0; 1986 int slot, need_restart = 0; 1987 int process_error = 0; 1988 struct ahci_ccb *ccb; 1989 volatile u_int32_t *active; 1990 #ifdef DIAGNOSTIC 1991 u_int32_t tmp; 1992 #endif 1993 1994 is = ahci_pread(ap, AHCI_PREG_IS); 1995 1996 /* Ack port interrupt only if checking all command slots. */ 1997 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 1998 ahci_pwrite(ap, AHCI_PREG_IS, is); 1999 2000 if (is) 2001 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 2002 is, AHCI_PFMT_IS); 2003 2004 if (ap->ap_sactive) { 2005 /* Active NCQ commands - use SActive instead of CI */ 2006 KASSERT(ap->ap_active == 0); 2007 KASSERT(ap->ap_active_cnt == 0); 2008 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2009 active = &ap->ap_sactive; 2010 } else { 2011 /* Save CI */ 2012 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2013 active = &ap->ap_active; 2014 } 2015 2016 if (is & AHCI_PREG_IS_TFES) { 2017 process_error = 1; 2018 } else if (is & AHCI_PREG_IS_DHRS) { 2019 u_int32_t tfd; 2020 u_int32_t cmd; 2021 u_int32_t serr; 2022 2023 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2024 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2025 serr = ahci_pread(ap, AHCI_PREG_SERR); 2026 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2027 (cmd & AHCI_PREG_CMD_CR) == 0) { 2028 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 2029 " %b, DIAG: %b\n", PORTNAME(ap), tfd, 2030 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2031 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2032 AHCI_PFMT_SERR_DIAG); 2033 process_error = 1; 2034 } else { 2035 /* rfis copy back is in the normal execution path */ 2036 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2037 } 2038 } 2039 2040 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 2041 if (process_error) { 2042 u_int32_t tfd, serr; 2043 int err_slot; 2044 2045 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2046 serr = ahci_pread(ap, AHCI_PREG_SERR); 2047 2048 if (ap->ap_sactive == 0) { 2049 /* Errored slot is easy to determine from CMD. */ 2050 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 2051 AHCI_PREG_CMD)); 2052 2053 if ((ci_saved & (1 << err_slot)) == 0) { 2054 /* 2055 * Hardware doesn't seem to report correct 2056 * slot number. If there's only one 2057 * outstanding command we can cope, 2058 * otherwise fail all active commands. 2059 */ 2060 if (ap->ap_active_cnt == 1) 2061 err_slot = ffs(ap->ap_active) - 1; 2062 else 2063 goto failall; 2064 } 2065 2066 ccb = &ap->ap_ccbs[err_slot]; 2067 2068 /* Preserve received taskfile data from the RFIS. */ 2069 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2070 sizeof(struct ata_fis_d2h)); 2071 } else 2072 err_slot = -1; /* Must extract error from log page */ 2073 2074 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 2075 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 2076 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2077 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2078 AHCI_PFMT_SERR_DIAG); 2079 2080 /* Turn off ST to clear CI and SACT. */ 2081 ahci_port_stop(ap, 0); 2082 need_restart = 1; 2083 2084 /* Clear SERR to enable capturing new errors. */ 2085 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2086 2087 /* Acknowledge the interrupts we can recover from. */ 2088 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2089 AHCI_PREG_IS_IFS); 2090 is = ahci_pread(ap, AHCI_PREG_IS); 2091 2092 /* If device hasn't cleared its busy status, try to idle it. */ 2093 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2094 2095 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 2096 (ap->ap_state == AP_S_ERROR_RECOVERY)) { 2097 /* can't reset the port here, just make sure 2098 * the operation fails and the port still works. 2099 */ 2100 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 2101 printf("%s: error on PMP port %d, idling " 2102 "device\n", PORTNAME(ap), 2103 ccb->ccb_xa.pmp_port); 2104 if (ahci_pmp_port_softreset(ap, 2105 ccb->ccb_xa.pmp_port) == 0) { 2106 printf("%s: unable to softreset port " 2107 "%d\n", PORTNAME(ap), 2108 ccb->ccb_xa.pmp_port); 2109 if (ahci_pmp_port_portreset(ap, 2110 ccb->ccb_xa.pmp_port)) { 2111 printf("%s: failed to port " 2112 " reset %d, giving up on " 2113 "it\n", PORTNAME(ap), 2114 ccb->ccb_xa.pmp_port); 2115 goto fatal; 2116 } 2117 } 2118 } else { 2119 printf("%s: attempting to idle device\n", 2120 PORTNAME(ap)); 2121 if (ahci_port_softreset(ap)) { 2122 printf("%s: failed to soft reset " 2123 "device\n", PORTNAME(ap)); 2124 if (ahci_port_portreset(ap, 0)) { 2125 printf("%s: failed to port " 2126 "reset device, give up on " 2127 "it\n", PORTNAME(ap)); 2128 goto fatal; 2129 } 2130 } 2131 } 2132 2133 /* Had to reset device, can't gather extended info. */ 2134 } else if (ap->ap_sactive) { 2135 /* Recover the NCQ error from log page 10h. 2136 * XXX FBS: need to do things to figure out where the 2137 * error came from. without FBS, we know the PMP port 2138 * responsible because we can only have queued commands 2139 * active for one port at a time. 2140 */ 2141 ahci_port_read_ncq_error(ap, &err_slot, 2142 ap->ap_pmp_ncq_port); 2143 if (err_slot < 0) 2144 goto failall; 2145 2146 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 2147 PORTNAME(ap), err_slot); 2148 2149 ccb = &ap->ap_ccbs[err_slot]; 2150 } else { 2151 /* Didn't reset, could gather extended info from log. */ 2152 } 2153 2154 /* 2155 * If we couldn't determine the errored slot, reset the port 2156 * and fail all the active slots. 2157 */ 2158 if (err_slot == -1) { 2159 if (ahci_port_softreset(ap) != 0 && 2160 ahci_port_portreset(ap, 0) != 0) { 2161 printf("%s: couldn't reset after NCQ error, " 2162 "disabling device.\n", PORTNAME(ap)); 2163 goto fatal; 2164 } 2165 printf("%s: couldn't recover NCQ error, failing " 2166 "all outstanding commands.\n", PORTNAME(ap)); 2167 goto failall; 2168 } 2169 2170 /* Clear the failed command in saved CI so completion runs. */ 2171 ci_saved &= ~(1 << err_slot); 2172 2173 /* Note the error in the ata_xfer. */ 2174 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2175 ccb->ccb_xa.state = ATA_S_ERROR; 2176 2177 #ifdef DIAGNOSTIC 2178 /* There may only be one outstanding standard command now. */ 2179 if (ap->ap_sactive == 0) { 2180 tmp = ci_saved; 2181 if (tmp) { 2182 slot = ffs(tmp) - 1; 2183 tmp &= ~(1 << slot); 2184 KASSERT(tmp == 0); 2185 } 2186 } 2187 #endif 2188 } 2189 2190 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 2191 * when there's a normal SATA device attached. 2192 */ 2193 if ((ap->ap_state == AP_S_PMP_PROBE) && 2194 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 2195 (is & AHCI_PREG_IS_IPMS)) { 2196 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2197 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 2198 PORTNAME(ap), slot); 2199 2200 ccb = &ap->ap_ccbs[slot]; 2201 ccb->ccb_xa.state = ATA_S_ERROR; 2202 2203 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 2204 is &= ~AHCI_PREG_IS_IPMS; 2205 } 2206 2207 /* ignore IFS errors while resetting a PMP port */ 2208 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 2209 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 2210 "port\n", PORTNAME(ap)); 2211 2212 need_restart = 1; 2213 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2214 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2215 is &= ~AHCI_PREG_IS_IFS; 2216 goto failall; 2217 } 2218 2219 /* Check for remaining errors - they are fatal. */ 2220 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2221 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2222 printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 2223 PORTNAME(ap), is, AHCI_PFMT_IS); 2224 2225 /* XXX try recovery first */ 2226 goto fatal; 2227 } 2228 2229 /* Fail all outstanding commands if we know the port won't recover. */ 2230 if (ap->ap_state == AP_S_FATAL_ERROR) { 2231 fatal: 2232 ap->ap_state = AP_S_FATAL_ERROR; 2233 failall: 2234 2235 /* Ensure port is shut down. */ 2236 ahci_port_stop(ap, 1); 2237 2238 /* Error all the active slots. */ 2239 ci_masked = ci_saved & *active; 2240 while (ci_masked) { 2241 slot = ffs(ci_masked) - 1; 2242 ccb = &ap->ap_ccbs[slot]; 2243 ci_masked &= ~(1 << slot); 2244 ccb->ccb_xa.state = ATA_S_ERROR; 2245 } 2246 2247 /* Run completion for all active slots. */ 2248 ci_saved &= ~*active; 2249 2250 /* Don't restart the port if our problems were deemed fatal. */ 2251 if (ap->ap_state == AP_S_FATAL_ERROR) 2252 need_restart = 0; 2253 } 2254 2255 /* 2256 * CCB completion is detected by noticing its slot's bit in CI has 2257 * changed to zero some time after we activated it. 2258 * If we are polling, we may only be interested in particular slot(s). 2259 */ 2260 ci_masked = ~ci_saved & *active & ci_mask; 2261 while (ci_masked) { 2262 slot = ffs(ci_masked) - 1; 2263 ccb = &ap->ap_ccbs[slot]; 2264 ci_masked &= ~(1 << slot); 2265 2266 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2267 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2268 " (error)" : ""); 2269 2270 bus_dmamap_sync(sc->sc_dmat, 2271 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2272 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 2273 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 2274 2275 bus_dmamap_sync(sc->sc_dmat, 2276 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2277 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 2278 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 2279 2280 bus_dmamap_sync(sc->sc_dmat, 2281 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 2282 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 2283 2284 *active &= ~(1 << ccb->ccb_slot); 2285 /* Copy the rfis into the ccb if we were asked for it */ 2286 if (ccb->ccb_xa.state == ATA_S_ONCHIP && 2287 ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 2288 memcpy(&ccb->ccb_xa.rfis, 2289 ap->ap_rfis->rfis, 2290 sizeof(struct ata_fis_d2h)); 2291 } 2292 2293 ccb->ccb_done(ccb); 2294 2295 processed |= 1 << ccb->ccb_slot; 2296 } 2297 2298 if (need_restart) { 2299 /* Restart command DMA on the port */ 2300 ahci_port_start(ap, 0); 2301 2302 /* Re-enable outstanding commands on port. */ 2303 if (ci_saved) { 2304 #ifdef DIAGNOSTIC 2305 tmp = ci_saved; 2306 while (tmp) { 2307 slot = ffs(tmp) - 1; 2308 tmp &= ~(1 << slot); 2309 ccb = &ap->ap_ccbs[slot]; 2310 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2311 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 2312 (!!ap->ap_sactive)); 2313 } 2314 #endif 2315 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 2316 "re-enabling%s slots %08x\n", PORTNAME(ap), 2317 ap->ap_sactive ? " NCQ" : "", ci_saved); 2318 2319 /* XXX FBS: 2320 * - need to set AHCI_PREG_FBS_DEV for each command 2321 * - can't do multiple commands with a single write to 2322 * AHCI_PREG_CI 2323 */ 2324 if (ap->ap_sactive) 2325 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 2326 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 2327 } 2328 } 2329 2330 return (processed); 2331 } 2332 2333 struct ahci_ccb * 2334 ahci_get_ccb(struct ahci_port *ap) 2335 { 2336 struct ahci_ccb *ccb; 2337 2338 mtx_enter(&ap->ap_ccb_mtx); 2339 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2340 if (ccb != NULL) { 2341 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2342 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2343 ccb->ccb_xa.state = ATA_S_SETUP; 2344 } 2345 mtx_leave(&ap->ap_ccb_mtx); 2346 2347 return (ccb); 2348 } 2349 2350 void 2351 ahci_put_ccb(struct ahci_ccb *ccb) 2352 { 2353 struct ahci_port *ap = ccb->ccb_port; 2354 2355 #ifdef DIAGNOSTIC 2356 if (ccb->ccb_xa.state != ATA_S_COMPLETE && 2357 ccb->ccb_xa.state != ATA_S_TIMEOUT && 2358 ccb->ccb_xa.state != ATA_S_ERROR) { 2359 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 2360 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 2361 ccb->ccb_slot); 2362 } 2363 #endif 2364 2365 ccb->ccb_xa.state = ATA_S_PUT; 2366 mtx_enter(&ap->ap_ccb_mtx); 2367 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2368 mtx_leave(&ap->ap_ccb_mtx); 2369 } 2370 2371 struct ahci_ccb * 2372 ahci_get_err_ccb(struct ahci_port *ap) 2373 { 2374 struct ahci_ccb *err_ccb; 2375 u_int32_t sact; 2376 2377 splassert(IPL_BIO); 2378 2379 /* No commands may be active on the chip. */ 2380 sact = ahci_pread(ap, AHCI_PREG_SACT); 2381 if (sact != 0) 2382 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 2383 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2384 2385 #ifdef DIAGNOSTIC 2386 KASSERT(ap->ap_err_busy == 0); 2387 ap->ap_err_busy = 1; 2388 #endif 2389 /* Save outstanding command state. */ 2390 ap->ap_err_saved_active = ap->ap_active; 2391 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2392 ap->ap_err_saved_sactive = ap->ap_sactive; 2393 2394 /* 2395 * Pretend we have no commands outstanding, so that completions won't 2396 * run prematurely. 2397 */ 2398 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2399 2400 /* 2401 * Grab a CCB to use for error recovery. This should never fail, as 2402 * we ask atascsi to reserve one for us at init time. 2403 */ 2404 err_ccb = ap->ap_ccb_err; 2405 err_ccb->ccb_xa.flags = 0; 2406 err_ccb->ccb_xa.state = ATA_S_SETUP; 2407 err_ccb->ccb_done = ahci_empty_done; 2408 2409 return (err_ccb); 2410 } 2411 2412 void 2413 ahci_put_err_ccb(struct ahci_ccb *ccb) 2414 { 2415 struct ahci_port *ap = ccb->ccb_port; 2416 u_int32_t sact; 2417 2418 splassert(IPL_BIO); 2419 2420 #ifdef DIAGNOSTIC 2421 KASSERT(ap->ap_err_busy); 2422 #endif 2423 /* No commands may be active on the chip */ 2424 sact = ahci_pread(ap, AHCI_PREG_SACT); 2425 if (sact != 0) 2426 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2427 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2428 2429 #ifdef DIAGNOSTIC 2430 /* Done with the CCB */ 2431 KASSERT(ccb == ap->ap_ccb_err); 2432 #endif 2433 2434 /* Restore outstanding command state */ 2435 ap->ap_sactive = ap->ap_err_saved_sactive; 2436 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2437 ap->ap_active = ap->ap_err_saved_active; 2438 2439 #ifdef DIAGNOSTIC 2440 ap->ap_err_busy = 0; 2441 #endif 2442 } 2443 2444 struct ahci_ccb * 2445 ahci_get_pmp_ccb(struct ahci_port *ap) 2446 { 2447 struct ahci_ccb *ccb; 2448 u_int32_t sact; 2449 2450 /* some PMP commands need to be issued on slot 1, 2451 * particularly the command that clears SRST and 2452 * fetches the device signature. 2453 * 2454 * ensure the chip is idle and ccb 1 is available. 2455 */ 2456 splassert(IPL_BIO); 2457 2458 sact = ahci_pread(ap, AHCI_PREG_SACT); 2459 if (sact != 0) 2460 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 2461 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2462 2463 ccb = &ap->ap_ccbs[1]; 2464 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2465 ccb->ccb_xa.flags = 0; 2466 ccb->ccb_done = ahci_pmp_cmd_done; 2467 2468 mtx_enter(&ap->ap_ccb_mtx); 2469 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2470 mtx_leave(&ap->ap_ccb_mtx); 2471 2472 return ccb; 2473 } 2474 2475 void 2476 ahci_put_pmp_ccb(struct ahci_ccb *ccb) 2477 { 2478 struct ahci_port *ap = ccb->ccb_port; 2479 u_int32_t sact; 2480 2481 /* make sure this is the right ccb */ 2482 KASSERT(ccb == &ap->ap_ccbs[1]); 2483 2484 /* No commands may be active on the chip */ 2485 sact = ahci_pread(ap, AHCI_PREG_SACT); 2486 if (sact != 0) 2487 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2488 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2489 2490 ccb->ccb_xa.state = ATA_S_PUT; 2491 mtx_enter(&ap->ap_ccb_mtx); 2492 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2493 mtx_leave(&ap->ap_ccb_mtx); 2494 } 2495 2496 int 2497 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 2498 { 2499 struct ahci_ccb *ccb; 2500 struct ahci_cmd_hdr *cmd_slot; 2501 u_int32_t cmd; 2502 struct ata_fis_h2d *fis; 2503 int rc = EIO, oldstate; 2504 2505 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 2506 oldstate = ap->ap_state; 2507 ap->ap_state = AP_S_ERROR_RECOVERY; 2508 2509 /* Save command register state. */ 2510 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 2511 2512 /* Port should have been idled already. Start it. */ 2513 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 2514 ahci_port_start(ap, 0); 2515 2516 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 2517 ccb = ahci_get_err_ccb(ap); 2518 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 2519 ccb->ccb_xa.data = ap->ap_err_scratch; 2520 ccb->ccb_xa.datalen = 512; 2521 cmd_slot = ccb->ccb_cmd_hdr; 2522 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2523 2524 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2525 fis->type = ATA_FIS_TYPE_H2D; 2526 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 2527 fis->command = ATA_C_READ_LOG_EXT; 2528 fis->lba_low = 0x10; /* queued error log page (10h) */ 2529 fis->sector_count = 1; /* number of sectors (1) */ 2530 fis->sector_count_exp = 0; 2531 fis->lba_mid = 0; /* starting offset */ 2532 fis->lba_mid_exp = 0; 2533 fis->device = 0; 2534 2535 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 2536 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 2537 2538 if (ahci_load_prdt(ccb) != 0) { 2539 rc = ENOMEM; /* XXX caller must abort all commands */ 2540 goto err; 2541 } 2542 2543 ccb->ccb_xa.state = ATA_S_PENDING; 2544 if (ahci_poll(ccb, 1000, NULL) != 0 || 2545 ccb->ccb_xa.state == ATA_S_ERROR) 2546 goto err; 2547 2548 rc = 0; 2549 err: 2550 /* Abort our command, if it failed, by stopping command DMA. */ 2551 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 2552 printf("%s: log page read failed, slot %d was still active.\n", 2553 PORTNAME(ap), ccb->ccb_slot); 2554 ahci_port_stop(ap, 0); 2555 } 2556 2557 /* Done with the error CCB now. */ 2558 ahci_unload_prdt(ccb); 2559 ahci_put_err_ccb(ccb); 2560 2561 /* Extract failed register set and tags from the scratch space. */ 2562 if (rc == 0) { 2563 struct ata_log_page_10h *log; 2564 int err_slot; 2565 2566 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2567 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 2568 /* Not queued bit was set - wasn't an NCQ error? */ 2569 printf("%s: read NCQ error page, but not an NCQ " 2570 "error?\n", PORTNAME(ap)); 2571 rc = ESRCH; 2572 } else { 2573 /* Copy back the log record as a D2H register FIS. */ 2574 *err_slotp = err_slot = log->err_regs.type & 2575 ATA_LOG_10H_TYPE_TAG_MASK; 2576 2577 ccb = &ap->ap_ccbs[err_slot]; 2578 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2579 sizeof(struct ata_fis_d2h)); 2580 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2581 ccb->ccb_xa.rfis.flags = 0; 2582 } 2583 } 2584 2585 /* Restore saved CMD register state */ 2586 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 2587 ap->ap_state = oldstate; 2588 2589 return (rc); 2590 } 2591 2592 struct ahci_dmamem * 2593 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 2594 { 2595 struct ahci_dmamem *adm; 2596 int nsegs; 2597 2598 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 2599 if (adm == NULL) 2600 return (NULL); 2601 2602 adm->adm_size = size; 2603 2604 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2605 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2606 goto admfree; 2607 2608 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 2609 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 2610 goto destroy; 2611 2612 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2613 &adm->adm_kva, BUS_DMA_NOWAIT) != 0) 2614 goto free; 2615 2616 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 2617 NULL, BUS_DMA_NOWAIT) != 0) 2618 goto unmap; 2619 2620 return (adm); 2621 2622 unmap: 2623 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 2624 free: 2625 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2626 destroy: 2627 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2628 admfree: 2629 free(adm, M_DEVBUF, sizeof(*adm)); 2630 2631 return (NULL); 2632 } 2633 2634 void 2635 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 2636 { 2637 bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 2638 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 2639 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2640 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2641 free(adm, M_DEVBUF, sizeof(*adm)); 2642 } 2643 2644 u_int32_t 2645 ahci_read(struct ahci_softc *sc, bus_size_t r) 2646 { 2647 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2648 BUS_SPACE_BARRIER_READ); 2649 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 2650 } 2651 2652 void 2653 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 2654 { 2655 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 2656 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2657 BUS_SPACE_BARRIER_WRITE); 2658 } 2659 2660 int 2661 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 2662 u_int32_t target) 2663 { 2664 int i; 2665 2666 for (i = 0; i < 1000; i++) { 2667 if ((ahci_read(sc, r) & mask) != target) 2668 return (0); 2669 delay(1000); 2670 } 2671 2672 return (1); 2673 } 2674 2675 u_int32_t 2676 ahci_pread(struct ahci_port *ap, bus_size_t r) 2677 { 2678 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2679 BUS_SPACE_BARRIER_READ); 2680 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 2681 } 2682 2683 void 2684 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 2685 { 2686 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 2687 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2688 BUS_SPACE_BARRIER_WRITE); 2689 } 2690 2691 int 2692 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 2693 u_int32_t target, int n) 2694 { 2695 int i; 2696 2697 for (i = 0; i < n * 1000; i++) { 2698 if ((ahci_pread(ap, r) & mask) == target) 2699 return (0); 2700 delay(1000); 2701 } 2702 2703 return (1); 2704 } 2705 2706 int 2707 ahci_ata_probe(void *xsc, int port, int lun) 2708 { 2709 struct ahci_softc *sc = xsc; 2710 struct ahci_port *ap = sc->sc_ports[port]; 2711 2712 if (ap == NULL) 2713 return (ATA_PORT_T_NONE); 2714 2715 if (lun != 0) { 2716 int pmp_port = lun - 1; 2717 if (pmp_port >= ap->ap_pmp_ports) { 2718 return (ATA_PORT_T_NONE); 2719 } 2720 return (ahci_pmp_port_probe(ap, pmp_port)); 2721 } else { 2722 return (ahci_port_signature(ap)); 2723 } 2724 } 2725 2726 void 2727 ahci_ata_free(void *xsc, int port, int lun) 2728 { 2729 2730 } 2731 2732 struct ata_xfer * 2733 ahci_ata_get_xfer(void *aaa_cookie, int port) 2734 { 2735 struct ahci_softc *sc = aaa_cookie; 2736 struct ahci_port *ap = sc->sc_ports[port]; 2737 struct ahci_ccb *ccb; 2738 2739 ccb = ahci_get_ccb(ap); 2740 if (ccb == NULL) { 2741 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 2742 PORTNAME(ap)); 2743 return (NULL); 2744 } 2745 2746 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 2747 PORTNAME(ap), ccb->ccb_slot); 2748 2749 return ((struct ata_xfer *)ccb); 2750 } 2751 2752 void 2753 ahci_ata_put_xfer(struct ata_xfer *xa) 2754 { 2755 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2756 2757 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 2758 2759 ahci_put_ccb(ccb); 2760 } 2761 2762 void 2763 ahci_ata_cmd(struct ata_xfer *xa) 2764 { 2765 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2766 struct ahci_cmd_hdr *cmd_slot; 2767 int s; 2768 u_int16_t flags; 2769 2770 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 2771 goto failcmd; 2772 2773 ccb->ccb_done = ahci_ata_cmd_done; 2774 2775 cmd_slot = ccb->ccb_cmd_hdr; 2776 flags = 5 /* FIS length (in DWORDs) */; 2777 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 2778 2779 if (xa->flags & ATA_F_WRITE) 2780 flags |= AHCI_CMD_LIST_FLAG_W; 2781 2782 if (xa->flags & ATA_F_PACKET) 2783 flags |= AHCI_CMD_LIST_FLAG_A; 2784 2785 htolem16(&cmd_slot->flags, flags); 2786 2787 if (ahci_load_prdt(ccb) != 0) 2788 goto failcmd; 2789 2790 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 2791 2792 xa->state = ATA_S_PENDING; 2793 2794 if (xa->flags & ATA_F_POLL) 2795 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 2796 else { 2797 s = splbio(); 2798 timeout_add_msec(&xa->stimeout, xa->timeout); 2799 ahci_start(ccb); 2800 splx(s); 2801 } 2802 2803 return; 2804 2805 failcmd: 2806 s = splbio(); 2807 xa->state = ATA_S_ERROR; 2808 ata_complete(xa); 2809 splx(s); 2810 } 2811 2812 void 2813 ahci_pmp_cmd_done(struct ahci_ccb *ccb) 2814 { 2815 struct ata_xfer *xa = &ccb->ccb_xa; 2816 2817 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2818 ahci_issue_pending_commands(ccb->ccb_port, 2819 xa->flags & ATA_F_NCQ); 2820 2821 xa->state = ATA_S_COMPLETE; 2822 } 2823 2824 2825 void 2826 ahci_ata_cmd_done(struct ahci_ccb *ccb) 2827 { 2828 struct ata_xfer *xa = &ccb->ccb_xa; 2829 2830 timeout_del(&xa->stimeout); 2831 2832 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2833 ahci_issue_pending_commands(ccb->ccb_port, 2834 xa->flags & ATA_F_NCQ); 2835 2836 ahci_unload_prdt(ccb); 2837 2838 if (xa->state == ATA_S_ONCHIP) 2839 xa->state = ATA_S_COMPLETE; 2840 #ifdef DIAGNOSTIC 2841 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 2842 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 2843 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 2844 ccb->ccb_slot); 2845 #endif 2846 if (xa->state != ATA_S_TIMEOUT) 2847 ata_complete(xa); 2848 } 2849 2850 void 2851 ahci_ata_cmd_timeout(void *arg) 2852 { 2853 struct ahci_ccb *ccb = arg; 2854 struct ata_xfer *xa = &ccb->ccb_xa; 2855 struct ahci_port *ap = ccb->ccb_port; 2856 int s, ccb_was_started, ncq_cmd; 2857 volatile u_int32_t *active; 2858 2859 s = splbio(); 2860 2861 ncq_cmd = (xa->flags & ATA_F_NCQ); 2862 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 2863 2864 if (ccb->ccb_xa.state == ATA_S_PENDING) { 2865 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 2866 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 2867 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2868 ccb_was_started = 0; 2869 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 2870 1 << ccb->ccb_slot)) { 2871 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 2872 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 2873 goto ret; 2874 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2875 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 2876 "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 2877 ISSET(*active, 1 << ccb->ccb_slot) ? 2878 " but slot is still active?" : "."); 2879 goto ret; 2880 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 2881 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 2882 1 << ccb->ccb_slot)) { 2883 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 2884 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 2885 ccb->ccb_slot); 2886 *active &= ~(1 << ccb->ccb_slot); 2887 ccb->ccb_done(ccb); 2888 goto ret; 2889 } else { 2890 ccb_was_started = 1; 2891 } 2892 2893 /* Complete the slot with a timeout error. */ 2894 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2895 *active &= ~(1 << ccb->ccb_slot); 2896 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 2897 ccb->ccb_done(ccb); /* This won't issue pending commands or run the 2898 atascsi completion. */ 2899 2900 /* Reset port to abort running command. */ 2901 if (ccb_was_started) { 2902 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 2903 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 2904 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 2905 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 2906 != 0) { 2907 printf("%s: failed to reset port during timeout " 2908 "handling, disabling it\n", PORTNAME(ap)); 2909 ap->ap_state = AP_S_FATAL_ERROR; 2910 } 2911 2912 /* Restart any other commands that were aborted by the reset. */ 2913 if (*active) { 2914 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 2915 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 2916 *active); 2917 if (ncq_cmd) 2918 ahci_pwrite(ap, AHCI_PREG_SACT, *active); 2919 ahci_pwrite(ap, AHCI_PREG_CI, *active); 2920 } 2921 } 2922 2923 /* Issue any pending commands now. */ 2924 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 2925 if (ccb_was_started) 2926 ahci_issue_pending_commands(ap, ncq_cmd); 2927 else if (ap->ap_active == 0) 2928 ahci_issue_pending_ncq_commands(ap); 2929 2930 /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 2931 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 2932 ata_complete(xa); 2933 2934 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 2935 ret: 2936 splx(s); 2937 } 2938 2939 void 2940 ahci_empty_done(struct ahci_ccb *ccb) 2941 { 2942 if (ccb->ccb_xa.state != ATA_S_ERROR) 2943 ccb->ccb_xa.state = ATA_S_COMPLETE; 2944 } 2945 2946 int 2947 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 2948 { 2949 struct ahci_ccb *ccb; 2950 struct ata_fis_h2d *fis; 2951 int error; 2952 2953 ccb = ahci_get_pmp_ccb(ap); 2954 if (ccb == NULL) { 2955 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2956 return (1); 2957 } 2958 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 2959 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2960 ccb->ccb_xa.state = ATA_S_PENDING; 2961 2962 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2963 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2964 fis->type = ATA_FIS_TYPE_H2D; 2965 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 2966 fis->command = ATA_C_READ_PM; 2967 fis->features = which; 2968 fis->device = target | ATA_H2D_DEVICE_LBA; 2969 fis->control = ATA_FIS_CONTROL_4BIT; 2970 2971 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 2972 error = 1; 2973 } else { 2974 *datap = ccb->ccb_xa.rfis.sector_count | 2975 (ccb->ccb_xa.rfis.lba_low << 8) | 2976 (ccb->ccb_xa.rfis.lba_mid << 16) | 2977 (ccb->ccb_xa.rfis.lba_high << 24); 2978 error = 0; 2979 } 2980 ahci_put_pmp_ccb(ccb); 2981 return (error); 2982 } 2983 2984 int 2985 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 2986 { 2987 struct ahci_ccb *ccb; 2988 struct ata_fis_h2d *fis; 2989 int error; 2990 2991 ccb = ahci_get_pmp_ccb(ap); 2992 if (ccb == NULL) { 2993 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2994 return (1); 2995 } 2996 ccb->ccb_xa.flags = ATA_F_POLL; 2997 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2998 ccb->ccb_xa.state = ATA_S_PENDING; 2999 3000 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3001 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3002 fis->type = ATA_FIS_TYPE_H2D; 3003 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3004 fis->command = ATA_C_WRITE_PM; 3005 fis->features = which; 3006 fis->device = target | ATA_H2D_DEVICE_LBA; 3007 fis->sector_count = (u_int8_t)data; 3008 fis->lba_low = (u_int8_t)(data >> 8); 3009 fis->lba_mid = (u_int8_t)(data >> 16); 3010 fis->lba_high = (u_int8_t)(data >> 24); 3011 fis->control = ATA_FIS_CONTROL_4BIT; 3012 3013 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 3014 ahci_put_pmp_ccb(ccb); 3015 return (error); 3016 } 3017 3018 int 3019 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 3020 { 3021 int error; 3022 3023 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 3024 if (error == 0) 3025 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 3026 if (error) 3027 *datap = 0; 3028 3029 return (error); 3030 } 3031 3032 int 3033 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 3034 { 3035 u_int32_t chipid; 3036 u_int32_t rev; 3037 u_int32_t nports; 3038 u_int32_t features; 3039 u_int32_t enabled; 3040 int s; 3041 3042 s = splbio(); 3043 3044 if (ahci_pmp_read(ap, 15, 0, &chipid) || 3045 ahci_pmp_read(ap, 15, 1, &rev) || 3046 ahci_pmp_read(ap, 15, 2, &nports) || 3047 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 3048 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 3049 printf("%s: port multiplier identification failed\n", 3050 PORTNAME(ap)); 3051 splx(s); 3052 return (1); 3053 } 3054 splx(s); 3055 3056 nports &= 0x0F; 3057 3058 /* ignore SEMB port on SiI3726 port multiplier chips */ 3059 if (chipid == 0x37261095) { 3060 nports--; 3061 } 3062 3063 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 3064 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 3065 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 3066 SATA_PFMT_PM_FEA); 3067 3068 *ret_nports = nports; 3069 return (0); 3070 } 3071 3072 3073 #ifdef HIBERNATE 3074 void 3075 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 3076 { 3077 ccb->ccb_cmd_hdr->prdbc = 0; 3078 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 3079 } 3080 3081 int 3082 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 3083 { 3084 u_int32_t is, ci_saved; 3085 int process_error = 0; 3086 3087 is = ahci_pread(ap, AHCI_PREG_IS); 3088 3089 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 3090 3091 if (is & AHCI_PREG_IS_DHRS) { 3092 u_int32_t tfd; 3093 u_int32_t cmd; 3094 u_int32_t serr; 3095 3096 tfd = ahci_pread(ap, AHCI_PREG_TFD); 3097 cmd = ahci_pread(ap, AHCI_PREG_CMD); 3098 serr = ahci_pread(ap, AHCI_PREG_SERR); 3099 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 3100 (cmd & AHCI_PREG_CMD_CR) == 0) { 3101 process_error = 1; 3102 } else { 3103 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 3104 } 3105 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 3106 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 3107 process_error = 1; 3108 } 3109 3110 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 3111 if (process_error) { 3112 3113 /* Turn off ST to clear CI and SACT. */ 3114 ahci_port_stop(ap, 0); 3115 3116 /* just return an error indicator? we can't meaningfully 3117 * recover, and on the way back out we'll DVACT_RESUME which 3118 * resets and reinits the port. 3119 */ 3120 return (EIO); 3121 } 3122 3123 /* command is finished when the bit in CI for the slot goes to 0 */ 3124 if (ci_saved & (1 << ccb->ccb_slot)) { 3125 return (EAGAIN); 3126 } 3127 3128 return (0); 3129 } 3130 3131 void 3132 ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 3133 { 3134 struct ata_xfer *xa = &ccb->ccb_xa; 3135 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd; 3136 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 3137 int i; 3138 paddr_t data_phys; 3139 u_int64_t data_bus_phys; 3140 vaddr_t data_addr; 3141 size_t seglen; 3142 size_t buflen; 3143 3144 if (xa->datalen == 0) { 3145 ccb->ccb_cmd_hdr->prdtl = 0; 3146 return; 3147 } 3148 3149 /* derived from i386/amd64 _bus_dma_load_buffer; 3150 * for amd64 the buffer will always be dma safe. 3151 */ 3152 3153 buflen = xa->datalen; 3154 data_addr = (vaddr_t)xa->data; 3155 for (i = 0; buflen > 0; i++) { 3156 prd = &prdt[i]; 3157 3158 pmap_extract(pmap_kernel(), data_addr, &data_phys); 3159 data_bus_phys = data_phys; 3160 3161 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 3162 if (buflen < seglen) 3163 seglen = buflen; 3164 3165 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3166 3167 data_addr += seglen; 3168 buflen -= seglen; 3169 } 3170 3171 htolem16(&cmd_slot->prdtl, i); 3172 } 3173 3174 int 3175 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 3176 int op, void *page) 3177 { 3178 /* we use the 'real' ahci_port and ahci_softc here, but 3179 * never write to them 3180 */ 3181 struct { 3182 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 3183 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 3184 /* cmd table isn't actually used because of mysteries */ 3185 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 3186 struct ahci_port *ap; 3187 struct ahci_ccb ccb_buf; 3188 struct ahci_ccb *ccb; 3189 struct ahci_cmd_hdr *hdr_buf; 3190 int pmp_port; 3191 daddr_t poffset; 3192 size_t psize; 3193 } *my = page; 3194 struct ata_fis_h2d *fis; 3195 u_int32_t sector_count; 3196 struct ahci_cmd_hdr *cmd_slot; 3197 int rc; 3198 int timeout; 3199 u_int16_t flags; 3200 3201 if (op == HIB_INIT) { 3202 struct device *disk; 3203 struct device *scsibus; 3204 struct ahci_softc *sc; 3205 extern struct cfdriver sd_cd; 3206 struct scsi_link *link; 3207 struct scsibus_softc *bus_sc; 3208 int port; 3209 paddr_t page_phys; 3210 u_int64_t item_phys; 3211 u_int32_t cmd; 3212 3213 my->poffset = blkno; 3214 my->psize = size; 3215 3216 /* map dev to an ahci port */ 3217 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 3218 scsibus = disk->dv_parent; 3219 sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 3220 3221 /* find the scsi_link for the device, which has the port */ 3222 port = -1; 3223 bus_sc = (struct scsibus_softc *)scsibus; 3224 SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) { 3225 if (link->device_softc == disk) { 3226 /* link->adapter_softc == sc->sc_atascsi */ 3227 port = link->target; 3228 if (link->lun > 0) 3229 my->pmp_port = link->lun - 1; 3230 else 3231 my->pmp_port = 0; 3232 3233 break; 3234 } 3235 } 3236 if (port == -1) { 3237 /* don't know where the disk is */ 3238 return (EIO); 3239 } 3240 3241 my->ap = sc->sc_ports[port]; 3242 3243 /* we're going to use the first command slot, 3244 * so ensure it's not already in use 3245 */ 3246 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 3247 /* this shouldn't happen, we should be idle */ 3248 return (EIO); 3249 } 3250 3251 /* stop the port so we can relocate to the hibernate page */ 3252 if (ahci_port_stop(my->ap, 1)) { 3253 return (EIO); 3254 } 3255 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 3256 3257 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 3258 3259 /* Setup RFIS base address */ 3260 item_phys = page_phys + ((void *)&my->rfis - page); 3261 ahci_pwrite(my->ap, AHCI_PREG_FBU, 3262 (u_int32_t)(item_phys >> 32)); 3263 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 3264 3265 /* Enable FIS reception and activate port. */ 3266 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3267 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 3268 AHCI_PREG_CMD_SUD; 3269 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 3270 AHCI_PREG_CMD_ICC_ACTIVE); 3271 3272 /* Check whether port activated. */ 3273 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3274 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 3275 return (EIO); 3276 } 3277 3278 /* Set up the single CCB */ 3279 my->ccb = &my->ccb_buf; 3280 my->ccb->ccb_slot = 0; 3281 my->ccb->ccb_port = my->ap; 3282 3283 /* Setup command list base address */ 3284 item_phys = page_phys + ((void *)&my->cmd_hdr - page); 3285 ahci_pwrite(my->ap, AHCI_PREG_CLBU, 3286 (u_int32_t)(item_phys >> 32)); 3287 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 3288 3289 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 3290 3291 /* use existing cmd table - don't know why moving to a new one fails */ 3292 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 3293 pmap_extract(pmap_kernel(), 3294 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 3295 &page_phys); 3296 item_phys = page_phys; 3297 #if 0 3298 /* use cmd table in hibernate page (doesn't work) */ 3299 my->ccb->ccb_cmd_table = &my->cmd_table; 3300 item_phys = page_phys + ((void *)&my->cmd_table - page); 3301 #endif 3302 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 3303 3304 my->ccb->ccb_xa.fis = 3305 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 3306 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 3307 my->ccb->ccb_xa.tag = 0; 3308 3309 /* Wait for ICC change to complete */ 3310 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 3311 3312 if (ahci_port_start(my->ap, 0)) { 3313 return (EIO); 3314 } 3315 3316 /* Flush interrupts for port */ 3317 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 3318 AHCI_PREG_IS)); 3319 ahci_write(sc, AHCI_REG_IS, 1 << port); 3320 3321 ahci_enable_interrupts(my->ap); 3322 return (0); 3323 } else if (op == HIB_DONE) { 3324 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 3325 return (0); 3326 } 3327 3328 if (blkno > my->psize) 3329 return (E2BIG); 3330 blkno += my->poffset; 3331 3332 /* build fis */ 3333 sector_count = size / 512; /* dlg promises this is okay */ 3334 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 3335 fis = my->ccb->ccb_xa.fis; 3336 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 3337 fis->lba_low = blkno & 0xff; 3338 fis->lba_mid = (blkno >> 8) & 0xff; 3339 fis->lba_high = (blkno >> 16) & 0xff; 3340 3341 if (sector_count > 0x100 || blkno > 0xfffffff) { 3342 /* Use LBA48 */ 3343 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 3344 ATA_C_READDMA_EXT; 3345 fis->device = ATA_H2D_DEVICE_LBA; 3346 fis->lba_low_exp = (blkno >> 24) & 0xff; 3347 fis->lba_mid_exp = (blkno >> 32) & 0xff; 3348 fis->lba_high_exp = (blkno >> 40) & 0xff; 3349 fis->sector_count = sector_count & 0xff; 3350 fis->sector_count_exp = (sector_count >> 8) & 0xff; 3351 } else { 3352 /* Use LBA */ 3353 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 3354 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 3355 fis->sector_count = sector_count & 0xff; 3356 } 3357 3358 my->ccb->ccb_xa.data = (void *)addr; 3359 my->ccb->ccb_xa.datalen = size; 3360 my->ccb->ccb_xa.pmp_port = my->pmp_port; 3361 my->ccb->ccb_xa.flags |= ATA_F_POLL; 3362 3363 cmd_slot = my->ccb->ccb_cmd_hdr; 3364 flags = 5; /* FIS length (in DWORDs) */ 3365 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 3366 3367 if (op == HIB_W) 3368 flags |= AHCI_CMD_LIST_FLAG_W; 3369 3370 htolem16(&cmd_slot->flags, flags); 3371 3372 ahci_hibernate_load_prdt(my->ccb); 3373 3374 ahci_hibernate_io_start(my->ap, my->ccb); 3375 timeout = 1000; 3376 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 3377 delay(1000); 3378 timeout--; 3379 if (timeout == 0) { 3380 return (EIO); 3381 } 3382 } 3383 3384 return (0); 3385 } 3386 3387 #endif 3388