1 /* 2 * (MPSAFE) 3 * 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 * 18 * 19 * Copyright (c) 2009 The DragonFly Project. All rights reserved. 20 * 21 * This code is derived from software contributed to The DragonFly Project 22 * by Matthew Dillon <dillon@backplane.com> 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 28 * 1. Redistributions of source code must retain the above copyright 29 * notice, this list of conditions and the following disclaimer. 30 * 2. Redistributions in binary form must reproduce the above copyright 31 * notice, this list of conditions and the following disclaimer in 32 * the documentation and/or other materials provided with the 33 * distribution. 34 * 3. Neither the name of The DragonFly Project nor the names of its 35 * contributors may be used to endorse or promote products derived 36 * from this software without specific, prior written permission. 37 * 38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 39 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 40 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 41 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 42 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 43 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 44 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 45 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 46 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 47 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 48 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 49 * SUCH DAMAGE. 50 * 51 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $ 52 */ 53 54 #include "ahci.h" 55 56 void ahci_port_interrupt_enable(struct ahci_port *ap); 57 58 int ahci_load_prdt(struct ahci_ccb *); 59 void ahci_unload_prdt(struct ahci_ccb *); 60 static void ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, 61 int nsegs, int error); 62 void ahci_start(struct ahci_ccb *); 63 int ahci_port_softreset(struct ahci_port *ap); 64 int ahci_port_hardreset(struct ahci_port *ap, int hard); 65 void ahci_port_hardstop(struct ahci_port *ap); 66 67 static void ahci_ata_cmd_timeout_unserialized(void *); 68 void ahci_check_active_timeouts(struct ahci_port *ap); 69 70 void ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at); 71 void ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at); 72 void ahci_issue_pending_commands(struct ahci_port *ap, struct ahci_ccb *ccb); 73 void ahci_issue_saved_commands(struct ahci_port *ap, u_int32_t mask); 74 75 int ahci_port_read_ncq_error(struct ahci_port *, int); 76 77 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, bus_dma_tag_t tag); 78 void ahci_dmamem_free(struct ahci_softc *, struct ahci_dmamem *); 79 static void ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error); 80 81 static void ahci_dummy_done(struct ata_xfer *xa); 82 static void ahci_empty_done(struct ahci_ccb *ccb); 83 static void ahci_ata_cmd_done(struct ahci_ccb *ccb); 84 static u_int32_t ahci_pactive(struct ahci_port *ap); 85 86 /* 87 * Initialize the global AHCI hardware. This code does not set up any of 88 * its ports. 89 */ 90 int 91 ahci_init(struct ahci_softc *sc) 92 { 93 u_int32_t cap, pi, pleft; 94 int i; 95 struct ahci_port *ap; 96 97 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", 98 ahci_read(sc, AHCI_REG_GHC), AHCI_FMT_GHC); 99 100 /* 101 * save BIOS initialised parameters, enable staggered spin up 102 */ 103 cap = ahci_read(sc, AHCI_REG_CAP); 104 cap &= AHCI_REG_CAP_SMPS; 105 cap |= AHCI_REG_CAP_SSS; 106 pi = ahci_read(sc, AHCI_REG_PI); 107 108 /* 109 * Unconditionally reset the controller, do not conditionalize on 110 * trying to figure it if it was previously active or not. 111 * 112 * NOTE: On AE before HR. The AHCI-1.1 spec has a note in section 113 * 5.2.2.1 regarding this. HR should be set to 1 only after 114 * AE is set to 1. The reset sequence will clear HR when 115 * it completes, and will also clear AE if SAM is 0. AE must 116 * then be set again. When SAM is 1 the AE bit typically reads 117 * as 1 (and is read-only). 118 * 119 * NOTE: Avoid PCI[e] transaction burst by issuing dummy reads, 120 * otherwise the writes will only be separated by a few 121 * nanoseconds. 122 * 123 * NOTE BRICKS (1) 124 * 125 * If you have a port multiplier and it does not have a device 126 * in target 0, and it probes normally, but a later operation 127 * mis-probes a target behind that PM, it is possible for the 128 * port to brick such that only (a) a power cycle of the host 129 * or (b) placing a device in target 0 will fix the problem. 130 * Power cycling the PM has no effect (it works fine on another 131 * host port). This issue is unrelated to CLO. 132 */ 133 /* 134 * Wait for any prior reset sequence to complete 135 */ 136 if (ahci_wait_ne(sc, AHCI_REG_GHC, 137 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) { 138 device_printf(sc->sc_dev, "Controller is stuck in reset\n"); 139 return (1); 140 } 141 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 142 ahci_os_sleep(500); 143 ahci_read(sc, AHCI_REG_GHC); /* flush */ 144 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_HR); 145 ahci_os_sleep(500); 146 ahci_read(sc, AHCI_REG_GHC); /* flush */ 147 if (ahci_wait_ne(sc, AHCI_REG_GHC, 148 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) { 149 device_printf(sc->sc_dev, "unable to reset controller\n"); 150 return (1); 151 } 152 if (ahci_read(sc, AHCI_REG_GHC) & AHCI_REG_GHC_AE) { 153 device_printf(sc->sc_dev, "AE did not auto-clear!\n"); 154 ahci_write(sc, AHCI_REG_GHC, 0); 155 ahci_os_sleep(500); 156 } 157 158 /* 159 * Enable ahci (global interrupts disabled) 160 * 161 * Restore saved parameters. Avoid pci transaction burst write 162 * by issuing dummy reads. 163 */ 164 ahci_os_sleep(500); 165 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 166 ahci_os_sleep(500); 167 168 ahci_read(sc, AHCI_REG_GHC); /* flush */ 169 ahci_write(sc, AHCI_REG_CAP, cap); 170 ahci_write(sc, AHCI_REG_PI, pi); 171 ahci_read(sc, AHCI_REG_GHC); /* flush */ 172 173 /* 174 * Intel hocus pocus in case the BIOS has not set the chip up 175 * properly for AHCI operation. 176 */ 177 if (pci_get_vendor(sc->sc_dev) == PCI_VENDOR_INTEL) { 178 if ((pci_read_config(sc->sc_dev, 0x92, 2) & 0x0F) != 0x0F) 179 device_printf(sc->sc_dev, "Intel hocus pocus\n"); 180 pci_write_config(sc->sc_dev, 0x92, 181 pci_read_config(sc->sc_dev, 0x92, 2) | 0x0F, 2); 182 } 183 184 /* 185 * This is a hack that currently does not appear to have 186 * a significant effect, but I noticed the port registers 187 * do not appear to be completely cleared after the host 188 * controller is reset. 189 * 190 * Use a temporary ap structure so we can call ahci_pwrite(). 191 * 192 * We must be sure to stop the port 193 */ 194 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO); 195 ap->ap_sc = sc; 196 pleft = pi; 197 for (i = 0; i < AHCI_MAX_PORTS; ++i) { 198 if (pleft == 0) 199 break; 200 if ((pi & (1 << i)) == 0) 201 continue; 202 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 203 AHCI_PORT_REGION(i), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 204 device_printf(sc->sc_dev, "can't map port\n"); 205 return (1); 206 } 207 /* 208 * NOTE! Setting AHCI_PREG_SCTL_DET_DISABLE on AHCI1.0 or 209 * AHCI1.1 can brick the chipset. Not only brick it, 210 * but also crash the PC. The bit seems unreliable 211 * on AHCI1.2 as well. 212 */ 213 ahci_port_stop(ap, 1); 214 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 215 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 216 ahci_pwrite(ap, AHCI_PREG_IE, 0); 217 ahci_write(ap->ap_sc, AHCI_REG_IS, 1 << i); 218 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 219 ahci_pwrite(ap, AHCI_PREG_IS, -1); 220 sc->sc_portmask |= (1 << i); 221 pleft &= ~(1 << i); 222 } 223 sc->sc_numports = i; 224 kfree(ap, M_DEVBUF); 225 226 return (0); 227 } 228 229 /* 230 * Allocate and initialize an AHCI port. 231 */ 232 int 233 ahci_port_alloc(struct ahci_softc *sc, u_int port) 234 { 235 struct ahci_port *ap; 236 struct ata_port *at; 237 struct ahci_ccb *ccb; 238 u_int64_t dva; 239 u_int32_t cmd; 240 u_int32_t data; 241 struct ahci_cmd_hdr *hdr; 242 struct ahci_cmd_table *table; 243 int rc = ENOMEM; 244 int error; 245 int i; 246 247 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO); 248 ap->ap_err_scratch = kmalloc(512, M_DEVBUF, M_WAITOK | M_ZERO); 249 250 ksnprintf(ap->ap_name, sizeof(ap->ap_name), "%s%d.%d", 251 device_get_name(sc->sc_dev), 252 device_get_unit(sc->sc_dev), 253 port); 254 sc->sc_ports[port] = ap; 255 256 /* 257 * Allocate enough so we never have to reallocate, it makes 258 * it easier. 259 * 260 * ap_pmcount will be reduced by the scan if we encounter the 261 * port multiplier port prior to target 15. 262 * 263 * kmalloc power-of-2 allocations are guaranteed not to cross 264 * a page boundary. Make sure the identify sub-structure in the 265 * at structure does not cross a page boundary, just in case the 266 * part is AHCI-1.1 and can't handle multiple DRQ blocks. 267 */ 268 if (ap->ap_ata[0] == NULL) { 269 int pw2; 270 271 for (pw2 = 1; pw2 < sizeof(*at); pw2 <<= 1) 272 ; 273 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) { 274 at = kmalloc(pw2, M_DEVBUF, M_INTWAIT | M_ZERO); 275 ap->ap_ata[i] = at; 276 at->at_ahci_port = ap; 277 at->at_target = i; 278 at->at_probe = ATA_PROBE_NEED_INIT; 279 at->at_features |= ATA_PORT_F_RESCAN; 280 ksnprintf(at->at_name, sizeof(at->at_name), 281 "%s.%d", ap->ap_name, i); 282 } 283 } 284 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 285 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 286 device_printf(sc->sc_dev, 287 "unable to create register window for port %d\n", 288 port); 289 goto freeport; 290 } 291 292 ap->ap_sc = sc; 293 ap->ap_num = port; 294 ap->ap_probe = ATA_PROBE_NEED_INIT; 295 ap->link_pwr_mgmt = AHCI_LINK_PWR_MGMT_NONE; 296 ap->sysctl_tree = NULL; 297 TAILQ_INIT(&ap->ap_ccb_free); 298 TAILQ_INIT(&ap->ap_ccb_pending); 299 lockinit(&ap->ap_ccb_lock, "ahcipo", 0, 0); 300 301 /* Disable port interrupts */ 302 ahci_pwrite(ap, AHCI_PREG_IE, 0); 303 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 304 305 /* 306 * Sec 10.1.2 - deinitialise port if it is already running 307 */ 308 cmd = ahci_pread(ap, AHCI_PREG_CMD); 309 kprintf("%s: Caps %b\n", PORTNAME(ap), cmd, AHCI_PFMT_CMD); 310 311 if ((cmd & (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 312 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 313 (ahci_pread(ap, AHCI_PREG_SCTL) & AHCI_PREG_SCTL_DET)) { 314 int r; 315 316 r = ahci_port_stop(ap, 1); 317 if (r) { 318 device_printf(sc->sc_dev, 319 "unable to disable %s, ignoring port %d\n", 320 ((r == 2) ? "CR" : "FR"), port); 321 rc = ENXIO; 322 goto freeport; 323 } 324 325 /* Write DET to zero */ 326 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 327 } 328 329 /* Allocate RFIS */ 330 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sc->sc_tag_rfis); 331 if (ap->ap_dmamem_rfis == NULL) { 332 kprintf("%s: NORFIS\n", PORTNAME(ap)); 333 goto nomem; 334 } 335 336 /* Setup RFIS base address */ 337 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 338 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 339 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 340 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 341 342 /* Clear SERR before starting FIS reception or ST or anything */ 343 ahci_flush_tfd(ap); 344 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 345 346 /* Enable FIS reception and activate port. */ 347 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 348 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA); 349 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 350 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 351 352 /* Check whether port activated. Skip it if not. */ 353 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 354 if ((cmd & AHCI_PREG_CMD_FRE) == 0) { 355 kprintf("%s: NOT-ACTIVATED\n", PORTNAME(ap)); 356 rc = ENXIO; 357 goto freeport; 358 } 359 360 /* Allocate a CCB for each command slot */ 361 ap->ap_ccbs = kmalloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF, 362 M_WAITOK | M_ZERO); 363 if (ap->ap_ccbs == NULL) { 364 device_printf(sc->sc_dev, 365 "unable to allocate command list for port %d\n", 366 port); 367 goto freeport; 368 } 369 370 /* Command List Structures and Command Tables */ 371 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, sc->sc_tag_cmdh); 372 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, sc->sc_tag_cmdt); 373 if (ap->ap_dmamem_cmd_table == NULL || 374 ap->ap_dmamem_cmd_list == NULL) { 375 nomem: 376 device_printf(sc->sc_dev, 377 "unable to allocate DMA memory for port %d\n", 378 port); 379 goto freeport; 380 } 381 382 /* Setup command list base address */ 383 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 384 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 385 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 386 387 /* Split CCB allocation into CCBs and assign to command header/table */ 388 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 389 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 390 for (i = 0; i < sc->sc_ncmds; i++) { 391 ccb = &ap->ap_ccbs[i]; 392 393 error = bus_dmamap_create(sc->sc_tag_data, BUS_DMA_ALLOCNOW, 394 &ccb->ccb_dmamap); 395 if (error) { 396 device_printf(sc->sc_dev, 397 "unable to create dmamap for port %d " 398 "ccb %d\n", port, i); 399 goto freeport; 400 } 401 402 callout_init(&ccb->ccb_timeout); 403 ccb->ccb_slot = i; 404 ccb->ccb_port = ap; 405 ccb->ccb_cmd_hdr = &hdr[i]; 406 ccb->ccb_cmd_table = &table[i]; 407 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 408 ccb->ccb_slot * sizeof(struct ahci_cmd_table); 409 ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32)); 410 ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva); 411 412 ccb->ccb_xa.fis = 413 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 414 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 415 ccb->ccb_xa.tag = i; 416 417 ccb->ccb_xa.state = ATA_S_COMPLETE; 418 419 /* 420 * CCB[1] is the error CCB and is not get or put. It is 421 * also used for probing. Numerous HBAs only load the 422 * signature from CCB[1] so it MUST be used for the second 423 * FIS. 424 */ 425 if (i == 1) 426 ap->ap_err_ccb = ccb; 427 else 428 ahci_put_ccb(ccb); 429 } 430 431 /* 432 * Wait for ICC change to complete 433 */ 434 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC); 435 436 /* 437 * Calculate the interrupt mask 438 */ 439 data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 440 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | 441 AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE | 442 AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE | 443 AHCI_PREG_IE_DHRE | AHCI_PREG_IE_SDBE; 444 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 445 data |= AHCI_PREG_IE_IPME; 446 #ifdef AHCI_COALESCE 447 if (sc->sc_ccc_ports & (1 << port) 448 data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE); 449 #endif 450 ap->ap_intmask = data; 451 452 /* 453 * Start the port helper thread. The helper thread will call 454 * ahci_port_init() so the ports can all be started in parallel. 455 * A failure by ahci_port_init() does not deallocate the port 456 * since we still want hot-plug events. 457 */ 458 ahci_os_start_port(ap); 459 return(0); 460 freeport: 461 ahci_port_free(sc, port); 462 return (rc); 463 } 464 465 /* 466 * [re]initialize an idle port. No CCBs should be active. (from port thread) 467 * 468 * This function is called during the initial port allocation sequence 469 * and is also called on hot-plug insertion. We take no chances and 470 * use a portreset instead of a softreset. 471 * 472 * This function is the only way to move a failed port back to active 473 * status. 474 * 475 * Returns 0 if a device is successfully detected. 476 */ 477 int 478 ahci_port_init(struct ahci_port *ap) 479 { 480 u_int32_t cmd; 481 482 /* 483 * Register [re]initialization 484 * 485 * Flush the TFD and SERR and make sure the port is stopped before 486 * enabling its interrupt. We no longer cycle the port start as 487 * the port should not be started unless a device is present. 488 * 489 * XXX should we enable FIS reception? (FRE)? 490 */ 491 ahci_pwrite(ap, AHCI_PREG_IE, 0); 492 ahci_port_stop(ap, 0); 493 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 494 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 495 ahci_flush_tfd(ap); 496 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 497 498 /* 499 * If we are being harsh try to kill the port completely. 500 * 501 * AP_F_HARSH_REINIT is cleared in the hard reset state 502 */ 503 if (ap->ap_flags & AP_F_HARSH_REINIT) { 504 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 505 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 506 507 ahci_os_sleep(1000); 508 509 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 510 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA); 511 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 512 AHCI_PREG_CMD_SUD; 513 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 514 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 515 if ((cmd & AHCI_PREG_CMD_FRE) == 0) { 516 kprintf("%s: Warning: FRE did not come up during " 517 "harsh reinitialization\n", 518 PORTNAME(ap)); 519 } 520 ahci_os_sleep(1000); 521 } 522 523 /* 524 * Clear any pending garbage and re-enable the interrupt before 525 * going to the next stage. 526 */ 527 ap->ap_probe = ATA_PROBE_NEED_HARD_RESET; 528 ap->ap_pmcount = 0; 529 530 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 531 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 532 ahci_flush_tfd(ap); 533 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 534 ahci_pwrite(ap, AHCI_PREG_IS, -1); 535 536 ahci_port_interrupt_enable(ap); 537 538 return (0); 539 } 540 541 /* 542 * Enable or re-enable interrupts on a port. 543 * 544 * This routine is called from the port initialization code or from the 545 * helper thread as the real interrupt may be forced to turn off certain 546 * interrupt sources. 547 */ 548 void 549 ahci_port_interrupt_enable(struct ahci_port *ap) 550 { 551 ahci_pwrite(ap, AHCI_PREG_IE, ap->ap_intmask); 552 } 553 554 /* 555 * Manage the agressive link power management capability. 556 */ 557 void 558 ahci_port_link_pwr_mgmt(struct ahci_port *ap, int link_pwr_mgmt) 559 { 560 u_int32_t cmd, sctl; 561 562 if (link_pwr_mgmt == ap->link_pwr_mgmt) 563 return; 564 565 if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SALP) == 0) { 566 kprintf("%s: link power management not supported.\n", 567 PORTNAME(ap)); 568 return; 569 } 570 571 ahci_os_lock_port(ap); 572 573 if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_AGGR && 574 (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSC)) { 575 kprintf("%s: enabling aggressive link power management.\n", 576 PORTNAME(ap)); 577 578 ap->link_pwr_mgmt = link_pwr_mgmt; 579 580 ap->ap_intmask &= ~AHCI_PREG_IE_PRCE; 581 ahci_port_interrupt_enable(ap); 582 583 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 584 sctl &= ~(AHCI_PREG_SCTL_IPM_DISABLED); 585 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 586 587 /* 588 * Enable device initiated link power management for 589 * directly attached devices that support it. 590 */ 591 if (ap->ap_type != ATA_PORT_T_PM && 592 ap->ap_ata[0]->at_identify.satafsup & (1 << 3)) { 593 if (ahci_set_feature(ap, NULL, ATA_SATAFT_DEVIPS, 1)) 594 kprintf("%s: Could not enable device initiated " 595 "link power management.\n", 596 PORTNAME(ap)); 597 } 598 599 cmd = ahci_pread(ap, AHCI_PREG_CMD); 600 cmd |= AHCI_PREG_CMD_ASP; 601 cmd |= AHCI_PREG_CMD_ALPE; 602 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 603 604 } else if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_MEDIUM && 605 (ap->ap_sc->sc_cap & AHCI_REG_CAP_PSC)) { 606 kprintf("%s: enabling medium link power management.\n", 607 PORTNAME(ap)); 608 609 ap->link_pwr_mgmt = link_pwr_mgmt; 610 611 ap->ap_intmask &= ~AHCI_PREG_IE_PRCE; 612 ahci_port_interrupt_enable(ap); 613 614 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 615 sctl |= AHCI_PREG_SCTL_IPM_DISABLED; 616 sctl &= ~AHCI_PREG_SCTL_IPM_NOPARTIAL; 617 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 618 619 cmd = ahci_pread(ap, AHCI_PREG_CMD); 620 cmd &= ~AHCI_PREG_CMD_ASP; 621 cmd |= AHCI_PREG_CMD_ALPE; 622 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 623 624 } else if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_NONE) { 625 kprintf("%s: disabling link power management.\n", 626 PORTNAME(ap)); 627 628 /* Disable device initiated link power management */ 629 if (ap->ap_type != ATA_PORT_T_PM && 630 ap->ap_ata[0]->at_identify.satafsup & (1 << 3)) 631 ahci_set_feature(ap, NULL, ATA_SATAFT_DEVIPS, 0); 632 633 cmd = ahci_pread(ap, AHCI_PREG_CMD); 634 cmd &= ~(AHCI_PREG_CMD_ALPE | AHCI_PREG_CMD_ASP); 635 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 636 637 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 638 sctl |= AHCI_PREG_SCTL_IPM_DISABLED; 639 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 640 641 /* let the drive come back to avoid PRCS interrupts later */ 642 ahci_os_unlock_port(ap); 643 ahci_os_sleep(1000); 644 ahci_os_lock_port(ap); 645 646 ahci_pwrite(ap, AHCI_PREG_SERR, 647 AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_W); 648 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS); 649 650 ap->ap_intmask |= AHCI_PREG_IE_PRCE; 651 ahci_port_interrupt_enable(ap); 652 653 ap->link_pwr_mgmt = link_pwr_mgmt; 654 } else { 655 kprintf("%s: unsupported link power management state %d.\n", 656 PORTNAME(ap), link_pwr_mgmt); 657 } 658 659 ahci_os_unlock_port(ap); 660 } 661 662 /* 663 * Return current link power state. 664 */ 665 int 666 ahci_port_link_pwr_state(struct ahci_port *ap) 667 { 668 uint32_t r; 669 670 r = ahci_pread(ap, AHCI_PREG_SSTS); 671 switch (r & SATA_PM_SSTS_IPM) { 672 case SATA_PM_SSTS_IPM_ACTIVE: 673 return 1; 674 case SATA_PM_SSTS_IPM_PARTIAL: 675 return 2; 676 case SATA_PM_SSTS_IPM_SLUMBER: 677 return 3; 678 default: 679 return 0; 680 } 681 } 682 683 /* 684 * Run the port / target state machine from a main context. 685 * 686 * The state machine for the port is always run. 687 * 688 * If atx is non-NULL run the state machine for a particular target. 689 * If atx is NULL run the state machine for all targets. 690 */ 691 void 692 ahci_port_state_machine(struct ahci_port *ap, int initial) 693 { 694 struct ata_port *at; 695 u_int32_t data; 696 int target; 697 int didsleep; 698 int loop; 699 700 /* 701 * State machine for port. Note that CAM is not yet associated 702 * during the initial parallel probe and the port's probe state 703 * will not get past ATA_PROBE_NEED_IDENT. 704 */ 705 { 706 if (initial == 0 && ap->ap_probe <= ATA_PROBE_NEED_HARD_RESET) { 707 kprintf("%s: Waiting 10 seconds on insertion\n", 708 PORTNAME(ap)); 709 ahci_os_sleep(10000); 710 initial = 1; 711 } 712 if (ap->ap_probe == ATA_PROBE_NEED_INIT) 713 ahci_port_init(ap); 714 if (ap->ap_probe == ATA_PROBE_NEED_HARD_RESET) 715 ahci_port_reset(ap, NULL, 1); 716 if (ap->ap_probe == ATA_PROBE_NEED_SOFT_RESET) 717 ahci_port_reset(ap, NULL, 0); 718 if (ap->ap_probe == ATA_PROBE_NEED_IDENT) 719 ahci_cam_probe(ap, NULL); 720 } 721 if (ap->ap_type != ATA_PORT_T_PM) { 722 if (ap->ap_probe == ATA_PROBE_FAILED) { 723 ahci_cam_changed(ap, NULL, 0); 724 } else if (ap->ap_probe >= ATA_PROBE_NEED_IDENT) { 725 ahci_cam_changed(ap, NULL, 1); 726 } 727 return; 728 } 729 730 /* 731 * Port Multiplier state machine. 732 * 733 * Get a mask of changed targets and combine with any runnable 734 * states already present. 735 */ 736 for (loop = 0; ;++loop) { 737 if (ahci_pm_read(ap, 15, SATA_PMREG_EINFO, &data)) { 738 kprintf("%s: PM unable to read hot-plug bitmap\n", 739 PORTNAME(ap)); 740 break; 741 } 742 743 /* 744 * Do at least one loop, then stop if no more state changes 745 * have occured. The PM might not generate a new 746 * notification until we clear the entire bitmap. 747 */ 748 if (loop && data == 0) 749 break; 750 751 /* 752 * New devices showing up in the bitmap require some spin-up 753 * time before we start probing them. Reset didsleep. The 754 * first new device we detect will sleep before probing. 755 * 756 * This only applies to devices whos change bit is set in 757 * the data, and does not apply to the initial boot-time 758 * probe. 759 */ 760 didsleep = 0; 761 762 for (target = 0; target < ap->ap_pmcount; ++target) { 763 at = ap->ap_ata[target]; 764 765 /* 766 * Check the target state for targets behind the PM 767 * which have changed state. This will adjust 768 * at_probe and set ATA_PORT_F_RESCAN 769 * 770 * We want to wait at least 10 seconds before probing 771 * a newly inserted device. If the check status 772 * indicates a device is present and in need of a 773 * hard reset, we make sure we have slept before 774 * continuing. 775 * 776 * We also need to wait at least 1 second for the 777 * PHY state to change after insertion, if we 778 * haven't already waited the 10 seconds. 779 * 780 * NOTE: When pm_check_good finds a good port it 781 * typically starts us in probe state 782 * NEED_HARD_RESET rather than INIT. 783 */ 784 if (data & (1 << target)) { 785 if (initial == 0 && didsleep == 0) 786 ahci_os_sleep(1000); 787 ahci_pm_check_good(ap, target); 788 if (initial == 0 && didsleep == 0 && 789 at->at_probe <= ATA_PROBE_NEED_HARD_RESET 790 ) { 791 didsleep = 1; 792 kprintf("%s: Waiting 10 seconds on insertion\n", PORTNAME(ap)); 793 ahci_os_sleep(10000); 794 } 795 } 796 797 /* 798 * Report hot-plug events before the probe state 799 * really gets hot. Only actual events are reported 800 * here to reduce spew. 801 */ 802 if (data & (1 << target)) { 803 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap, at)); 804 switch(at->at_probe) { 805 case ATA_PROBE_NEED_INIT: 806 case ATA_PROBE_NEED_HARD_RESET: 807 kprintf("Device inserted\n"); 808 break; 809 case ATA_PROBE_FAILED: 810 kprintf("Device removed\n"); 811 break; 812 default: 813 kprintf("Device probe in progress\n"); 814 break; 815 } 816 } 817 818 /* 819 * Run through the state machine as necessary if 820 * the port is not marked failed. 821 * 822 * The state machine may stop at NEED_IDENT if 823 * CAM is not yet attached. 824 * 825 * Acquire exclusive access to the port while we 826 * are doing this. This prevents command-completion 827 * from queueing commands for non-polled targets 828 * inbetween our probe steps. We need to do this 829 * because the reset probes can generate severe PHY 830 * and protocol errors and soft-brick the port. 831 */ 832 if (at->at_probe != ATA_PROBE_FAILED && 833 at->at_probe != ATA_PROBE_GOOD) { 834 ahci_beg_exclusive_access(ap, at); 835 if (at->at_probe == ATA_PROBE_NEED_INIT) 836 ahci_pm_port_init(ap, at); 837 if (at->at_probe == ATA_PROBE_NEED_HARD_RESET) 838 ahci_port_reset(ap, at, 1); 839 if (at->at_probe == ATA_PROBE_NEED_SOFT_RESET) 840 ahci_port_reset(ap, at, 0); 841 if (at->at_probe == ATA_PROBE_NEED_IDENT) 842 ahci_cam_probe(ap, at); 843 ahci_end_exclusive_access(ap, at); 844 } 845 846 /* 847 * Add or remove from CAM 848 */ 849 if (at->at_features & ATA_PORT_F_RESCAN) { 850 at->at_features &= ~ATA_PORT_F_RESCAN; 851 if (at->at_probe == ATA_PROBE_FAILED) { 852 ahci_cam_changed(ap, at, 0); 853 } else if (at->at_probe >= ATA_PROBE_NEED_IDENT) { 854 ahci_cam_changed(ap, at, 1); 855 } 856 } 857 data &= ~(1 << target); 858 } 859 if (data) { 860 kprintf("%s: WARNING (PM): extra bits set in " 861 "EINFO: %08x\n", PORTNAME(ap), data); 862 while (target < AHCI_MAX_PMPORTS) { 863 ahci_pm_check_good(ap, target); 864 ++target; 865 } 866 } 867 } 868 } 869 870 871 /* 872 * De-initialize and detach a port. 873 */ 874 void 875 ahci_port_free(struct ahci_softc *sc, u_int port) 876 { 877 struct ahci_port *ap = sc->sc_ports[port]; 878 struct ahci_ccb *ccb; 879 int i; 880 881 /* 882 * Ensure port is disabled and its interrupts are all flushed. 883 */ 884 if (ap->ap_sc) { 885 ahci_port_stop(ap, 1); 886 ahci_os_stop_port(ap); 887 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 888 ahci_pwrite(ap, AHCI_PREG_IE, 0); 889 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 890 ahci_write(sc, AHCI_REG_IS, 1 << port); 891 } 892 893 if (ap->ap_ccbs) { 894 while ((ccb = ahci_get_ccb(ap)) != NULL) { 895 if (ccb->ccb_dmamap) { 896 bus_dmamap_destroy(sc->sc_tag_data, 897 ccb->ccb_dmamap); 898 ccb->ccb_dmamap = NULL; 899 } 900 } 901 if ((ccb = ap->ap_err_ccb) != NULL) { 902 if (ccb->ccb_dmamap) { 903 bus_dmamap_destroy(sc->sc_tag_data, 904 ccb->ccb_dmamap); 905 ccb->ccb_dmamap = NULL; 906 } 907 ap->ap_err_ccb = NULL; 908 } 909 kfree(ap->ap_ccbs, M_DEVBUF); 910 ap->ap_ccbs = NULL; 911 } 912 913 if (ap->ap_dmamem_cmd_list) { 914 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 915 ap->ap_dmamem_cmd_list = NULL; 916 } 917 if (ap->ap_dmamem_rfis) { 918 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 919 ap->ap_dmamem_rfis = NULL; 920 } 921 if (ap->ap_dmamem_cmd_table) { 922 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 923 ap->ap_dmamem_cmd_table = NULL; 924 } 925 if (ap->ap_ata) { 926 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) { 927 if (ap->ap_ata[i]) { 928 kfree(ap->ap_ata[i], M_DEVBUF); 929 ap->ap_ata[i] = NULL; 930 } 931 } 932 } 933 if (ap->ap_err_scratch) { 934 kfree(ap->ap_err_scratch, M_DEVBUF); 935 ap->ap_err_scratch = NULL; 936 } 937 938 /* bus_space(9) says we dont free the subregions handle */ 939 940 kfree(ap, M_DEVBUF); 941 sc->sc_ports[port] = NULL; 942 } 943 944 static 945 u_int32_t 946 ahci_pactive(struct ahci_port *ap) 947 { 948 u_int32_t mask; 949 950 mask = ahci_pread(ap, AHCI_PREG_CI); 951 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 952 mask |= ahci_pread(ap, AHCI_PREG_SACT); 953 return(mask); 954 } 955 956 /* 957 * Start high-level command processing on the port 958 */ 959 int 960 ahci_port_start(struct ahci_port *ap) 961 { 962 u_int32_t r, s, is, tfd; 963 964 /* 965 * FRE must be turned on before ST. Wait for FR to go active 966 * before turning on ST. The spec doesn't seem to think this 967 * is necessary but waiting here avoids an on-off race in the 968 * ahci_port_stop() code. 969 */ 970 r = ahci_pread(ap, AHCI_PREG_CMD); 971 if ((r & AHCI_PREG_CMD_FRE) == 0) { 972 r |= AHCI_PREG_CMD_FRE; 973 ahci_pwrite(ap, AHCI_PREG_CMD, r); 974 } 975 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) { 976 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) { 977 kprintf("%s: Cannot start FIS reception\n", 978 PORTNAME(ap)); 979 return (2); 980 } 981 } else { 982 ahci_os_sleep(10); 983 } 984 985 /* 986 * Turn on ST, wait for CR to come up. 987 */ 988 r |= AHCI_PREG_CMD_ST; 989 ahci_pwrite(ap, AHCI_PREG_CMD, r); 990 if (ahci_pwait_set_to(ap, 2000, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) { 991 s = ahci_pread(ap, AHCI_PREG_SERR); 992 is = ahci_pread(ap, AHCI_PREG_IS); 993 tfd = ahci_pread(ap, AHCI_PREG_TFD); 994 kprintf("%s: Cannot start command DMA\n" 995 "NCMP=%b NSERR=%b\n" 996 "NEWIS=%b\n" 997 "NEWTFD=%b\n", 998 PORTNAME(ap), 999 r, AHCI_PFMT_CMD, s, AHCI_PFMT_SERR, 1000 is, AHCI_PFMT_IS, 1001 tfd, AHCI_PFMT_TFD_STS); 1002 return (1); 1003 } 1004 1005 #ifdef AHCI_COALESCE 1006 /* 1007 * (Re-)enable coalescing on the port. 1008 */ 1009 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 1010 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 1011 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 1012 ap->ap_sc->sc_ccc_ports_cur); 1013 } 1014 #endif 1015 1016 return (0); 1017 } 1018 1019 /* 1020 * Stop high-level command processing on a port 1021 * 1022 * WARNING! If the port is stopped while CR is still active our saved 1023 * CI/SACT will race any commands completed by the command 1024 * processor prior to being able to stop. Thus we never call 1025 * this function unless we intend to dispose of any remaining 1026 * active commands. In particular, this complicates the timeout 1027 * code. 1028 */ 1029 int 1030 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 1031 { 1032 u_int32_t r; 1033 1034 #ifdef AHCI_COALESCE 1035 /* 1036 * Disable coalescing on the port while it is stopped. 1037 */ 1038 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 1039 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 1040 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 1041 ap->ap_sc->sc_ccc_ports_cur); 1042 } 1043 #endif 1044 1045 /* 1046 * Turn off ST, then wait for CR to go off. 1047 */ 1048 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1049 r &= ~AHCI_PREG_CMD_ST; 1050 ahci_pwrite(ap, AHCI_PREG_CMD, r); 1051 1052 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) { 1053 kprintf("%s: Port bricked, unable to stop (ST)\n", 1054 PORTNAME(ap)); 1055 return (1); 1056 } 1057 1058 #if 0 1059 /* 1060 * Turn off FRE, then wait for FR to go off. FRE cannot 1061 * be turned off until CR transitions to 0. 1062 */ 1063 if ((r & AHCI_PREG_CMD_FR) == 0) { 1064 kprintf("%s: FR stopped, clear FRE for next start\n", 1065 PORTNAME(ap)); 1066 stop_fis_rx = 2; 1067 } 1068 #endif 1069 if (stop_fis_rx) { 1070 r &= ~AHCI_PREG_CMD_FRE; 1071 ahci_pwrite(ap, AHCI_PREG_CMD, r); 1072 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) { 1073 kprintf("%s: Port bricked, unable to stop (FRE)\n", 1074 PORTNAME(ap)); 1075 return (2); 1076 } 1077 } 1078 1079 return (0); 1080 } 1081 1082 /* 1083 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} 1084 */ 1085 int 1086 ahci_port_clo(struct ahci_port *ap) 1087 { 1088 struct ahci_softc *sc = ap->ap_sc; 1089 u_int32_t cmd; 1090 1091 /* Only attempt CLO if supported by controller */ 1092 if ((ahci_read(sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) == 0) 1093 return (1); 1094 1095 /* Issue CLO */ 1096 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1097 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 1098 1099 /* Wait for completion */ 1100 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) { 1101 kprintf("%s: CLO did not complete\n", PORTNAME(ap)); 1102 return (1); 1103 } 1104 1105 return (0); 1106 } 1107 1108 /* 1109 * Reset a port. 1110 * 1111 * If hard is 0 perform a softreset of the port. 1112 * If hard is 1 perform a hard reset of the port. 1113 * 1114 * If at is non-NULL an indirect port via a port-multiplier is being 1115 * reset, otherwise a direct port is being reset. 1116 * 1117 * NOTE: Indirect ports can only be soft-reset. 1118 */ 1119 int 1120 ahci_port_reset(struct ahci_port *ap, struct ata_port *at, int hard) 1121 { 1122 int rc; 1123 1124 if (hard) { 1125 if (at) 1126 rc = ahci_pm_hardreset(ap, at->at_target, hard); 1127 else 1128 rc = ahci_port_hardreset(ap, hard); 1129 } else { 1130 if (at) 1131 rc = ahci_pm_softreset(ap, at->at_target); 1132 else 1133 rc = ahci_port_softreset(ap); 1134 } 1135 return(rc); 1136 } 1137 1138 /* 1139 * AHCI soft reset, Section 10.4.1 1140 * 1141 * (at) will be NULL when soft-resetting a directly-attached device, and 1142 * non-NULL when soft-resetting a device through a port multiplier. 1143 * 1144 * This function keeps port communications intact and attempts to generate 1145 * a reset to the connected device using device commands. 1146 */ 1147 int 1148 ahci_port_softreset(struct ahci_port *ap) 1149 { 1150 struct ahci_ccb *ccb = NULL; 1151 struct ahci_cmd_hdr *cmd_slot; 1152 u_int8_t *fis; 1153 int error; 1154 1155 error = EIO; 1156 1157 if (bootverbose) { 1158 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap), 1159 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD); 1160 } 1161 1162 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 1163 1164 crit_enter(); 1165 ap->ap_flags |= AP_F_IN_RESET; 1166 ap->ap_state = AP_S_NORMAL; 1167 1168 /* 1169 * Remember port state in cmd (main to restore start/stop) 1170 * 1171 * Idle port. 1172 */ 1173 if (ahci_port_stop(ap, 0)) { 1174 kprintf("%s: failed to stop port, cannot softreset\n", 1175 PORTNAME(ap)); 1176 goto err; 1177 } 1178 1179 /* 1180 * Request CLO if device appears hung. 1181 */ 1182 if (ahci_pread(ap, AHCI_PREG_TFD) & 1183 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1184 ahci_port_clo(ap); 1185 } 1186 1187 /* 1188 * This is an attempt to clear errors so a new signature will 1189 * be latched. It isn't working properly. XXX 1190 */ 1191 ahci_flush_tfd(ap); 1192 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1193 1194 /* Restart port */ 1195 if (ahci_port_start(ap)) { 1196 kprintf("%s: failed to start port, cannot softreset\n", 1197 PORTNAME(ap)); 1198 goto err; 1199 } 1200 1201 /* Check whether CLO worked */ 1202 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1203 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1204 kprintf("%s: CLO %s, need port reset\n", 1205 PORTNAME(ap), 1206 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) 1207 ? "failed" : "unsupported"); 1208 error = EBUSY; 1209 goto err; 1210 } 1211 1212 /* 1213 * Prep first D2H command with SRST feature & clear busy/reset flags 1214 * 1215 * It is unclear which other fields in the FIS are used. Just zero 1216 * everything. 1217 * 1218 * NOTE! This CCB is used for both the first and second commands. 1219 * The second command must use CCB slot 1 to properly load 1220 * the signature. 1221 */ 1222 ccb = ahci_get_err_ccb(ap); 1223 ccb->ccb_xa.complete = ahci_dummy_done; 1224 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_EXCLUSIVE; 1225 KKASSERT(ccb->ccb_slot == 1); 1226 ccb->ccb_xa.at = NULL; 1227 cmd_slot = ccb->ccb_cmd_hdr; 1228 1229 fis = ccb->ccb_cmd_table->cfis; 1230 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis)); 1231 fis[0] = ATA_FIS_TYPE_H2D; 1232 fis[15] = ATA_FIS_CONTROL_SRST|ATA_FIS_CONTROL_4BIT; 1233 1234 cmd_slot->prdtl = 0; 1235 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 1236 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */ 1237 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */ 1238 1239 ccb->ccb_xa.state = ATA_S_PENDING; 1240 1241 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 1242 kprintf("%s: First FIS failed\n", PORTNAME(ap)); 1243 goto err; 1244 } 1245 1246 /* 1247 * WARNING! TIME SENSITIVE SPACE! WARNING! 1248 * 1249 * The two FISes are supposed to be back to back. Don't issue other 1250 * commands or even delay if we can help it. 1251 */ 1252 1253 /* 1254 * Prep second D2H command to read status and complete reset sequence 1255 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA 1256 * Rev 2.6 and it is unclear how the second FIS should be set up 1257 * from the AHCI document. 1258 * 1259 * Give the device 3ms before sending the second FIS. 1260 * 1261 * It is unclear which other fields in the FIS are used. Just zero 1262 * everything. 1263 */ 1264 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_AUTOSENSE | ATA_F_EXCLUSIVE; 1265 1266 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis)); 1267 fis[0] = ATA_FIS_TYPE_H2D; 1268 fis[15] = ATA_FIS_CONTROL_4BIT; 1269 1270 cmd_slot->prdtl = 0; 1271 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 1272 1273 ccb->ccb_xa.state = ATA_S_PENDING; 1274 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 1275 kprintf("%s: Second FIS failed\n", PORTNAME(ap)); 1276 goto err; 1277 } 1278 1279 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1280 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1281 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1282 PORTNAME(ap), 1283 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 1284 error = EBUSY; 1285 goto err; 1286 } 1287 ahci_os_sleep(10); 1288 1289 /* 1290 * If the softreset is trying to clear a BSY condition after a 1291 * normal portreset we assign the port type. 1292 * 1293 * If the softreset is being run first as part of the ccb error 1294 * processing code then report if the device signature changed 1295 * unexpectedly. 1296 */ 1297 if (ap->ap_type == ATA_PORT_T_NONE) { 1298 ap->ap_type = ahci_port_signature_detect(ap, NULL); 1299 } else { 1300 if (ahci_port_signature_detect(ap, NULL) != ap->ap_type) { 1301 kprintf("%s: device signature unexpectedly " 1302 "changed\n", PORTNAME(ap)); 1303 error = EBUSY; /* XXX */ 1304 } 1305 } 1306 error = 0; 1307 1308 ahci_os_sleep(3); 1309 err: 1310 if (ccb != NULL) { 1311 ahci_put_err_ccb(ccb); 1312 1313 /* 1314 * If the target is busy use CLO to clear the busy 1315 * condition. The BSY should be cleared on the next 1316 * start. 1317 */ 1318 if (ahci_pread(ap, AHCI_PREG_TFD) & 1319 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1320 ahci_port_clo(ap); 1321 } 1322 } 1323 1324 /* 1325 * If we failed to softreset make the port quiescent, otherwise 1326 * make sure the port's start/stop state matches what it was on 1327 * entry. 1328 * 1329 * Don't kill the port if the softreset is on a port multiplier 1330 * target, that would kill all the targets! 1331 */ 1332 if (error) { 1333 ahci_port_hardstop(ap); 1334 /* ap_probe set to failed */ 1335 } else { 1336 ap->ap_probe = ATA_PROBE_NEED_IDENT; 1337 ap->ap_pmcount = 1; 1338 ahci_port_start(ap); 1339 } 1340 ap->ap_flags &= ~AP_F_IN_RESET; 1341 crit_exit(); 1342 1343 if (bootverbose) 1344 kprintf("%s: END SOFTRESET\n", PORTNAME(ap)); 1345 1346 return (error); 1347 } 1348 1349 /* 1350 * AHCI port reset, Section 10.4.2 1351 * 1352 * This function does a hard reset of the port. Note that the device 1353 * connected to the port could still end-up hung. 1354 */ 1355 int 1356 ahci_port_hardreset(struct ahci_port *ap, int hard) 1357 { 1358 u_int32_t cmd, r; 1359 u_int32_t data; 1360 int error; 1361 int loop; 1362 1363 if (bootverbose) 1364 kprintf("%s: START HARDRESET\n", PORTNAME(ap)); 1365 ap->ap_flags |= AP_F_IN_RESET; 1366 1367 /* 1368 * Idle the port, 1369 */ 1370 ahci_port_stop(ap, 0); 1371 ap->ap_state = AP_S_NORMAL; 1372 1373 /* 1374 * The port may have been quiescent with its SUD bit cleared, so 1375 * set the SUD (spin up device). 1376 */ 1377 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1378 cmd |= AHCI_PREG_CMD_SUD; 1379 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1380 1381 /* 1382 * Perform device detection. 1383 * 1384 * NOTE! AHCi_PREG_SCTL_DET_DISABLE seems to be highly unreliable 1385 * on multiple chipsets and can brick the chipset or even 1386 * the whole PC. Never use it. 1387 */ 1388 ap->ap_type = ATA_PORT_T_NONE; 1389 1390 r = AHCI_PREG_SCTL_IPM_DISABLED; 1391 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1392 ahci_os_sleep(10); 1393 1394 /* 1395 * Start transmitting COMRESET. COMRESET must be sent for at 1396 * least 1ms. 1397 */ 1398 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1399 if (AhciForceGen1 & (1 << ap->ap_num)) 1400 r |= AHCI_PREG_SCTL_SPD_GEN1; 1401 else 1402 r |= AHCI_PREG_SCTL_SPD_ANY; 1403 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1404 1405 /* 1406 * Through trial and error it seems to take around 100ms 1407 * for the detect logic to settle down. If this is too 1408 * short the softreset code will fail. 1409 */ 1410 if (ap->ap_flags & AP_F_HARSH_REINIT) 1411 ahci_os_sleep(1000); 1412 else 1413 ahci_os_sleep(200); 1414 ap->ap_flags &= ~AP_F_HARSH_REINIT; 1415 1416 /* 1417 * Only SERR_DIAG_X needs to be cleared for TFD updates, but 1418 * since we are hard-resetting the port we might as well clear 1419 * the whole enchillada 1420 */ 1421 ahci_flush_tfd(ap); 1422 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1423 r &= ~AHCI_PREG_SCTL_DET_INIT; 1424 r |= AHCI_PREG_SCTL_DET_NONE; 1425 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1426 1427 /* 1428 * Try to determine if there is a device on the port. 1429 * 1430 * Give the device 3/10 second to at least be detected. 1431 * If we fail clear PRCS (phy detect) since we may cycled 1432 * the phy and probably caused another PRCS interrupt. 1433 */ 1434 loop = 300; 1435 while (loop > 0) { 1436 r = ahci_pread(ap, AHCI_PREG_SSTS); 1437 if (r & AHCI_PREG_SSTS_DET) 1438 break; 1439 loop -= ahci_os_softsleep(); 1440 } 1441 if (loop == 0) { 1442 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS); 1443 if (bootverbose) { 1444 kprintf("%s: Port appears to be unplugged\n", 1445 PORTNAME(ap)); 1446 } 1447 error = ENODEV; 1448 goto done; 1449 } 1450 1451 /* 1452 * There is something on the port. Give the device 3 seconds 1453 * to fully negotiate. 1454 */ 1455 if (ahci_pwait_eq(ap, 3000, AHCI_PREG_SSTS, 1456 AHCI_PREG_SSTS_DET, AHCI_PREG_SSTS_DET_DEV)) { 1457 if (bootverbose) { 1458 kprintf("%s: Device may be powered down\n", 1459 PORTNAME(ap)); 1460 } 1461 error = ENODEV; 1462 goto pmdetect; 1463 } 1464 1465 /* 1466 * We got something that definitely looks like a device. Give 1467 * the device time to send us its first D2H FIS. Waiting for 1468 * BSY to clear accomplishes this. 1469 * 1470 * NOTE that a port multiplier may or may not clear BSY here, 1471 * depending on what is sitting in target 0 behind it. 1472 */ 1473 ahci_flush_tfd(ap); 1474 1475 if (ahci_pwait_clr_to(ap, 3000, AHCI_PREG_TFD, 1476 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1477 error = EBUSY; 1478 } else { 1479 error = 0; 1480 } 1481 1482 pmdetect: 1483 /* 1484 * Do the PM port probe regardless of how things turned out on 1485 * the BSY check. 1486 */ 1487 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SPM) 1488 error = ahci_pm_port_probe(ap, error); 1489 1490 done: 1491 /* 1492 * Finish up. 1493 */ 1494 switch(error) { 1495 case 0: 1496 /* 1497 * All good, make sure the port is running and set the 1498 * probe state. Ignore the signature junk (it's unreliable) 1499 * until we get to the softreset code. 1500 */ 1501 if (ahci_port_start(ap)) { 1502 kprintf("%s: failed to start command DMA on port, " 1503 "disabling\n", PORTNAME(ap)); 1504 error = EBUSY; 1505 goto done; 1506 } 1507 if (ap->ap_type == ATA_PORT_T_PM) 1508 ap->ap_probe = ATA_PROBE_GOOD; 1509 else 1510 ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET; 1511 break; 1512 case ENODEV: 1513 /* 1514 * Normal device probe failure 1515 */ 1516 data = ahci_pread(ap, AHCI_PREG_SSTS); 1517 1518 switch(data & AHCI_PREG_SSTS_DET) { 1519 case AHCI_PREG_SSTS_DET_DEV_NE: 1520 kprintf("%s: Device not communicating\n", 1521 PORTNAME(ap)); 1522 break; 1523 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 1524 kprintf("%s: PHY offline\n", 1525 PORTNAME(ap)); 1526 break; 1527 default: 1528 kprintf("%s: No device detected\n", 1529 PORTNAME(ap)); 1530 break; 1531 } 1532 ahci_port_hardstop(ap); 1533 break; 1534 default: 1535 /* 1536 * Abnormal probe (EBUSY) 1537 */ 1538 kprintf("%s: Device on port is bricked\n", 1539 PORTNAME(ap)); 1540 ahci_port_hardstop(ap); 1541 #if 0 1542 rc = ahci_port_reset(ap, atx, 0); 1543 if (rc) { 1544 kprintf("%s: Unable unbrick device\n", 1545 PORTNAME(ap)); 1546 } else { 1547 kprintf("%s: Successfully unbricked\n", 1548 PORTNAME(ap)); 1549 } 1550 #endif 1551 break; 1552 } 1553 1554 /* 1555 * Clean up 1556 */ 1557 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1558 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS); 1559 1560 ap->ap_flags &= ~AP_F_IN_RESET; 1561 1562 if (bootverbose) 1563 kprintf("%s: END HARDRESET %d\n", PORTNAME(ap), error); 1564 return (error); 1565 } 1566 1567 /* 1568 * Hard-stop on hot-swap device removal. See 10.10.1 1569 * 1570 * Place the port in a mode that will allow it to detect hot-swap insertions. 1571 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't 1572 * seem to do the job. 1573 * 1574 * FIS reception is left enabled but command processing is disabled. 1575 * Cycling FIS reception (FRE) can brick ports. 1576 */ 1577 void 1578 ahci_port_hardstop(struct ahci_port *ap) 1579 { 1580 struct ahci_ccb *ccb; 1581 struct ata_port *at; 1582 u_int32_t r; 1583 u_int32_t cmd; 1584 int slot; 1585 int i; 1586 1587 /* 1588 * Stop the port. We can't modify things like SUD if the port 1589 * is running. 1590 */ 1591 ap->ap_state = AP_S_FATAL_ERROR; 1592 ap->ap_probe = ATA_PROBE_FAILED; 1593 ap->ap_type = ATA_PORT_T_NONE; 1594 ahci_port_stop(ap, 0); 1595 cmd = ahci_pread(ap, AHCI_PREG_CMD); 1596 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA | AHCI_PREG_CMD_ICC); 1597 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1598 1599 /* 1600 * Clean up AT sub-ports on SATA port. 1601 */ 1602 for (i = 0; ap->ap_ata && i < AHCI_MAX_PMPORTS; ++i) { 1603 at = ap->ap_ata[i]; 1604 at->at_type = ATA_PORT_T_NONE; 1605 at->at_probe = ATA_PROBE_FAILED; 1606 } 1607 1608 /* 1609 * Make sure FRE is active. There isn't anything we can do if it 1610 * fails so just ignore errors. 1611 */ 1612 if ((cmd & AHCI_PREG_CMD_FRE) == 0) { 1613 cmd |= AHCI_PREG_CMD_FRE; 1614 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1615 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) 1616 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR); 1617 } 1618 1619 /* 1620 * 10.10.3 DET must be set to 0 before setting SUD to 0. 1621 * 10.10.1 place us in the Listen state. 1622 * 1623 * Deactivating SUD only applies if the controller supports SUD. 1624 */ 1625 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 1626 ahci_os_sleep(1); 1627 if (cmd & AHCI_PREG_CMD_SUD) { 1628 cmd &= ~AHCI_PREG_CMD_SUD; 1629 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1630 } 1631 ahci_os_sleep(1); 1632 1633 /* 1634 * Transition su to the spin-up state. HVA shall send COMRESET and 1635 * begin initialization sequence (whatever that means). 1636 * 1637 * This only applies if the controller supports SUD. 1638 * NEVER use AHCI_PREG_DET_DISABLE. 1639 */ 1640 cmd |= AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 1641 cmd |= AHCI_PREG_CMD_ICC_ACTIVE; 1642 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1643 ahci_os_sleep(1); 1644 1645 /* 1646 * Transition us to the Reset state. Theoretically we send a 1647 * continuous stream of COMRESETs in this state. 1648 */ 1649 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1650 if (AhciForceGen1 & (1 << ap->ap_num)) { 1651 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap)); 1652 r |= AHCI_PREG_SCTL_SPD_GEN1; 1653 } else { 1654 r |= AHCI_PREG_SCTL_SPD_ANY; 1655 } 1656 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1657 ahci_os_sleep(1); 1658 1659 /* 1660 * Flush SERR_DIAG_X so the TFD can update. 1661 */ 1662 ahci_flush_tfd(ap); 1663 1664 /* 1665 * Clean out pending ccbs 1666 */ 1667 while (ap->ap_active) { 1668 slot = ffs(ap->ap_active) - 1; 1669 ap->ap_active &= ~(1 << slot); 1670 ap->ap_expired &= ~(1 << slot); 1671 --ap->ap_active_cnt; 1672 ccb = &ap->ap_ccbs[slot]; 1673 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_RUNNING) { 1674 callout_stop(&ccb->ccb_timeout); 1675 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 1676 } 1677 ccb->ccb_xa.flags &= ~(ATA_F_TIMEOUT_DESIRED | 1678 ATA_F_TIMEOUT_EXPIRED); 1679 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1680 ccb->ccb_done(ccb); 1681 ccb->ccb_xa.complete(&ccb->ccb_xa); 1682 } 1683 while (ap->ap_sactive) { 1684 slot = ffs(ap->ap_sactive) - 1; 1685 ap->ap_sactive &= ~(1 << slot); 1686 ap->ap_expired &= ~(1 << slot); 1687 ccb = &ap->ap_ccbs[slot]; 1688 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_RUNNING) { 1689 callout_stop(&ccb->ccb_timeout); 1690 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 1691 } 1692 ccb->ccb_xa.flags &= ~(ATA_F_TIMEOUT_DESIRED | 1693 ATA_F_TIMEOUT_EXPIRED); 1694 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1695 ccb->ccb_done(ccb); 1696 ccb->ccb_xa.complete(&ccb->ccb_xa); 1697 } 1698 KKASSERT(ap->ap_active_cnt == 0); 1699 1700 while ((ccb = TAILQ_FIRST(&ap->ap_ccb_pending)) != NULL) { 1701 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1702 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1703 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_DESIRED; 1704 ccb->ccb_done(ccb); 1705 ccb->ccb_xa.complete(&ccb->ccb_xa); 1706 } 1707 1708 /* 1709 * Leave us in COMRESET (both SUD and INIT active), the HBA should 1710 * hopefully send us a DIAG_X-related interrupt if it receives 1711 * a COMINIT, and if not that then at least a Phy transition 1712 * interrupt. 1713 * 1714 * If we transition INIT from 1->0 to begin the initalization 1715 * sequence it is unclear if that sequence will remain active 1716 * until the next device insertion. 1717 * 1718 * If we go back to the listen state it is unclear if the 1719 * device will actually send us a COMINIT, since we aren't 1720 * sending any COMRESET's 1721 */ 1722 /* NOP */ 1723 } 1724 1725 /* 1726 * We can't loop on the X bit, a continuous COMINIT received will make 1727 * it loop forever. Just assume one event has built up and clear X 1728 * so the task file descriptor can update. 1729 */ 1730 void 1731 ahci_flush_tfd(struct ahci_port *ap) 1732 { 1733 u_int32_t r; 1734 1735 r = ahci_pread(ap, AHCI_PREG_SERR); 1736 if (r & AHCI_PREG_SERR_DIAG_X) 1737 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1738 } 1739 1740 /* 1741 * Figure out what type of device is connected to the port, ATAPI or 1742 * DISK. 1743 */ 1744 int 1745 ahci_port_signature_detect(struct ahci_port *ap, struct ata_port *at) 1746 { 1747 u_int32_t sig; 1748 1749 sig = ahci_pread(ap, AHCI_PREG_SIG); 1750 if (bootverbose) 1751 kprintf("%s: sig %08x\n", ATANAME(ap, at), sig); 1752 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) { 1753 return(ATA_PORT_T_ATAPI); 1754 } else if ((sig & 0xffff0000) == 1755 (SATA_SIGNATURE_PORT_MULTIPLIER & 0xffff0000)) { 1756 return(ATA_PORT_T_PM); 1757 } else { 1758 return(ATA_PORT_T_DISK); 1759 } 1760 } 1761 1762 /* 1763 * Load the DMA descriptor table for a CCB's buffer. 1764 */ 1765 int 1766 ahci_load_prdt(struct ahci_ccb *ccb) 1767 { 1768 struct ahci_port *ap = ccb->ccb_port; 1769 struct ahci_softc *sc = ap->ap_sc; 1770 struct ata_xfer *xa = &ccb->ccb_xa; 1771 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1772 bus_dmamap_t dmap = ccb->ccb_dmamap; 1773 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1774 int error; 1775 1776 if (xa->datalen == 0) { 1777 ccb->ccb_cmd_hdr->prdtl = 0; 1778 return (0); 1779 } 1780 1781 error = bus_dmamap_load(sc->sc_tag_data, dmap, 1782 xa->data, xa->datalen, 1783 ahci_load_prdt_callback, 1784 &prdt, 1785 ((xa->flags & ATA_F_NOWAIT) ? 1786 BUS_DMA_NOWAIT : BUS_DMA_WAITOK)); 1787 if (error != 0) { 1788 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1789 return (1); 1790 } 1791 #if 0 1792 if (xa->flags & ATA_F_PIO) 1793 prdt->flags |= htole32(AHCI_PRDT_FLAG_INTR); 1794 #endif 1795 1796 cmd_slot->prdtl = htole16(prdt - ccb->ccb_cmd_table->prdt + 1); 1797 1798 if (xa->flags & ATA_F_READ) 1799 bus_dmamap_sync(sc->sc_tag_data, dmap, BUS_DMASYNC_PREREAD); 1800 if (xa->flags & ATA_F_WRITE) 1801 bus_dmamap_sync(sc->sc_tag_data, dmap, BUS_DMASYNC_PREWRITE); 1802 1803 return (0); 1804 } 1805 1806 /* 1807 * Callback from BUSDMA system to load the segment list. The passed segment 1808 * list is a temporary structure. 1809 */ 1810 static 1811 void 1812 ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, int nsegs, 1813 int error) 1814 { 1815 struct ahci_prdt *prd = *(void **)info; 1816 u_int64_t addr; 1817 1818 KKASSERT(nsegs <= AHCI_MAX_PRDT); 1819 1820 while (nsegs) { 1821 addr = segs->ds_addr; 1822 prd->dba_hi = htole32((u_int32_t)(addr >> 32)); 1823 prd->dba_lo = htole32((u_int32_t)addr); 1824 prd->flags = htole32(segs->ds_len - 1); 1825 --nsegs; 1826 if (nsegs) 1827 ++prd; 1828 ++segs; 1829 } 1830 *(void **)info = prd; /* return last valid segment */ 1831 } 1832 1833 void 1834 ahci_unload_prdt(struct ahci_ccb *ccb) 1835 { 1836 struct ahci_port *ap = ccb->ccb_port; 1837 struct ahci_softc *sc = ap->ap_sc; 1838 struct ata_xfer *xa = &ccb->ccb_xa; 1839 bus_dmamap_t dmap = ccb->ccb_dmamap; 1840 1841 if (xa->datalen != 0) { 1842 if (xa->flags & ATA_F_READ) { 1843 bus_dmamap_sync(sc->sc_tag_data, dmap, 1844 BUS_DMASYNC_POSTREAD); 1845 } 1846 if (xa->flags & ATA_F_WRITE) { 1847 bus_dmamap_sync(sc->sc_tag_data, dmap, 1848 BUS_DMASYNC_POSTWRITE); 1849 } 1850 bus_dmamap_unload(sc->sc_tag_data, dmap); 1851 1852 /* 1853 * prdbc is only updated by hardware for non-NCQ commands. 1854 */ 1855 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1856 xa->resid = 0; 1857 } else { 1858 if (ccb->ccb_cmd_hdr->prdbc == 0 && 1859 ccb->ccb_xa.state == ATA_S_COMPLETE) { 1860 kprintf("%s: WARNING! Unload prdbc resid " 1861 "was zero! tag=%d\n", 1862 ATANAME(ap, xa->at), ccb->ccb_slot); 1863 } 1864 xa->resid = xa->datalen - 1865 le32toh(ccb->ccb_cmd_hdr->prdbc); 1866 } 1867 } 1868 } 1869 1870 /* 1871 * Start a command and poll for completion. 1872 * 1873 * timeout is in ms and only counts once the command gets on-chip. 1874 * 1875 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine 1876 * that no error occured. 1877 * 1878 * NOTE: If the caller specifies a NULL timeout function the caller is 1879 * responsible for clearing hardware state on failure, but we will 1880 * deal with removing the ccb from any pending queue. 1881 * 1882 * NOTE: NCQ should never be used with this function. 1883 * 1884 * NOTE: If the port is in a failed state and stopped we do not try 1885 * to activate the ccb. 1886 */ 1887 int 1888 ahci_poll(struct ahci_ccb *ccb, int timeout, 1889 void (*timeout_fn)(struct ahci_ccb *)) 1890 { 1891 struct ahci_port *ap = ccb->ccb_port; 1892 1893 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) { 1894 ccb->ccb_xa.state = ATA_S_ERROR; 1895 return(ccb->ccb_xa.state); 1896 } 1897 crit_enter(); 1898 #if 0 1899 kprintf("%s: Start command %02x tag=%d\n", 1900 ATANAME(ccb->ccb_port, ccb->ccb_xa.at), 1901 ccb->ccb_xa.fis->command, ccb->ccb_slot); 1902 #endif 1903 ahci_start(ccb); 1904 1905 do { 1906 ahci_port_intr(ap, 1); 1907 switch(ccb->ccb_xa.state) { 1908 case ATA_S_ONCHIP: 1909 timeout -= ahci_os_softsleep(); 1910 break; 1911 case ATA_S_PENDING: 1912 ahci_os_softsleep(); 1913 ahci_check_active_timeouts(ap); 1914 break; 1915 default: 1916 crit_exit(); 1917 return (ccb->ccb_xa.state); 1918 } 1919 } while (timeout > 0); 1920 1921 if ((ccb->ccb_xa.flags & ATA_F_SILENT) == 0) { 1922 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n", 1923 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_slot, 1924 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD, 1925 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS, 1926 ahci_pread(ap, AHCI_PREG_SERR), AHCI_PFMT_SERR); 1927 } 1928 1929 timeout_fn(ccb); 1930 1931 crit_exit(); 1932 1933 return(ccb->ccb_xa.state); 1934 } 1935 1936 /* 1937 * When polling we have to check if the currently active CCB(s) 1938 * have timed out as the callout will be deadlocked while we 1939 * hold the port lock. 1940 */ 1941 void 1942 ahci_check_active_timeouts(struct ahci_port *ap) 1943 { 1944 struct ahci_ccb *ccb; 1945 u_int32_t mask; 1946 int tag; 1947 1948 mask = ap->ap_active | ap->ap_sactive; 1949 while (mask) { 1950 tag = ffs(mask) - 1; 1951 mask &= ~(1 << tag); 1952 ccb = &ap->ap_ccbs[tag]; 1953 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_EXPIRED) { 1954 ahci_ata_cmd_timeout(ccb); 1955 } 1956 } 1957 } 1958 1959 static 1960 __inline 1961 void 1962 ahci_start_timeout(struct ahci_ccb *ccb) 1963 { 1964 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_DESIRED) { 1965 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_RUNNING; 1966 callout_reset(&ccb->ccb_timeout, 1967 (ccb->ccb_xa.timeout * hz + 999) / 1000, 1968 ahci_ata_cmd_timeout_unserialized, ccb); 1969 } 1970 } 1971 1972 void 1973 ahci_start(struct ahci_ccb *ccb) 1974 { 1975 struct ahci_port *ap = ccb->ccb_port; 1976 struct ahci_softc *sc = ap->ap_sc; 1977 1978 KKASSERT(ccb->ccb_xa.state == ATA_S_PENDING); 1979 1980 /* Zero transferred byte count before transfer */ 1981 ccb->ccb_cmd_hdr->prdbc = 0; 1982 1983 /* Sync command list entry and corresponding command table entry */ 1984 bus_dmamap_sync(sc->sc_tag_cmdh, 1985 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1986 BUS_DMASYNC_PREWRITE); 1987 bus_dmamap_sync(sc->sc_tag_cmdt, 1988 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1989 BUS_DMASYNC_PREWRITE); 1990 1991 /* Prepare RFIS area for write by controller */ 1992 bus_dmamap_sync(sc->sc_tag_rfis, 1993 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 1994 BUS_DMASYNC_PREREAD); 1995 1996 /* 1997 * There's no point trying to optimize this, it only shaves a few 1998 * nanoseconds so just queue the command and call our generic issue. 1999 */ 2000 ahci_issue_pending_commands(ap, ccb); 2001 } 2002 2003 /* 2004 * While holding the port lock acquire exclusive access to the port. 2005 * 2006 * This is used when running the state machine to initialize and identify 2007 * targets over a port multiplier. Setting exclusive access prevents 2008 * ahci_port_intr() from activating any requests sitting on the pending 2009 * queue. 2010 */ 2011 void 2012 ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at) 2013 { 2014 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) == 0); 2015 ap->ap_flags |= AP_F_EXCLUSIVE_ACCESS; 2016 while (ap->ap_active || ap->ap_sactive) { 2017 ahci_port_intr(ap, 1); 2018 ahci_os_softsleep(); 2019 } 2020 } 2021 2022 void 2023 ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at) 2024 { 2025 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) != 0); 2026 ap->ap_flags &= ~AP_F_EXCLUSIVE_ACCESS; 2027 ahci_issue_pending_commands(ap, NULL); 2028 } 2029 2030 #if 0 2031 2032 static void 2033 fubar(struct ahci_ccb *ccb) 2034 { 2035 struct ahci_port *ap = ccb->ccb_port; 2036 struct ahci_cmd_hdr *cmd; 2037 struct ahci_cmd_table *tab; 2038 struct ahci_prdt *prdt; 2039 int i; 2040 2041 kprintf("%s: ISSUE %02x\n", 2042 ATANAME(ap, ccb->ccb_xa.at), 2043 ccb->ccb_xa.fis->command); 2044 cmd = ccb->ccb_cmd_hdr; 2045 tab = ccb->ccb_cmd_table; 2046 prdt = ccb->ccb_cmd_table->prdt; 2047 kprintf("cmd flags=%04x prdtl=%d prdbc=%d ctba=%08x%08x\n", 2048 cmd->flags, cmd->prdtl, cmd->prdbc, 2049 cmd->ctba_hi, cmd->ctba_lo); 2050 for (i = 0; i < cmd->prdtl; ++i) { 2051 kprintf("\t%d dba=%08x%08x res=%08x flags=%08x\n", 2052 i, prdt->dba_hi, prdt->dba_lo, prdt->reserved, 2053 prdt->flags); 2054 } 2055 kprintf("tab\n"); 2056 } 2057 2058 #endif 2059 2060 /* 2061 * If ccb is not NULL enqueue and/or issue it. 2062 * 2063 * If ccb is NULL issue whatever we can from the queue. However, nothing 2064 * new is issued if the exclusive access flag is set or expired ccb's are 2065 * present. 2066 * 2067 * If existing commands are still active (ap_active/ap_sactive) we can only 2068 * issue matching new commands. 2069 */ 2070 void 2071 ahci_issue_pending_commands(struct ahci_port *ap, struct ahci_ccb *ccb) 2072 { 2073 u_int32_t mask; 2074 int limit; 2075 2076 /* 2077 * Enqueue the ccb. 2078 * 2079 * If just running the queue and in exclusive access mode we 2080 * just return. Also in this case if there are any expired ccb's 2081 * we want to clear the queue so the port can be safely stopped. 2082 */ 2083 if (ccb) { 2084 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 2085 } else if ((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) || ap->ap_expired) { 2086 return; 2087 } 2088 2089 /* 2090 * Pull the next ccb off the queue and run it if possible. 2091 */ 2092 if ((ccb = TAILQ_FIRST(&ap->ap_ccb_pending)) == NULL) 2093 return; 2094 2095 /* 2096 * Handle exclusivity requirements. 2097 * 2098 * ATA_F_EXCLUSIVE is used when we want to be the only command 2099 * running. 2100 * 2101 * ATA_F_AUTOSENSE is used when we want the D2H rfis loaded 2102 * back into the ccb on a normal (non-errored) command completion. 2103 * For example, for PM requests to target 15. Because the AHCI 2104 * spec does not stop the command processor and has only one rfis 2105 * area (for non-FBSS anyway), AUTOSENSE currently implies EXCLUSIVE. 2106 * Otherwise multiple completions can destroy the rfis data before 2107 * we have a chance to copy it. 2108 */ 2109 if (ap->ap_active & ~ap->ap_expired) { 2110 /* 2111 * There may be multiple ccb's already running, 2112 * if any are running and ap_run_flags sets 2113 * one of these flags then we know only one is 2114 * running. 2115 * 2116 * XXX Current AUTOSENSE code forces exclusivity 2117 * to simplify the code. 2118 */ 2119 if (ap->ap_run_flags & 2120 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) { 2121 return; 2122 } 2123 2124 if (ccb->ccb_xa.flags & 2125 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) { 2126 return; 2127 } 2128 } 2129 2130 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 2131 /* 2132 * The next command is a NCQ command and can be issued as 2133 * long as currently active commands are not standard. 2134 */ 2135 if (ap->ap_active) { 2136 KKASSERT(ap->ap_active_cnt > 0); 2137 return; 2138 } 2139 KKASSERT(ap->ap_active_cnt == 0); 2140 2141 mask = 0; 2142 do { 2143 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2144 mask |= 1 << ccb->ccb_slot; 2145 ccb->ccb_xa.state = ATA_S_ONCHIP; 2146 ahci_start_timeout(ccb); 2147 ap->ap_run_flags = ccb->ccb_xa.flags; 2148 ccb = TAILQ_FIRST(&ap->ap_ccb_pending); 2149 } while (ccb && (ccb->ccb_xa.flags & ATA_F_NCQ) && 2150 (ap->ap_run_flags & 2151 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) == 0); 2152 2153 ap->ap_sactive |= mask; 2154 ahci_pwrite(ap, AHCI_PREG_SACT, mask); 2155 ahci_pwrite(ap, AHCI_PREG_CI, mask); 2156 } else { 2157 /* 2158 * The next command is a standard command and can be issued 2159 * as long as currently active commands are not NCQ. 2160 * 2161 * We limit ourself to 1 command if we have a port multiplier, 2162 * (at least without FBSS support), otherwise timeouts on 2163 * one port can race completions on other ports (see 2164 * ahci_ata_cmd_timeout() for more information). 2165 * 2166 * If not on a port multiplier generally allow up to 4 2167 * standard commands to be enqueued. Remember that the 2168 * command processor will still process them sequentially. 2169 */ 2170 if (ap->ap_sactive) 2171 return; 2172 if (ap->ap_type == ATA_PORT_T_PM) 2173 limit = 1; 2174 else if (ap->ap_sc->sc_ncmds > 4) 2175 limit = 4; 2176 else 2177 limit = 2; 2178 2179 while (ap->ap_active_cnt < limit && ccb && 2180 (ccb->ccb_xa.flags & ATA_F_NCQ) == 0) { 2181 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2182 #if 0 2183 fubar(ccb); 2184 #endif 2185 ap->ap_active |= 1 << ccb->ccb_slot; 2186 ap->ap_active_cnt++; 2187 ap->ap_run_flags = ccb->ccb_xa.flags; 2188 ccb->ccb_xa.state = ATA_S_ONCHIP; 2189 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 2190 ahci_start_timeout(ccb); 2191 if ((ap->ap_run_flags & 2192 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) == 0) { 2193 break; 2194 } 2195 ccb = TAILQ_FIRST(&ap->ap_ccb_pending); 2196 if (ccb && (ccb->ccb_xa.flags & 2197 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE))) { 2198 break; 2199 } 2200 } 2201 } 2202 } 2203 2204 void 2205 ahci_intr(void *arg) 2206 { 2207 struct ahci_softc *sc = arg; 2208 struct ahci_port *ap; 2209 u_int32_t is; 2210 u_int32_t ack; 2211 int port; 2212 2213 /* 2214 * Check if the master enable is up, and whether any interrupts are 2215 * pending. 2216 */ 2217 if ((sc->sc_flags & AHCI_F_INT_GOOD) == 0) 2218 return; 2219 is = ahci_read(sc, AHCI_REG_IS); 2220 if (is == 0 || is == 0xffffffff) { 2221 return; 2222 } 2223 is &= sc->sc_portmask; 2224 2225 #ifdef AHCI_COALESCE 2226 /* Check coalescing interrupt first */ 2227 if (is & sc->sc_ccc_mask) { 2228 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 2229 DEVNAME(sc)); 2230 is &= ~sc->sc_ccc_mask; 2231 is |= sc->sc_ccc_ports_cur; 2232 } 2233 #endif 2234 2235 /* 2236 * Process interrupts for each port in a non-blocking fashion. 2237 * 2238 * The global IS bit is forced on if any unmasked port interrupts 2239 * are pending, even if we clear. 2240 */ 2241 for (ack = 0; is; is &= ~(1 << port)) { 2242 port = ffs(is) - 1; 2243 ack |= 1 << port; 2244 2245 ap = sc->sc_ports[port]; 2246 if (ap == NULL) 2247 continue; 2248 2249 if (ahci_os_lock_port_nb(ap) == 0) { 2250 ahci_port_intr(ap, 0); 2251 ahci_os_unlock_port(ap); 2252 } else { 2253 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2254 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2255 } 2256 } 2257 ahci_write(sc, AHCI_REG_IS, ack); 2258 } 2259 2260 /* 2261 * Core called from helper thread. 2262 */ 2263 void 2264 ahci_port_thread_core(struct ahci_port *ap, int mask) 2265 { 2266 /* 2267 * Process any expired timedouts. 2268 */ 2269 ahci_os_lock_port(ap); 2270 if (mask & AP_SIGF_TIMEOUT) { 2271 ahci_check_active_timeouts(ap); 2272 } 2273 2274 /* 2275 * Process port interrupts which require a higher level of 2276 * intervention. 2277 */ 2278 if (mask & AP_SIGF_PORTINT) { 2279 ahci_port_intr(ap, 1); 2280 ahci_port_interrupt_enable(ap); 2281 ahci_os_unlock_port(ap); 2282 } else if (ap->ap_probe != ATA_PROBE_FAILED) { 2283 ahci_port_intr(ap, 1); 2284 ahci_port_interrupt_enable(ap); 2285 ahci_os_unlock_port(ap); 2286 } else { 2287 ahci_os_unlock_port(ap); 2288 } 2289 } 2290 2291 /* 2292 * Core per-port interrupt handler. 2293 * 2294 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only 2295 * deal with normal command completions which do not require blocking. 2296 */ 2297 void 2298 ahci_port_intr(struct ahci_port *ap, int blockable) 2299 { 2300 struct ahci_softc *sc = ap->ap_sc; 2301 u_int32_t is, ci_saved, ci_masked; 2302 int slot; 2303 int stopped = 0; 2304 struct ahci_ccb *ccb = NULL; 2305 struct ata_port *ccb_at = NULL; 2306 volatile u_int32_t *active; 2307 const u_int32_t blockable_mask = AHCI_PREG_IS_TFES | 2308 AHCI_PREG_IS_IFS | 2309 AHCI_PREG_IS_PCS | 2310 AHCI_PREG_IS_PRCS | 2311 AHCI_PREG_IS_HBFS | 2312 AHCI_PREG_IS_OFS | 2313 AHCI_PREG_IS_UFS; 2314 2315 enum { NEED_NOTHING, NEED_REINIT, NEED_RESTART, 2316 NEED_HOTPLUG_INSERT, NEED_HOTPLUG_REMOVE } need = NEED_NOTHING; 2317 2318 /* 2319 * All basic command completions are always processed. 2320 */ 2321 is = ahci_pread(ap, AHCI_PREG_IS); 2322 if (is & AHCI_PREG_IS_DPS) 2323 ahci_pwrite(ap, AHCI_PREG_IS, is & AHCI_PREG_IS_DPS); 2324 2325 /* 2326 * If we can't block then we can't handle these here. Disable 2327 * the interrupts in question so we don't live-lock, the helper 2328 * thread will re-enable them. 2329 * 2330 * If the port is in a completely failed state we do not want 2331 * to drop through to failed-command-processing if blockable is 0, 2332 * just let the thread deal with it all. 2333 * 2334 * Otherwise we fall through and still handle DHRS and any commands 2335 * which completed normally. Even if we are errored we haven't 2336 * stopped the port yet so CI/SACT are still good. 2337 */ 2338 if (blockable == 0) { 2339 if (ap->ap_state == AP_S_FATAL_ERROR) { 2340 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2341 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2342 return; 2343 } 2344 if (is & blockable_mask) { 2345 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2346 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2347 return; 2348 } 2349 } 2350 2351 /* 2352 * Either NCQ or non-NCQ commands will be active, never both. 2353 */ 2354 if (ap->ap_sactive) { 2355 KKASSERT(ap->ap_active == 0); 2356 KKASSERT(ap->ap_active_cnt == 0); 2357 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2358 active = &ap->ap_sactive; 2359 } else { 2360 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2361 active = &ap->ap_active; 2362 } 2363 KKASSERT(!(ap->ap_sactive && ap->ap_active)); 2364 #if 0 2365 kprintf("CHECK act=%08x/%08x sact=%08x/%08x\n", 2366 ap->ap_active, ahci_pread(ap, AHCI_PREG_CI), 2367 ap->ap_sactive, ahci_pread(ap, AHCI_PREG_SACT)); 2368 #endif 2369 2370 /* 2371 * Ignore AHCI_PREG_IS_PRCS when link power management is on 2372 */ 2373 if (ap->link_pwr_mgmt != AHCI_LINK_PWR_MGMT_NONE) { 2374 is &= ~AHCI_PREG_IS_PRCS; 2375 ahci_pwrite(ap, AHCI_PREG_SERR, 2376 AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_W); 2377 } 2378 2379 /* 2380 * Command failed (blockable). 2381 * 2382 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. 2383 * 2384 * This stops command processing. 2385 */ 2386 if (is & AHCI_PREG_IS_TFES) { 2387 u_int32_t tfd, serr; 2388 int err_slot; 2389 2390 process_error: 2391 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2392 serr = ahci_pread(ap, AHCI_PREG_SERR); 2393 2394 /* 2395 * Load the error slot and restart command processing. 2396 * CLO if we need to. The error slot may not be valid. 2397 * MUST BE DONE BEFORE CLEARING ST! 2398 * 2399 * Cycle ST. 2400 * 2401 * It is unclear but we may have to clear SERR to reenable 2402 * error processing. 2403 */ 2404 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2405 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2406 AHCI_PREG_IS_PSS | 2407 AHCI_PREG_IS_DHRS | 2408 AHCI_PREG_IS_SDBS); 2409 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_PSS | 2410 AHCI_PREG_IS_DHRS | AHCI_PREG_IS_SDBS); 2411 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2412 ahci_port_stop(ap, 0); 2413 ahci_os_hardsleep(10); 2414 if (tfd & (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2415 kprintf("%s: Issuing CLO\n", PORTNAME(ap)); 2416 ahci_port_clo(ap); 2417 } 2418 2419 /* 2420 * We are now stopped and need a restart. If we have to 2421 * process a NCQ error we will temporarily start and then 2422 * stop the port again, so this condition holds. 2423 */ 2424 stopped = 1; 2425 need = NEED_RESTART; 2426 2427 /* 2428 * ATAPI errors are fairly common from probing, just 2429 * report disk errors or if bootverbose is on. 2430 */ 2431 if (bootverbose || ap->ap_type != ATA_PORT_T_ATAPI) { 2432 kprintf("%s: TFES slot %d ci_saved = %08x\n", 2433 PORTNAME(ap), err_slot, ci_saved); 2434 } 2435 2436 /* 2437 * If we got an error on an error CCB just complete it 2438 * with an error. ci_saved has the mask to restart 2439 * (the err_ccb will be removed from it by finish_error). 2440 */ 2441 if (ap->ap_flags & AP_F_ERR_CCB_RESERVED) { 2442 err_slot = ap->ap_err_ccb->ccb_slot; 2443 goto finish_error; 2444 } 2445 2446 /* 2447 * If NCQ commands were active get the error slot from 2448 * the log page. NCQ is not supported for PM's so this 2449 * is a direct-attached target. 2450 * 2451 * Otherwise if no commands were active we have a problem. 2452 * 2453 * Otherwise if the error slot is bad we have a problem. 2454 * 2455 * Otherwise process the error for the slot. 2456 */ 2457 if (ap->ap_sactive) { 2458 ahci_port_start(ap); 2459 err_slot = ahci_port_read_ncq_error(ap, 0); 2460 ahci_port_stop(ap, 0); 2461 } else if (ap->ap_active == 0) { 2462 kprintf("%s: TFES with no commands pending\n", 2463 PORTNAME(ap)); 2464 err_slot = -1; 2465 } else if (err_slot < 0 || err_slot >= ap->ap_sc->sc_ncmds) { 2466 kprintf("%s: bad error slot %d\n", 2467 PORTNAME(ap), err_slot); 2468 err_slot = -1; 2469 } else { 2470 ccb = &ap->ap_ccbs[err_slot]; 2471 2472 /* 2473 * Validate the errored ccb. Note that ccb_at can 2474 * be NULL for direct-attached ccb's. 2475 * 2476 * Copy received taskfile data from the RFIS. 2477 */ 2478 if (ccb->ccb_xa.state == ATA_S_ONCHIP) { 2479 ccb_at = ccb->ccb_xa.at; 2480 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2481 sizeof(struct ata_fis_d2h)); 2482 if (bootverbose) { 2483 kprintf("%s: Copying rfis slot %d\n", 2484 ATANAME(ap, ccb_at), err_slot); 2485 } 2486 } else { 2487 kprintf("%s: Cannot copy rfis, CCB slot " 2488 "%d is not on-chip (state=%d)\n", 2489 ATANAME(ap, ccb->ccb_xa.at), 2490 err_slot, ccb->ccb_xa.state); 2491 err_slot = -1; 2492 } 2493 } 2494 2495 /* 2496 * If we could not determine the errored slot then 2497 * reset the port. 2498 */ 2499 if (err_slot < 0) { 2500 kprintf("%s: TFES: Unable to determine errored slot\n", 2501 PORTNAME(ap)); 2502 if (ap->ap_flags & AP_F_IN_RESET) 2503 goto fatal; 2504 goto failall; 2505 } 2506 2507 /* 2508 * Finish error on slot. We will restart ci_saved 2509 * commands except the errored slot which we generate 2510 * a failure for. 2511 */ 2512 finish_error: 2513 ccb = &ap->ap_ccbs[err_slot]; 2514 ci_saved &= ~(1 << err_slot); 2515 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2516 ccb->ccb_xa.state = ATA_S_ERROR; 2517 } else if (is & AHCI_PREG_IS_DHRS) { 2518 /* 2519 * Command posted D2H register FIS to the rfis (non-blocking). 2520 * 2521 * A normal completion with an error may set DHRS instead 2522 * of TFES. The CCS bits are only valid if ERR was set. 2523 * If ERR is set command processing was probably stopped. 2524 * 2525 * If ERR was not set we can only copy-back data for 2526 * exclusive-mode commands because otherwise we won't know 2527 * which tag the rfis belonged to. 2528 * 2529 * err_slot must be read from the CCS before any other port 2530 * action, such as stopping the port. 2531 * 2532 * WARNING! This is not well documented in the AHCI spec. 2533 * It can be found in the state machine tables 2534 * but not in the explanations. 2535 */ 2536 u_int32_t tfd; 2537 u_int32_t cmd; 2538 int err_slot; 2539 2540 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2541 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2542 2543 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2544 (cmd & AHCI_PREG_CMD_CR) == 0) { 2545 err_slot = AHCI_PREG_CMD_CCS( 2546 ahci_pread(ap, AHCI_PREG_CMD)); 2547 ccb = &ap->ap_ccbs[err_slot]; 2548 kprintf("%s: DHRS tfd=%b err_slot=%d cmd=%02x\n", 2549 PORTNAME(ap), 2550 tfd, AHCI_PFMT_TFD_STS, 2551 err_slot, ccb->ccb_xa.fis->command); 2552 goto process_error; 2553 } 2554 /* 2555 * NO ELSE... copy back is in the normal command completion 2556 * code and only if no error occured and ATA_F_AUTOSENSE 2557 * was set. 2558 */ 2559 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2560 } 2561 2562 /* 2563 * Device notification to us (non-blocking) 2564 * 2565 * NOTE! On some parts notification bits can cause an IPMS 2566 * interrupt instead of a SDBS interrupt. 2567 * 2568 * NOTE! On some parts (e.g. VBOX, probably intel ICHx), 2569 * SDBS notifies us of the completion of a NCQ command 2570 * and DBS does not. 2571 */ 2572 if (is & (AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS)) { 2573 u_int32_t data; 2574 2575 ahci_pwrite(ap, AHCI_PREG_IS, 2576 AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS); 2577 if (sc->sc_cap & AHCI_REG_CAP_SSNTF) { 2578 data = ahci_pread(ap, AHCI_PREG_SNTF); 2579 if (data) { 2580 ahci_pwrite(ap, AHCI_PREG_IS, 2581 AHCI_PREG_IS_SDBS); 2582 kprintf("%s: NOTIFY %08x\n", 2583 PORTNAME(ap), data); 2584 ahci_pwrite(ap, AHCI_PREG_SERR, 2585 AHCI_PREG_SERR_DIAG_N); 2586 ahci_pwrite(ap, AHCI_PREG_SNTF, data); 2587 ahci_cam_changed(ap, NULL, -1); 2588 } 2589 } 2590 is &= ~(AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS); 2591 } 2592 2593 /* 2594 * Spurious IFS errors (blockable) - when AP_F_IGNORE_IFS is set. 2595 * 2596 * Spurious IFS errors can occur while we are doing a reset 2597 * sequence through a PM, probably due to an unexpected FIS 2598 * being received during the PM target reset sequence. Chipsets 2599 * are supposed to mask these events but some do not. 2600 * 2601 * Try to recover from the condition. 2602 */ 2603 if ((is & AHCI_PREG_IS_IFS) && (ap->ap_flags & AP_F_IGNORE_IFS)) { 2604 u_int32_t serr = ahci_pread(ap, AHCI_PREG_SERR); 2605 if ((ap->ap_flags & AP_F_IFS_IGNORED) == 0) { 2606 kprintf("%s: IFS during PM probe (ignored) " 2607 "IS=%b, SERR=%b\n", 2608 PORTNAME(ap), 2609 is, AHCI_PFMT_IS, 2610 serr, AHCI_PFMT_SERR); 2611 ap->ap_flags |= AP_F_IFS_IGNORED; 2612 } 2613 2614 /* 2615 * Try to clear the error condition. The IFS error killed 2616 * the port so stop it so we can restart it. 2617 */ 2618 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2619 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2620 is &= ~AHCI_PREG_IS_IFS; 2621 need = NEED_RESTART; 2622 goto failall; 2623 } 2624 2625 /* 2626 * Port change (hot-plug) (blockable). 2627 * 2628 * A PRCS interrupt can occur: 2629 * (1) On hot-unplug / normal-unplug (phy lost) 2630 * (2) Sometimes on hot-plug too. 2631 * 2632 * A PCS interrupt can occur in a number of situations: 2633 * (1) On hot-plug once communication is established 2634 * (2) On hot-unplug sometimes. 2635 * (3) For chipsets with badly written firmware it can occur 2636 * during INIT/RESET sequences due to the device reset. 2637 * (4) For chipsets with badly written firmware it can occur 2638 * when it thinks an unsolicited COMRESET is received 2639 * during a INIT/RESET sequence, even though we actually 2640 * did request it. 2641 * 2642 * XXX We can then check the CPS (Cold Presence State) bit, if 2643 * supported, to determine if a device is plugged in or not and do 2644 * the right thing. 2645 * 2646 * PCS interrupts are cleared by clearing DIAG_X. If this occurs 2647 * command processing is automatically stopped (CR goes inactive) 2648 * and the port must be stopped and restarted. 2649 * 2650 * WARNING: AMD parts (e.g. 880G chipset, probably others) can 2651 * generate PCS on initialization even when device is 2652 * already connected up. It is unclear why this happens. 2653 * Depending on the state of the device detect this can 2654 * cause us to go into harsh reinit or hot-plug insertion 2655 * mode. 2656 * 2657 * WARNING: PCS errors can be repetitive (e.g. unsolicited COMRESET 2658 * continues to flow in from the device), we must clear the 2659 * interrupt in all cases and enforce a delay to prevent 2660 * a livelock and give the port time to settle down. 2661 * Only print something if we aren't in INIT/HARD-RESET. 2662 */ 2663 if (is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)) { 2664 /* 2665 * Try to clear the error. Because of the repetitiveness 2666 * of this interrupt avoid any harsh action if the port is 2667 * already in the init or hard-reset probe state. 2668 */ 2669 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2670 /* (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X) */ 2671 ahci_pwrite(ap, AHCI_PREG_IS, 2672 is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)); 2673 2674 if (ap->ap_probe == ATA_PROBE_NEED_INIT || 2675 ap->ap_probe == ATA_PROBE_NEED_HARD_RESET) { 2676 is &= ~(AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS); 2677 need = NEED_NOTHING; 2678 ahci_os_sleep(1000); 2679 goto failall; 2680 } 2681 kprintf("%s: Transient Errors: %b (%d)\n", 2682 PORTNAME(ap), is, AHCI_PFMT_IS, ap->ap_probe); 2683 is &= ~(AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS); 2684 ahci_os_sleep(200); 2685 2686 /* 2687 * Stop the port and figure out what to do next. 2688 */ 2689 ahci_port_stop(ap, 0); 2690 stopped = 1; 2691 2692 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 2693 case AHCI_PREG_SSTS_DET_DEV: 2694 /* 2695 * Device detect 2696 */ 2697 if (ap->ap_probe == ATA_PROBE_FAILED) { 2698 need = NEED_HOTPLUG_INSERT; 2699 goto fatal; 2700 } 2701 need = NEED_RESTART; 2702 break; 2703 case AHCI_PREG_SSTS_DET_DEV_NE: 2704 /* 2705 * Device not communicating. AMD parts seem to 2706 * like to throw this error on initialization 2707 * for no reason that I can fathom. 2708 */ 2709 kprintf("%s: Device present but not communicating, " 2710 "attempting port restart\n", 2711 PORTNAME(ap)); 2712 need = NEED_REINIT; 2713 goto fatal; 2714 default: 2715 if (ap->ap_probe != ATA_PROBE_FAILED) { 2716 need = NEED_HOTPLUG_REMOVE; 2717 goto fatal; 2718 } 2719 need = NEED_RESTART; 2720 break; 2721 } 2722 } 2723 2724 /* 2725 * Check for remaining errors - they are fatal. (blockable) 2726 */ 2727 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2728 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2729 u_int32_t serr; 2730 2731 ahci_pwrite(ap, AHCI_PREG_IS, 2732 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2733 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2734 AHCI_PREG_IS_UFS)); 2735 serr = ahci_pread(ap, AHCI_PREG_SERR); 2736 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), " 2737 "disabling port.\n", 2738 PORTNAME(ap), 2739 is, AHCI_PFMT_IS, 2740 serr, AHCI_PFMT_SERR 2741 ); 2742 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2743 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2744 AHCI_PREG_IS_UFS); 2745 2746 /* 2747 * Fail all commands but then what? For now try to 2748 * reinitialize the port. 2749 */ 2750 need = NEED_REINIT; 2751 goto fatal; 2752 } 2753 2754 /* 2755 * Fail all outstanding commands if we know the port won't recover. 2756 * 2757 * We may have a ccb_at if the failed command is known and was 2758 * being sent to a device over a port multiplier (PM). In this 2759 * case if the port itself has not completely failed we fail just 2760 * the commands related to that target. 2761 * 2762 * ci_saved contains the mask of active commands as of when the 2763 * error occured, prior to any port stops. 2764 */ 2765 if (ap->ap_state == AP_S_FATAL_ERROR) { 2766 fatal: 2767 ap->ap_state = AP_S_FATAL_ERROR; 2768 failall: 2769 ahci_port_stop(ap, 0); 2770 stopped = 1; 2771 2772 /* 2773 * Error all the active slots not already errored. 2774 */ 2775 ci_masked = ci_saved & *active & ~ap->ap_expired; 2776 if (ci_masked) { 2777 kprintf("%s: Failing all commands: %08x\n", 2778 PORTNAME(ap), ci_masked); 2779 } 2780 2781 while (ci_masked) { 2782 slot = ffs(ci_masked) - 1; 2783 ccb = &ap->ap_ccbs[slot]; 2784 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2785 ap->ap_expired |= 1 << slot; 2786 ci_saved &= ~(1 << slot); 2787 ci_masked &= ~(1 << slot); 2788 } 2789 2790 /* 2791 * Clear bits in ci_saved (cause completions to be run) 2792 * for all slots which are not active. 2793 */ 2794 ci_saved &= ~*active; 2795 2796 /* 2797 * Don't restart the port if our problems were deemed fatal. 2798 * 2799 * Also acknowlege all fatal interrupt sources to prevent 2800 * a livelock. 2801 */ 2802 if (ap->ap_state == AP_S_FATAL_ERROR) { 2803 if (need == NEED_RESTART) 2804 need = NEED_NOTHING; 2805 ahci_pwrite(ap, AHCI_PREG_IS, 2806 AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2807 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2808 AHCI_PREG_IS_UFS); 2809 } 2810 } 2811 2812 /* 2813 * If we are stopped the AHCI chipset is supposed to have cleared 2814 * CI and SACT. Did it? If it didn't we try very hard to clear 2815 * the fields otherwise we may end up completing CCBs which are 2816 * actually still active. 2817 * 2818 * IFS errors on (at least) AMD chipsets create this confusion. 2819 */ 2820 if (stopped) { 2821 u_int32_t mask; 2822 if ((mask = ahci_pactive(ap)) != 0) { 2823 kprintf("%s: chipset failed to clear " 2824 "active cmds %08x\n", 2825 PORTNAME(ap), mask); 2826 ahci_port_start(ap); 2827 ahci_port_stop(ap, 0); 2828 if ((mask = ahci_pactive(ap)) != 0) { 2829 kprintf("%s: unable to prod the chip into " 2830 "clearing active cmds %08x\n", 2831 PORTNAME(ap), mask); 2832 /* what do we do now? */ 2833 } 2834 } 2835 } 2836 2837 /* 2838 * CCB completion (non blocking). 2839 * 2840 * CCB completion is detected by noticing its slot's bit in CI has 2841 * changed to zero some time after we activated it. 2842 * If we are polling, we may only be interested in particular slot(s). 2843 * 2844 * Any active bits not saved are completed within the restrictions 2845 * imposed by the caller. 2846 */ 2847 ci_masked = ~ci_saved & *active; 2848 while (ci_masked) { 2849 slot = ffs(ci_masked) - 1; 2850 ccb = &ap->ap_ccbs[slot]; 2851 ci_masked &= ~(1 << slot); 2852 2853 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2854 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2855 " (error)" : ""); 2856 2857 bus_dmamap_sync(sc->sc_tag_cmdh, 2858 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2859 BUS_DMASYNC_POSTWRITE); 2860 2861 bus_dmamap_sync(sc->sc_tag_cmdt, 2862 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2863 BUS_DMASYNC_POSTWRITE); 2864 2865 bus_dmamap_sync(sc->sc_tag_rfis, 2866 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 2867 BUS_DMASYNC_POSTREAD); 2868 2869 *active &= ~(1 << ccb->ccb_slot); 2870 if (active == &ap->ap_active) { 2871 KKASSERT(ap->ap_active_cnt > 0); 2872 --ap->ap_active_cnt; 2873 } 2874 2875 /* 2876 * Complete the ccb. If the ccb was marked expired it 2877 * was probably already removed from the command processor, 2878 * so don't take the clear ci_saved bit as meaning the 2879 * command actually succeeded, it didn't. 2880 */ 2881 if (ap->ap_expired & (1 << ccb->ccb_slot)) { 2882 ap->ap_expired &= ~(1 << ccb->ccb_slot); 2883 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2884 ccb->ccb_done(ccb); 2885 ccb->ccb_xa.complete(&ccb->ccb_xa); 2886 } else { 2887 if (ccb->ccb_xa.state == ATA_S_ONCHIP) { 2888 ccb->ccb_xa.state = ATA_S_COMPLETE; 2889 if (ccb->ccb_xa.flags & ATA_F_AUTOSENSE) { 2890 memcpy(&ccb->ccb_xa.rfis, 2891 ap->ap_rfis->rfis, 2892 sizeof(struct ata_fis_d2h)); 2893 if (ccb->ccb_xa.state == ATA_S_TIMEOUT) 2894 ccb->ccb_xa.state = ATA_S_ERROR; 2895 } 2896 } 2897 ccb->ccb_done(ccb); 2898 } 2899 } 2900 2901 /* 2902 * Cleanup. Will not be set if non-blocking. 2903 */ 2904 switch(need) { 2905 case NEED_NOTHING: 2906 /* 2907 * If operating normally and not stopped the interrupt was 2908 * probably just a normal completion and we may be able to 2909 * issue more commands. 2910 */ 2911 if (stopped == 0 && ap->ap_state != AP_S_FATAL_ERROR) 2912 ahci_issue_pending_commands(ap, NULL); 2913 break; 2914 case NEED_RESTART: 2915 /* 2916 * A recoverable error occured and we can restart outstanding 2917 * commands on the port. 2918 */ 2919 ci_saved &= ~ap->ap_expired; 2920 if (ci_saved) { 2921 kprintf("%s: Restart %08x\n", PORTNAME(ap), ci_saved); 2922 ahci_issue_saved_commands(ap, ci_saved); 2923 } 2924 2925 /* 2926 * Potentially issue new commands if not in a failed 2927 * state. 2928 */ 2929 if (ap->ap_state != AP_S_FATAL_ERROR) { 2930 ahci_port_start(ap); 2931 ahci_issue_pending_commands(ap, NULL); 2932 } 2933 break; 2934 case NEED_REINIT: 2935 /* 2936 * Something horrible happened to the port and we 2937 * need to reinitialize it. 2938 */ 2939 kprintf("%s: REINIT - Attempting to reinitialize the port " 2940 "after it had a horrible accident\n", 2941 PORTNAME(ap)); 2942 ap->ap_flags |= AP_F_IN_RESET; 2943 ap->ap_flags |= AP_F_HARSH_REINIT; 2944 ap->ap_probe = ATA_PROBE_NEED_INIT; 2945 ahci_cam_changed(ap, NULL, -1); 2946 break; 2947 case NEED_HOTPLUG_INSERT: 2948 /* 2949 * A hot-plug insertion event has occured and all 2950 * outstanding commands have already been revoked. 2951 * 2952 * Don't recurse if this occurs while we are 2953 * resetting the port. 2954 */ 2955 if ((ap->ap_flags & AP_F_IN_RESET) == 0) { 2956 kprintf("%s: HOTPLUG - Device inserted\n", 2957 PORTNAME(ap)); 2958 ap->ap_probe = ATA_PROBE_NEED_INIT; 2959 ahci_cam_changed(ap, NULL, -1); 2960 } 2961 break; 2962 case NEED_HOTPLUG_REMOVE: 2963 /* 2964 * A hot-plug removal event has occured and all 2965 * outstanding commands have already been revoked. 2966 * 2967 * Don't recurse if this occurs while we are 2968 * resetting the port. 2969 */ 2970 if ((ap->ap_flags & AP_F_IN_RESET) == 0) { 2971 kprintf("%s: HOTPLUG - Device removed\n", 2972 PORTNAME(ap)); 2973 ahci_port_hardstop(ap); 2974 /* ap_probe set to failed */ 2975 ahci_cam_changed(ap, NULL, -1); 2976 } 2977 break; 2978 default: 2979 break; 2980 } 2981 } 2982 2983 struct ahci_ccb * 2984 ahci_get_ccb(struct ahci_port *ap) 2985 { 2986 struct ahci_ccb *ccb; 2987 2988 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); 2989 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2990 if (ccb != NULL) { 2991 KKASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2992 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2993 ccb->ccb_xa.state = ATA_S_SETUP; 2994 ccb->ccb_xa.flags = 0; 2995 ccb->ccb_xa.at = NULL; 2996 } 2997 lockmgr(&ap->ap_ccb_lock, LK_RELEASE); 2998 2999 return (ccb); 3000 } 3001 3002 void 3003 ahci_put_ccb(struct ahci_ccb *ccb) 3004 { 3005 struct ahci_port *ap = ccb->ccb_port; 3006 3007 ccb->ccb_xa.state = ATA_S_PUT; 3008 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); 3009 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 3010 lockmgr(&ap->ap_ccb_lock, LK_RELEASE); 3011 } 3012 3013 struct ahci_ccb * 3014 ahci_get_err_ccb(struct ahci_port *ap) 3015 { 3016 struct ahci_ccb *err_ccb; 3017 u_int32_t sact; 3018 u_int32_t ci; 3019 3020 /* No commands may be active on the chip. */ 3021 3022 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) { 3023 sact = ahci_pread(ap, AHCI_PREG_SACT); 3024 if (sact != 0) { 3025 kprintf("%s: ahci_get_err_ccb but SACT %08x != 0?\n", 3026 PORTNAME(ap), sact); 3027 } 3028 } 3029 ci = ahci_pread(ap, AHCI_PREG_CI); 3030 if (ci) { 3031 kprintf("%s: ahci_get_err_ccb: ci not 0 (%08x)\n", 3032 ap->ap_name, ci); 3033 } 3034 KKASSERT(ci == 0); 3035 KKASSERT((ap->ap_flags & AP_F_ERR_CCB_RESERVED) == 0); 3036 ap->ap_flags |= AP_F_ERR_CCB_RESERVED; 3037 3038 /* Save outstanding command state. */ 3039 ap->ap_err_saved_active = ap->ap_active; 3040 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 3041 ap->ap_err_saved_sactive = ap->ap_sactive; 3042 3043 /* 3044 * Pretend we have no commands outstanding, so that completions won't 3045 * run prematurely. 3046 */ 3047 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 3048 3049 /* 3050 * Grab a CCB to use for error recovery. This should never fail, as 3051 * we ask atascsi to reserve one for us at init time. 3052 */ 3053 err_ccb = ap->ap_err_ccb; 3054 KKASSERT(err_ccb != NULL); 3055 err_ccb->ccb_xa.flags = 0; 3056 err_ccb->ccb_done = ahci_empty_done; 3057 3058 return err_ccb; 3059 } 3060 3061 void 3062 ahci_put_err_ccb(struct ahci_ccb *ccb) 3063 { 3064 struct ahci_port *ap = ccb->ccb_port; 3065 u_int32_t sact; 3066 u_int32_t ci; 3067 3068 KKASSERT((ap->ap_flags & AP_F_ERR_CCB_RESERVED) != 0); 3069 3070 /* 3071 * No commands may be active on the chip 3072 */ 3073 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) { 3074 sact = ahci_pread(ap, AHCI_PREG_SACT); 3075 if (sact) { 3076 panic("ahci_port_err_ccb(%d) but SACT %08x != 0\n", 3077 ccb->ccb_slot, sact); 3078 } 3079 } 3080 ci = ahci_pread(ap, AHCI_PREG_CI); 3081 if (ci) { 3082 panic("ahci_put_err_ccb(%d) but CI %08x != 0 " 3083 "(act=%08x sact=%08x)\n", 3084 ccb->ccb_slot, ci, 3085 ap->ap_active, ap->ap_sactive); 3086 } 3087 3088 KKASSERT(ccb == ap->ap_err_ccb); 3089 3090 /* Restore outstanding command state */ 3091 ap->ap_sactive = ap->ap_err_saved_sactive; 3092 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 3093 ap->ap_active = ap->ap_err_saved_active; 3094 3095 ap->ap_flags &= ~AP_F_ERR_CCB_RESERVED; 3096 } 3097 3098 /* 3099 * Read log page to get NCQ error. 3100 * 3101 * NOTE: NCQ not currently supported on port multipliers. XXX 3102 */ 3103 int 3104 ahci_port_read_ncq_error(struct ahci_port *ap, int target) 3105 { 3106 struct ata_log_page_10h *log; 3107 struct ahci_ccb *ccb; 3108 struct ahci_ccb *ccb2; 3109 struct ahci_cmd_hdr *cmd_slot; 3110 struct ata_fis_h2d *fis; 3111 int err_slot; 3112 3113 if (bootverbose) { 3114 kprintf("%s: READ LOG PAGE target %d\n", PORTNAME(ap), 3115 target); 3116 } 3117 3118 /* 3119 * Prep error CCB for READ LOG EXT, page 10h, 1 sector. 3120 * 3121 * Getting err_ccb clears active/sactive/active_cnt, putting 3122 * it back restores the fields. 3123 */ 3124 ccb = ahci_get_err_ccb(ap); 3125 ccb->ccb_xa.flags = ATA_F_READ | ATA_F_POLL; 3126 ccb->ccb_xa.data = ap->ap_err_scratch; 3127 ccb->ccb_xa.datalen = 512; 3128 ccb->ccb_xa.complete = ahci_dummy_done; 3129 ccb->ccb_xa.at = ap->ap_ata[target]; 3130 3131 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3132 bzero(fis, sizeof(*fis)); 3133 fis->type = ATA_FIS_TYPE_H2D; 3134 fis->flags = ATA_H2D_FLAGS_CMD | target; 3135 fis->command = ATA_C_READ_LOG_EXT; 3136 fis->lba_low = 0x10; /* queued error log page (10h) */ 3137 fis->sector_count = 1; /* number of sectors (1) */ 3138 fis->sector_count_exp = 0; 3139 fis->lba_mid = 0; /* starting offset */ 3140 fis->lba_mid_exp = 0; 3141 fis->device = 0; 3142 3143 cmd_slot = ccb->ccb_cmd_hdr; 3144 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 3145 3146 if (ahci_load_prdt(ccb) != 0) { 3147 err_slot = -1; 3148 goto err; 3149 } 3150 3151 ccb->ccb_xa.state = ATA_S_PENDING; 3152 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 3153 err_slot = -1; 3154 ahci_unload_prdt(ccb); 3155 goto err; 3156 } 3157 ahci_unload_prdt(ccb); 3158 3159 /* 3160 * Success, extract failed register set and tags from the scratch 3161 * space. 3162 */ 3163 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 3164 if (log->err_regs.type & ATA_LOG_10H_TYPE_NOTQUEUED) { 3165 /* Not queued bit was set - wasn't an NCQ error? */ 3166 kprintf("%s: read NCQ error page, but not an NCQ error?\n", 3167 PORTNAME(ap)); 3168 err_slot = -1; 3169 } else { 3170 /* Copy back the log record as a D2H register FIS. */ 3171 err_slot = log->err_regs.type & ATA_LOG_10H_TYPE_TAG_MASK; 3172 3173 ccb2 = &ap->ap_ccbs[err_slot]; 3174 if (ccb2->ccb_xa.state == ATA_S_ONCHIP) { 3175 kprintf("%s: read NCQ error page slot=%d\n", 3176 ATANAME(ap, ccb2->ccb_xa.at), 3177 err_slot); 3178 memcpy(&ccb2->ccb_xa.rfis, &log->err_regs, 3179 sizeof(struct ata_fis_d2h)); 3180 ccb2->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 3181 ccb2->ccb_xa.rfis.flags = 0; 3182 } else { 3183 kprintf("%s: read NCQ error page slot=%d, " 3184 "slot does not match any cmds\n", 3185 ATANAME(ccb2->ccb_port, ccb2->ccb_xa.at), 3186 err_slot); 3187 err_slot = -1; 3188 } 3189 } 3190 err: 3191 ahci_put_err_ccb(ccb); 3192 kprintf("%s: DONE log page target %d err_slot=%d\n", 3193 PORTNAME(ap), target, err_slot); 3194 return (err_slot); 3195 } 3196 3197 /* 3198 * Allocate memory for various structures DMAd by hardware. The maximum 3199 * number of segments for these tags is 1 so the DMA memory will have a 3200 * single physical base address. 3201 */ 3202 struct ahci_dmamem * 3203 ahci_dmamem_alloc(struct ahci_softc *sc, bus_dma_tag_t tag) 3204 { 3205 struct ahci_dmamem *adm; 3206 int error; 3207 3208 adm = kmalloc(sizeof(*adm), M_DEVBUF, M_INTWAIT | M_ZERO); 3209 3210 error = bus_dmamem_alloc(tag, (void **)&adm->adm_kva, 3211 BUS_DMA_ZERO, &adm->adm_map); 3212 if (error == 0) { 3213 adm->adm_tag = tag; 3214 error = bus_dmamap_load(tag, adm->adm_map, 3215 adm->adm_kva, 3216 bus_dma_tag_getmaxsize(tag), 3217 ahci_dmamem_saveseg, &adm->adm_busaddr, 3218 0); 3219 } 3220 if (error) { 3221 if (adm->adm_map) { 3222 bus_dmamap_destroy(tag, adm->adm_map); 3223 adm->adm_map = NULL; 3224 adm->adm_tag = NULL; 3225 adm->adm_kva = NULL; 3226 } 3227 kfree(adm, M_DEVBUF); 3228 adm = NULL; 3229 } 3230 return (adm); 3231 } 3232 3233 static 3234 void 3235 ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error) 3236 { 3237 KKASSERT(error == 0); 3238 KKASSERT(nsegs == 1); 3239 *(bus_addr_t *)info = segs->ds_addr; 3240 } 3241 3242 3243 void 3244 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 3245 { 3246 if (adm->adm_map) { 3247 bus_dmamap_unload(adm->adm_tag, adm->adm_map); 3248 bus_dmamap_destroy(adm->adm_tag, adm->adm_map); 3249 adm->adm_map = NULL; 3250 adm->adm_tag = NULL; 3251 adm->adm_kva = NULL; 3252 } 3253 kfree(adm, M_DEVBUF); 3254 } 3255 3256 u_int32_t 3257 ahci_read(struct ahci_softc *sc, bus_size_t r) 3258 { 3259 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 3260 BUS_SPACE_BARRIER_READ); 3261 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 3262 } 3263 3264 void 3265 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 3266 { 3267 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 3268 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 3269 BUS_SPACE_BARRIER_WRITE); 3270 } 3271 3272 u_int32_t 3273 ahci_pread(struct ahci_port *ap, bus_size_t r) 3274 { 3275 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 3276 BUS_SPACE_BARRIER_READ); 3277 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 3278 } 3279 3280 void 3281 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 3282 { 3283 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 3284 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 3285 BUS_SPACE_BARRIER_WRITE); 3286 } 3287 3288 /* 3289 * Wait up to (timeout) milliseconds for the masked port register to 3290 * match the target. 3291 * 3292 * Timeout is in milliseconds. 3293 */ 3294 int 3295 ahci_pwait_eq(struct ahci_port *ap, int timeout, 3296 bus_size_t r, u_int32_t mask, u_int32_t target) 3297 { 3298 int t; 3299 3300 /* 3301 * Loop hard up to 100uS 3302 */ 3303 for (t = 0; t < 100; ++t) { 3304 if ((ahci_pread(ap, r) & mask) == target) 3305 return (0); 3306 ahci_os_hardsleep(1); /* us */ 3307 } 3308 3309 do { 3310 timeout -= ahci_os_softsleep(); 3311 if ((ahci_pread(ap, r) & mask) == target) 3312 return (0); 3313 } while (timeout > 0); 3314 return (1); 3315 } 3316 3317 int 3318 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 3319 u_int32_t target) 3320 { 3321 int t; 3322 3323 /* 3324 * Loop hard up to 100uS 3325 */ 3326 for (t = 0; t < 100; ++t) { 3327 if ((ahci_read(sc, r) & mask) != target) 3328 return (0); 3329 ahci_os_hardsleep(1); /* us */ 3330 } 3331 3332 /* 3333 * And one millisecond the slow way 3334 */ 3335 t = 1000; 3336 do { 3337 t -= ahci_os_softsleep(); 3338 if ((ahci_read(sc, r) & mask) != target) 3339 return (0); 3340 } while (t > 0); 3341 3342 return (1); 3343 } 3344 3345 3346 /* 3347 * Acquire an ata transfer. 3348 * 3349 * Pass a NULL at for direct-attached transfers, and a non-NULL at for 3350 * targets that go through the port multiplier. 3351 */ 3352 struct ata_xfer * 3353 ahci_ata_get_xfer(struct ahci_port *ap, struct ata_port *at) 3354 { 3355 struct ahci_ccb *ccb; 3356 3357 ccb = ahci_get_ccb(ap); 3358 if (ccb == NULL) { 3359 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 3360 PORTNAME(ap)); 3361 return (NULL); 3362 } 3363 3364 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 3365 PORTNAME(ap), ccb->ccb_slot); 3366 3367 bzero(ccb->ccb_xa.fis, sizeof(*ccb->ccb_xa.fis)); 3368 ccb->ccb_xa.at = at; 3369 ccb->ccb_xa.fis->type = ATA_FIS_TYPE_H2D; 3370 3371 return (&ccb->ccb_xa); 3372 } 3373 3374 void 3375 ahci_ata_put_xfer(struct ata_xfer *xa) 3376 { 3377 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 3378 3379 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 3380 3381 ahci_put_ccb(ccb); 3382 } 3383 3384 int 3385 ahci_ata_cmd(struct ata_xfer *xa) 3386 { 3387 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 3388 struct ahci_cmd_hdr *cmd_slot; 3389 3390 KKASSERT(xa->state == ATA_S_SETUP); 3391 3392 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 3393 goto failcmd; 3394 ccb->ccb_done = ahci_ata_cmd_done; 3395 3396 cmd_slot = ccb->ccb_cmd_hdr; 3397 cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */ 3398 if (ccb->ccb_xa.at) { 3399 cmd_slot->flags |= htole16(ccb->ccb_xa.at->at_target << 3400 AHCI_CMD_LIST_FLAG_PMP_SHIFT); 3401 } 3402 3403 if (xa->flags & ATA_F_WRITE) 3404 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); 3405 3406 if (xa->flags & ATA_F_PACKET) 3407 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A); 3408 3409 if (ahci_load_prdt(ccb) != 0) 3410 goto failcmd; 3411 3412 xa->state = ATA_S_PENDING; 3413 3414 if (xa->flags & ATA_F_POLL) 3415 return (ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout)); 3416 3417 crit_enter(); 3418 KKASSERT((xa->flags & ATA_F_TIMEOUT_EXPIRED) == 0); 3419 xa->flags |= ATA_F_TIMEOUT_DESIRED; 3420 ahci_start(ccb); 3421 crit_exit(); 3422 return (xa->state); 3423 3424 failcmd: 3425 crit_enter(); 3426 xa->state = ATA_S_ERROR; 3427 xa->complete(xa); 3428 crit_exit(); 3429 return (ATA_S_ERROR); 3430 } 3431 3432 void 3433 ahci_ata_cmd_done(struct ahci_ccb *ccb) 3434 { 3435 struct ata_xfer *xa = &ccb->ccb_xa; 3436 3437 /* 3438 * NOTE: callout does not lock port and may race us modifying 3439 * the flags, so make sure its stopped. 3440 */ 3441 if (xa->flags & ATA_F_TIMEOUT_RUNNING) { 3442 callout_stop(&ccb->ccb_timeout); 3443 xa->flags &= ~ATA_F_TIMEOUT_RUNNING; 3444 } 3445 xa->flags &= ~(ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_EXPIRED); 3446 3447 KKASSERT(xa->state != ATA_S_ONCHIP); 3448 ahci_unload_prdt(ccb); 3449 3450 if (xa->state != ATA_S_TIMEOUT) 3451 xa->complete(xa); 3452 } 3453 3454 /* 3455 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags 3456 * while the callout is runing. 3457 * 3458 * We can't safely get the port lock here or delay, we could block 3459 * the callout thread. 3460 */ 3461 static void 3462 ahci_ata_cmd_timeout_unserialized(void *arg) 3463 { 3464 struct ahci_ccb *ccb = arg; 3465 struct ahci_port *ap = ccb->ccb_port; 3466 3467 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 3468 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_EXPIRED; 3469 ahci_os_signal_port_thread(ap, AP_SIGF_TIMEOUT); 3470 } 3471 3472 /* 3473 * Timeout code, typically called when the port command processor is running. 3474 * 3475 * We have to be very very careful here. We cannot stop the port unless 3476 * CR is already clear or the only active commands remaining are timed-out 3477 * ones. Otherwise stopping the port will race the command processor and 3478 * we can lose events. While we can theoretically just restart everything 3479 * that could result in a double-issue which will not work for ATAPI commands. 3480 */ 3481 void 3482 ahci_ata_cmd_timeout(struct ahci_ccb *ccb) 3483 { 3484 struct ata_xfer *xa = &ccb->ccb_xa; 3485 struct ahci_port *ap = ccb->ccb_port; 3486 struct ata_port *at; 3487 u_int32_t ci_saved; 3488 u_int32_t mask; 3489 int slot; 3490 3491 at = ccb->ccb_xa.at; 3492 3493 kprintf("%s: CMD TIMEOUT state=%d slot=%d\n" 3494 "\tcmd-reg 0x%b\n" 3495 "\tsactive=%08x active=%08x expired=%08x\n" 3496 "\t sact=%08x ci=%08x\n" 3497 "\t STS=%b\n", 3498 ATANAME(ap, at), 3499 ccb->ccb_xa.state, ccb->ccb_slot, 3500 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD, 3501 ap->ap_sactive, ap->ap_active, ap->ap_expired, 3502 ahci_pread(ap, AHCI_PREG_SACT), 3503 ahci_pread(ap, AHCI_PREG_CI), 3504 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS 3505 ); 3506 3507 3508 /* 3509 * NOTE: Timeout will not be running if the command was polled. 3510 * If we got here at least one of these flags should be set. 3511 */ 3512 KKASSERT(xa->flags & (ATA_F_POLL | ATA_F_TIMEOUT_DESIRED | 3513 ATA_F_TIMEOUT_RUNNING)); 3514 xa->flags &= ~(ATA_F_TIMEOUT_RUNNING | ATA_F_TIMEOUT_EXPIRED); 3515 3516 if (ccb->ccb_xa.state == ATA_S_PENDING) { 3517 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 3518 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3519 ccb->ccb_done(ccb); 3520 xa->complete(xa); 3521 ahci_issue_pending_commands(ap, NULL); 3522 return; 3523 } 3524 if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 3525 kprintf("%s: Unexpected state during timeout: %d\n", 3526 ATANAME(ap, at), ccb->ccb_xa.state); 3527 return; 3528 } 3529 3530 /* 3531 * Ok, we can only get this command off the chip if CR is inactive 3532 * or if the only commands running on the chip are all expired. 3533 * Otherwise we have to wait until the port is in a safe state. 3534 * 3535 * Do not set state here, it will cause polls to return when the 3536 * ccb is not yet off the chip. 3537 */ 3538 ap->ap_expired |= 1 << ccb->ccb_slot; 3539 3540 if ((ahci_pread(ap, AHCI_PREG_CMD) & AHCI_PREG_CMD_CR) && 3541 (ap->ap_active | ap->ap_sactive) != ap->ap_expired) { 3542 /* 3543 * If using FBSS or NCQ we can't safely stop the port 3544 * right now. 3545 */ 3546 kprintf("%s: Deferred timeout until its safe, slot %d\n", 3547 ATANAME(ap, at), ccb->ccb_slot); 3548 return; 3549 } 3550 3551 /* 3552 * We can safely stop the port and process all expired ccb's, 3553 * which will include our current ccb. 3554 */ 3555 ci_saved = (ap->ap_sactive) ? ahci_pread(ap, AHCI_PREG_SACT) : 3556 ahci_pread(ap, AHCI_PREG_CI); 3557 ahci_port_stop(ap, 0); 3558 3559 while (ap->ap_expired) { 3560 slot = ffs(ap->ap_expired) - 1; 3561 ap->ap_expired &= ~(1 << slot); 3562 ci_saved &= ~(1 << slot); 3563 ccb = &ap->ap_ccbs[slot]; 3564 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3565 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 3566 KKASSERT(ap->ap_sactive & (1 << slot)); 3567 ap->ap_sactive &= ~(1 << slot); 3568 } else { 3569 KKASSERT(ap->ap_active & (1 << slot)); 3570 ap->ap_active &= ~(1 << slot); 3571 --ap->ap_active_cnt; 3572 } 3573 ccb->ccb_done(ccb); 3574 ccb->ccb_xa.complete(&ccb->ccb_xa); 3575 } 3576 /* ccb invalid now */ 3577 3578 /* 3579 * We can safely CLO the port to clear any BSY/DRQ, a case which 3580 * can occur with port multipliers. This will unbrick the port 3581 * and allow commands to other targets behind the PM continue. 3582 * (FBSS). 3583 * 3584 * Finally, once the port has been restarted we can issue any 3585 * previously saved pending commands, and run the port interrupt 3586 * code to handle any completions which may have occured when 3587 * we saved CI. 3588 */ 3589 if (ahci_pread(ap, AHCI_PREG_TFD) & 3590 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 3591 kprintf("%s: Warning, issuing CLO after timeout\n", 3592 ATANAME(ap, at)); 3593 ahci_port_clo(ap); 3594 } 3595 ahci_port_start(ap); 3596 3597 /* 3598 * We absolutely must make sure the chipset cleared activity on 3599 * all slots. This sometimes might not happen due to races with 3600 * a chipset interrupt which stops the port before we can manage 3601 * to. For some reason some chipsets don't clear the active 3602 * commands when we turn off CMD_ST after the chip has stopped 3603 * operations itself. 3604 */ 3605 if (ahci_pactive(ap) != 0) { 3606 ahci_port_stop(ap, 0); 3607 ahci_port_start(ap); 3608 if ((mask = ahci_pactive(ap)) != 0) { 3609 kprintf("%s: quick-timeout: chipset failed " 3610 "to clear active cmds %08x\n", 3611 PORTNAME(ap), mask); 3612 } 3613 } 3614 ahci_issue_saved_commands(ap, ci_saved & ~ap->ap_expired); 3615 ahci_issue_pending_commands(ap, NULL); 3616 ahci_port_intr(ap, 0); 3617 } 3618 3619 /* 3620 * Issue a previously saved set of commands 3621 */ 3622 void 3623 ahci_issue_saved_commands(struct ahci_port *ap, u_int32_t ci_saved) 3624 { 3625 if (ci_saved) { 3626 KKASSERT(!((ap->ap_active & ci_saved) && 3627 (ap->ap_sactive & ci_saved))); 3628 KKASSERT((ci_saved & ap->ap_expired) == 0); 3629 if (ap->ap_sactive & ci_saved) 3630 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 3631 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 3632 } 3633 } 3634 3635 /* 3636 * Used by the softreset, pmprobe, and read_ncq_error only, in very 3637 * specialized, controlled circumstances. 3638 * 3639 * Only one command may be pending. 3640 */ 3641 void 3642 ahci_quick_timeout(struct ahci_ccb *ccb) 3643 { 3644 struct ahci_port *ap = ccb->ccb_port; 3645 u_int32_t mask; 3646 3647 switch (ccb->ccb_xa.state) { 3648 case ATA_S_PENDING: 3649 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 3650 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3651 break; 3652 case ATA_S_ONCHIP: 3653 /* 3654 * We have to clear the command on-chip. 3655 */ 3656 KKASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 3657 ap->ap_sactive == 0); 3658 ahci_port_stop(ap, 0); 3659 ahci_port_start(ap); 3660 if (ahci_pactive(ap) != 0) { 3661 ahci_port_stop(ap, 0); 3662 ahci_port_start(ap); 3663 if ((mask = ahci_pactive(ap)) != 0) { 3664 kprintf("%s: quick-timeout: chipset failed " 3665 "to clear active cmds %08x\n", 3666 PORTNAME(ap), mask); 3667 } 3668 } 3669 3670 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3671 ap->ap_active &= ~(1 << ccb->ccb_slot); 3672 KKASSERT(ap->ap_active_cnt > 0); 3673 --ap->ap_active_cnt; 3674 break; 3675 default: 3676 panic("%s: ahci_quick_timeout: ccb in bad state %d", 3677 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_xa.state); 3678 } 3679 } 3680 3681 static void 3682 ahci_dummy_done(struct ata_xfer *xa) 3683 { 3684 } 3685 3686 static void 3687 ahci_empty_done(struct ahci_ccb *ccb) 3688 { 3689 } 3690 3691 int 3692 ahci_set_feature(struct ahci_port *ap, struct ata_port *atx, 3693 int feature, int enable) 3694 { 3695 struct ata_port *at; 3696 struct ata_xfer *xa; 3697 int error; 3698 3699 at = atx ? atx : ap->ap_ata[0]; 3700 3701 xa = ahci_ata_get_xfer(ap, atx); 3702 3703 xa->fis->type = ATA_FIS_TYPE_H2D; 3704 xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target; 3705 xa->fis->command = ATA_C_SET_FEATURES; 3706 xa->fis->features = enable ? ATA_C_SATA_FEATURE_ENA : 3707 ATA_C_SATA_FEATURE_DIS; 3708 xa->fis->sector_count = feature; 3709 xa->fis->control = ATA_FIS_CONTROL_4BIT; 3710 3711 xa->complete = ahci_dummy_done; 3712 xa->datalen = 0; 3713 xa->flags = ATA_F_POLL; 3714 xa->timeout = 1000; 3715 3716 if (ahci_ata_cmd(xa) == ATA_S_COMPLETE) 3717 error = 0; 3718 else 3719 error = EIO; 3720 ahci_ata_put_xfer(xa); 3721 return(error); 3722 } 3723