xref: /openbsd-src/sys/dev/ic/ahci.c (revision 9f11ffb7133c203312a01e4b986886bc88c7d74b)
1 /*	$OpenBSD: ahci.c,v 1.32 2017/08/21 21:43:46 jmatthew Exp $ */
2 
3 /*
4  * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5  * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6  * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7  *
8  * Permission to use, copy, modify, and distribute this software for any
9  * purpose with or without fee is hereby granted, provided that the above
10  * copyright notice and this permission notice appear in all copies.
11  *
12  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19  */
20 
21 #include <sys/param.h>
22 #include <sys/systm.h>
23 #include <sys/buf.h>
24 #include <sys/kernel.h>
25 #include <sys/malloc.h>
26 #include <sys/device.h>
27 #include <sys/queue.h>
28 #include <sys/mutex.h>
29 #include <sys/pool.h>
30 
31 #include <machine/bus.h>
32 
33 #include <dev/ic/ahcireg.h>
34 #include <dev/ic/ahcivar.h>
35 
36 #ifdef AHCI_DEBUG
37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38     while (0)
39 #define AHCI_D_TIMEOUT		0x00
40 #define AHCI_D_VERBOSE		0x01
41 #define AHCI_D_INTR		0x02
42 #define AHCI_D_XFER		0x08
43 int ahcidebug = AHCI_D_VERBOSE;
44 #else
45 #define DPRINTF(m, f...)
46 #endif
47 
48 #ifdef HIBERNATE
49 #include <uvm/uvm_extern.h>
50 #include <sys/hibernate.h>
51 #include <sys/disk.h>
52 #include <sys/disklabel.h>
53 
54 #include <scsi/scsi_all.h>
55 #include <scsi/scsiconf.h>
56 
57 void			ahci_hibernate_io_start(struct ahci_port *,
58 			    struct ahci_ccb *);
59 int			ahci_hibernate_io_poll(struct ahci_port *,
60 			    struct ahci_ccb *);
61 void			ahci_hibernate_load_prdt(struct ahci_ccb *);
62 
63 int			ahci_hibernate_io(dev_t dev, daddr_t blkno,
64 			    vaddr_t addr, size_t size, int wr, void *page);
65 #endif
66 
67 struct cfdriver ahci_cd = {
68 	NULL, "ahci", DV_DULL
69 };
70 
71 void			ahci_enable_interrupts(struct ahci_port *);
72 
73 int			ahci_init(struct ahci_softc *);
74 int			ahci_port_alloc(struct ahci_softc *, u_int);
75 void			ahci_port_detect(struct ahci_softc *, u_int);
76 void			ahci_port_free(struct ahci_softc *, u_int);
77 int			ahci_port_init(struct ahci_softc *, u_int);
78 
79 int			ahci_default_port_start(struct ahci_port *, int);
80 int			ahci_port_stop(struct ahci_port *, int);
81 int			ahci_port_clo(struct ahci_port *);
82 int			ahci_port_softreset(struct ahci_port *);
83 void			ahci_port_comreset(struct ahci_port *);
84 int			ahci_port_portreset(struct ahci_port *, int);
85 void			ahci_port_portreset_start(struct ahci_port *);
86 int			ahci_port_portreset_poll(struct ahci_port *);
87 void			ahci_port_portreset_wait(struct ahci_port *);
88 int			ahci_port_portreset_finish(struct ahci_port *, int);
89 int			ahci_port_signature(struct ahci_port *);
90 int			ahci_pmp_port_softreset(struct ahci_port *, int);
91 int			ahci_pmp_port_portreset(struct ahci_port *, int);
92 int			ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93 
94 int			ahci_load_prdt(struct ahci_ccb *);
95 void			ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96 			    u_int32_t, u_int32_t);
97 void			ahci_unload_prdt(struct ahci_ccb *);
98 
99 int			ahci_poll(struct ahci_ccb *, int, void (*)(void *));
100 void			ahci_start(struct ahci_ccb *);
101 
102 void			ahci_issue_pending_ncq_commands(struct ahci_port *);
103 void			ahci_issue_pending_commands(struct ahci_port *, int);
104 
105 int			ahci_intr(void *);
106 u_int32_t		ahci_port_intr(struct ahci_port *, u_int32_t);
107 
108 struct ahci_ccb		*ahci_get_ccb(struct ahci_port *);
109 void			ahci_put_ccb(struct ahci_ccb *);
110 
111 struct ahci_ccb		*ahci_get_err_ccb(struct ahci_port *);
112 void			ahci_put_err_ccb(struct ahci_ccb *);
113 
114 struct ahci_ccb		*ahci_get_pmp_ccb(struct ahci_port *);
115 void			ahci_put_pmp_ccb(struct ahci_ccb *);
116 
117 int			ahci_port_read_ncq_error(struct ahci_port *, int *, int);
118 
119 struct ahci_dmamem	*ahci_dmamem_alloc(struct ahci_softc *, size_t);
120 void			ahci_dmamem_free(struct ahci_softc *,
121 			    struct ahci_dmamem *);
122 
123 u_int32_t		ahci_read(struct ahci_softc *, bus_size_t);
124 void			ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125 int			ahci_wait_ne(struct ahci_softc *, bus_size_t,
126 			    u_int32_t, u_int32_t);
127 
128 u_int32_t		ahci_pread(struct ahci_port *, bus_size_t);
129 void			ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130 int			ahci_pwait_eq(struct ahci_port *, bus_size_t,
131 			    u_int32_t, u_int32_t, int);
132 void			ahci_flush_tfd(struct ahci_port *ap);
133 u_int32_t		ahci_active_mask(struct ahci_port *);
134 int			ahci_port_detect_pmp(struct ahci_port *);
135 void			ahci_pmp_probe_timeout(void *);
136 
137 /* pmp operations */
138 int			ahci_pmp_read(struct ahci_port *, int, int,
139 			    u_int32_t *);
140 int			ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141 int			ahci_pmp_phy_status(struct ahci_port *, int,
142 			    u_int32_t *);
143 int 			ahci_pmp_identify(struct ahci_port *, int *);
144 
145 
146 /* Wait for all bits in _b to be cleared */
147 #define ahci_pwait_clr(_ap, _r, _b, _n) \
148    ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149 
150 /* Wait for all bits in _b to be set */
151 #define ahci_pwait_set(_ap, _r, _b, _n) \
152    ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153 
154 
155 
156 /* provide methods for atascsi to call */
157 int			ahci_ata_probe(void *, int, int);
158 void			ahci_ata_free(void *, int, int);
159 struct ata_xfer *	ahci_ata_get_xfer(void *, int);
160 void			ahci_ata_put_xfer(struct ata_xfer *);
161 void			ahci_ata_cmd(struct ata_xfer *);
162 
163 struct atascsi_methods ahci_atascsi_methods = {
164 	ahci_ata_probe,
165 	ahci_ata_free,
166 	ahci_ata_get_xfer,
167 	ahci_ata_put_xfer,
168 	ahci_ata_cmd
169 };
170 
171 /* ccb completions */
172 void			ahci_ata_cmd_done(struct ahci_ccb *);
173 void			ahci_pmp_cmd_done(struct ahci_ccb *);
174 void			ahci_ata_cmd_timeout(void *);
175 void			ahci_empty_done(struct ahci_ccb *);
176 
177 int
178 ahci_attach(struct ahci_softc *sc)
179 {
180 	struct atascsi_attach_args	aaa;
181 	u_int32_t			pi;
182 	int				i, j, done;
183 
184 	if (sc->sc_port_start == NULL)
185 		sc->sc_port_start = ahci_default_port_start;
186 
187 	if (ahci_init(sc) != 0) {
188 		/* error already printed by ahci_init */
189 		goto unmap;
190 	}
191 
192 	printf("\n");
193 
194 	sc->sc_cap = ahci_read(sc, AHCI_REG_CAP);
195 	sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap);
196 #ifdef AHCI_DEBUG
197 	if (ahcidebug & AHCI_D_VERBOSE) {
198 		const char *gen;
199 
200 		switch (sc->sc_cap & AHCI_REG_CAP_ISS) {
201 		case AHCI_REG_CAP_ISS_G1:
202 			gen = "1 (1.5Gbps)";
203 			break;
204 		case AHCI_REG_CAP_ISS_G2:
205 			gen = "2 (3.0Gb/s)";
206 			break;
207 		case AHCI_REG_CAP_ISS_G3:
208 			gen = "3 (6.0Gb/s)";
209 			break;
210 		default:
211 			gen = "unknown";
212 			break;
213 		}
214 
215 		printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216 		    DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP,
217 		    AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen);
218 		printf("%s: extended capabilities 0x%b\n", DEVNAME(sc),
219 		    ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2);
220 	}
221 #endif
222 
223 	pi = ahci_read(sc, AHCI_REG_PI);
224 	DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225 	    DEVNAME(sc), pi);
226 
227 #ifdef AHCI_COALESCE
228 	/* Naive coalescing support - enable for all ports. */
229 	if (sc->sc_cap & AHCI_REG_CAP_CCCS) {
230 		u_int16_t		ccc_timeout = 20;
231 		u_int8_t		ccc_numcomplete = 12;
232 		u_int32_t		ccc_ctl;
233 
234 		/* disable coalescing during reconfiguration. */
235 		ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
236 		ccc_ctl &= ~0x00000001;
237 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
238 
239 		sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
240 		if (pi & sc->sc_ccc_mask) {
241 			/* A conflict with the implemented port list? */
242 			printf("%s: coalescing interrupt/implemented port list "
243 			    "conflict, PI: %08x, ccc_mask: %08x\n",
244 			    DEVNAME(sc), pi, sc->sc_ccc_mask);
245 			sc->sc_ccc_mask = 0;
246 			goto noccc;
247 		}
248 
249 		/* ahci_port_start will enable each port when it starts. */
250 		sc->sc_ccc_ports = pi;
251 		sc->sc_ccc_ports_cur = 0;
252 
253 		/* program thresholds and enable overall coalescing. */
254 		ccc_ctl &= ~0xffffff00;
255 		ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
256 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
257 		ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
258 		ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
259 	}
260 noccc:
261 #endif
262 	/*
263 	 * Given that ahci_port_alloc() will grab one CCB for error recovery
264 	 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265 	 * pretend at least 2 command slots for devices without NCQ support.
266 	 * That way, also at least 1 slot is made available for atascsi(4).
267 	 */
268 	sc->sc_ncmds = max(2, sc->sc_ncmds);
269 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
270 		if (!ISSET(pi, 1 << i)) {
271 			/* dont allocate stuff if the port isnt implemented */
272 			continue;
273 		}
274 
275 		if (ahci_port_alloc(sc, i) == ENOMEM)
276 			goto freeports;
277 
278 		if (sc->sc_ports[i] != NULL)
279 			ahci_port_portreset_start(sc->sc_ports[i]);
280 	}
281 
282 	/*
283 	 * Poll for device detection until all ports report a device, or one
284 	 * second has elapsed.
285 	 */
286 	for (i = 0; i < 1000; i++) {
287 		done = 1;
288 		for (j = 0; j < AHCI_MAX_PORTS; j++) {
289 			if (sc->sc_ports[j] == NULL)
290 				continue;
291 
292 			if (ahci_port_portreset_poll(sc->sc_ports[j]))
293 				done = 0;
294 		}
295 
296 		if (done)
297 			break;
298 
299 		delay(1000);
300 	}
301 
302 	/*
303 	 * Finish device detection on all ports that initialized.
304 	 */
305 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
306 		if (sc->sc_ports[i] != NULL)
307 			ahci_port_detect(sc, i);
308 	}
309 
310 	memset(&aaa, 0, sizeof(aaa));
311 	aaa.aaa_cookie = sc;
312 	aaa.aaa_methods = &ahci_atascsi_methods;
313 	aaa.aaa_minphys = NULL;
314 	aaa.aaa_nports = AHCI_MAX_PORTS;
315 	aaa.aaa_ncmds = sc->sc_ncmds - 1;
316 	if (!(sc->sc_flags & AHCI_F_NO_NCQ) &&
317 	    sc->sc_ncmds > 2 &&
318 	    (sc->sc_cap & AHCI_REG_CAP_SNCQ)) {
319 		aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ;
320 	}
321 
322 	sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323 
324 	/* Enable interrupts */
325 	ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
326 
327 	return 0;
328 
329 freeports:
330 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
331 		if (sc->sc_ports[i] != NULL)
332 			ahci_port_free(sc, i);
333 	}
334 unmap:
335 	/* Disable controller */
336 	ahci_write(sc, AHCI_REG_GHC, 0);
337 	return 1;
338 }
339 
340 int
341 ahci_detach(struct ahci_softc *sc, int flags)
342 {
343 	int				 rv, i;
344 
345 	if (sc->sc_atascsi != NULL) {
346 		rv = atascsi_detach(sc->sc_atascsi, flags);
347 		if (rv != 0)
348 			return (rv);
349 	}
350 
351 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
352 		if (sc->sc_ports[i] != NULL)
353 			ahci_port_free(sc, i);
354 	}
355 
356 	return (0);
357 }
358 
359 int
360 ahci_activate(struct device *self, int act)
361 {
362 	struct ahci_softc		*sc = (struct ahci_softc *)self;
363 	int				 i, rv = 0;
364 
365 	switch (act) {
366 	case DVACT_RESUME:
367 		/* enable ahci (global interrupts disabled) */
368 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
369 
370 		/* restore BIOS initialised parameters */
371 		ahci_write(sc, AHCI_REG_CAP, sc->sc_cap);
372 
373 		for (i = 0; i < AHCI_MAX_PORTS; i++) {
374 			if (sc->sc_ports[i] != NULL)
375 				ahci_port_init(sc, i);
376 		}
377 
378 		/* Enable interrupts */
379 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
380 
381 		rv = config_activate_children(self, act);
382 		break;
383 	case DVACT_POWERDOWN:
384 		rv = config_activate_children(self, act);
385 		for (i = 0; i < AHCI_MAX_PORTS; i++) {
386 			if (sc->sc_ports[i] != NULL)
387 				ahci_port_stop(sc->sc_ports[i], 1);
388 		}
389 		break;
390 	default:
391 		rv = config_activate_children(self, act);
392 		break;
393 	}
394 	return (rv);
395 }
396 
397 int
398 ahci_init(struct ahci_softc *sc)
399 {
400 	u_int32_t			reg, cap, pi;
401 	const char			*revision;
402 
403 	DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
404 	    AHCI_FMT_GHC);
405 
406 	/* save BIOS initialised parameters, enable staggered spin up */
407 	cap = ahci_read(sc, AHCI_REG_CAP);
408 	cap &= AHCI_REG_CAP_SMPS;
409 	cap |= AHCI_REG_CAP_SSS;
410 	pi = ahci_read(sc, AHCI_REG_PI);
411 
412 	if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
413 		/* reset the controller */
414 		ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
415 		if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
416 		    AHCI_REG_GHC_HR) != 0) {
417 			printf(" unable to reset controller\n");
418 			return (1);
419 		}
420 	}
421 
422 	/* enable ahci (global interrupts disabled) */
423 	ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
424 
425 	/* restore parameters */
426 	ahci_write(sc, AHCI_REG_CAP, cap);
427 	ahci_write(sc, AHCI_REG_PI, pi);
428 
429 	/* check the revision */
430 	reg = ahci_read(sc, AHCI_REG_VS);
431 	switch (reg) {
432 	case AHCI_REG_VS_0_95:
433 		revision = "0.95";
434 		break;
435 	case AHCI_REG_VS_1_0:
436 		revision = "1.0";
437 		break;
438 	case AHCI_REG_VS_1_1:
439 		revision = "1.1";
440 		break;
441 	case AHCI_REG_VS_1_2:
442 		revision = "1.2";
443 		break;
444 	case AHCI_REG_VS_1_3:
445 		revision = "1.3";
446 		break;
447 	case AHCI_REG_VS_1_3_1:
448 		revision = "1.3.1";
449 		break;
450 
451 	default:
452 		printf(" unsupported AHCI revision 0x%08x\n", reg);
453 		return (1);
454 	}
455 
456 	printf(" AHCI %s", revision);
457 
458 	return (0);
459 }
460 
461 void
462 ahci_enable_interrupts(struct ahci_port *ap)
463 {
464 	ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
465 	    AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
466 	    AHCI_PREG_IE_UFE |
467 	    ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) |
468 #ifdef AHCI_COALESCE
469 	    ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
470 	     (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE))
471 #else
472 	    AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
473 #endif
474 	    );
475 }
476 
477 int
478 ahci_port_alloc(struct ahci_softc *sc, u_int port)
479 {
480 	struct ahci_port		*ap;
481 	struct ahci_ccb			*ccb;
482 	u_int64_t			dva;
483 	u_int32_t			cmd;
484 	struct ahci_cmd_hdr		*hdr;
485 	struct ahci_cmd_table		*table;
486 	int				i, rc = ENOMEM;
487 
488 	ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO);
489 	if (ap == NULL) {
490 		printf("%s: unable to allocate memory for port %d\n",
491 		    DEVNAME(sc), port);
492 		goto reterr;
493 	}
494 	ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO);
495 	if (ap->ap_err_scratch == NULL) {
496 		printf("%s: unable to allocate DMA scratch buf for port %d\n",
497 		    DEVNAME(sc), port);
498 		free(ap, M_DEVBUF, sizeof(*ap));
499 		goto reterr;
500 	}
501 
502 #ifdef AHCI_DEBUG
503 	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
504 	    DEVNAME(sc), port);
505 #endif
506 	ap->ap_port = port;
507 	sc->sc_ports[port] = ap;
508 
509 	if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
510 	    AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
511 		printf("%s: unable to create register window for port %d\n",
512 		    DEVNAME(sc), port);
513 		goto freeport;
514 	}
515 
516 	ap->ap_sc = sc;
517 #ifdef AHCI_COALESCE
518 	ap->ap_num = port;
519 #endif
520 	TAILQ_INIT(&ap->ap_ccb_free);
521 	TAILQ_INIT(&ap->ap_ccb_pending);
522 	mtx_init(&ap->ap_ccb_mtx, IPL_BIO);
523 
524 	/* Disable port interrupts */
525 	ahci_pwrite(ap, AHCI_PREG_IE, 0);
526 
527 	/* Sec 10.1.2 - deinitialise port if it is already running */
528 	cmd = ahci_pread(ap, AHCI_PREG_CMD);
529 	if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
530 	    AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
531 	    ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
532 		int r;
533 
534 		r = ahci_port_stop(ap, 1);
535 		if (r) {
536 			printf("%s: unable to disable %s, ignoring port %d\n",
537 			    DEVNAME(sc), r == 2 ? "CR" : "FR", port);
538 			rc = ENXIO;
539 			goto freeport;
540 		}
541 
542 		/* Write DET to zero */
543 		ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
544 	}
545 
546 	/* Allocate RFIS */
547 	ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
548 	if (ap->ap_dmamem_rfis == NULL)
549 		goto nomem;
550 
551 	/* Setup RFIS base address */
552 	ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
553 	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
554 	ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
555 	ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
556 
557 	/* Enable FIS reception and activate port. */
558 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
559 	cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
560 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
561 
562 	/* Check whether port activated.  Skip it if not. */
563 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
564 	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
565 		rc = ENXIO;
566 		goto freeport;
567 	}
568 
569 	/* Allocate a CCB for each command slot */
570 	ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
571 	    M_DEVBUF, M_NOWAIT | M_ZERO);
572 	if (ap->ap_ccbs == NULL) {
573 		printf("%s: unable to allocate command list for port %d\n",
574 		    DEVNAME(sc), port);
575 		goto freeport;
576 	}
577 
578 	/* Command List Structures and Command Tables */
579 	ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
580 	    sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
581 	ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
582 	    sc->sc_ncmds * sizeof(struct ahci_cmd_table));
583 	if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
584 nomem:
585 		printf("%s: unable to allocate DMA memory for port %d\n",
586 		    DEVNAME(sc), port);
587 		goto freeport;
588 	}
589 
590 	/* Setup command list base address */
591 	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
592 	ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
593 	ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
594 
595 	/* Split CCB allocation into CCBs and assign to command header/table */
596 	hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
597 	table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
598 	for (i = 0; i < sc->sc_ncmds; i++) {
599 		ccb = &ap->ap_ccbs[i];
600 
601 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
602 		    (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
603 		    &ccb->ccb_dmamap) != 0) {
604 			printf("%s: unable to create dmamap for port %d "
605 			    "ccb %d\n", DEVNAME(sc), port, i);
606 			goto freeport;
607 		}
608 
609 		ccb->ccb_slot = i;
610 		ccb->ccb_port = ap;
611 		ccb->ccb_cmd_hdr = &hdr[i];
612 		ccb->ccb_cmd_table = &table[i];
613 		htolem64(&ccb->ccb_cmd_hdr->ctba,
614 		    AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
615 		    ccb->ccb_slot * sizeof(struct ahci_cmd_table));
616 
617 		ccb->ccb_xa.fis =
618 		    (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
619 		ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
620 		ccb->ccb_xa.tag = i;
621 
622 		ccb->ccb_xa.state = ATA_S_COMPLETE;
623 		ahci_put_ccb(ccb);
624 	}
625 
626 	/* grab a ccb for use during error recovery */
627 	ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
628 	TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry);
629 	ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE;
630 
631 	/* Wait for ICC change to complete */
632 	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
633 	rc = 0;
634 
635 freeport:
636 	if (rc != 0)
637 		ahci_port_free(sc, port);
638 reterr:
639 	return (rc);
640 }
641 
642 void
643 ahci_port_detect(struct ahci_softc *sc, u_int port)
644 {
645 	struct ahci_port		*ap;
646 	const char			*speed;
647 	int				rc;
648 
649 	ap = sc->sc_ports[port];
650 
651 	rc = ahci_port_portreset_finish(ap, 1);
652 	switch (rc) {
653 	case ENODEV:
654 		switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
655 		case AHCI_PREG_SSTS_DET_DEV_NE:
656 			printf("%s: device not communicating on port %d\n",
657 			    DEVNAME(sc), port);
658 			break;
659 		case AHCI_PREG_SSTS_DET_PHYOFFLINE:
660 			printf("%s: PHY offline on port %d\n", DEVNAME(sc),
661 			    port);
662 			break;
663 		default:
664 			DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
665 			    "on port %d\n", DEVNAME(sc), port);
666 			break;
667 		}
668 		goto freeport;
669 
670 	case EBUSY:
671 		printf("%s: device on port %d didn't come ready, "
672 		    "TFD: 0x%b\n", DEVNAME(sc), port,
673 		    ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
674 
675 		/* Try a soft reset to clear busy */
676 		rc = ahci_port_softreset(ap);
677 		if (rc) {
678 			printf("%s: unable to communicate "
679 			    "with device on port %d\n", DEVNAME(sc), port);
680 			goto freeport;
681 		}
682 		break;
683 
684 	default:
685 		break;
686 	}
687 
688 	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
689 	    DEVNAME(sc), port, rc);
690 
691 	/* Read current link speed */
692 	switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) {
693 	case AHCI_PREG_SSTS_SPD_GEN1:
694 		speed = "1.5Gb/s";
695 		break;
696 	case AHCI_PREG_SSTS_SPD_GEN2:
697 		speed = "3.0Gb/s";
698 		break;
699 	case AHCI_PREG_SSTS_SPD_GEN3:
700 		speed = "6.0Gb/s";
701 		break;
702 	default:
703 		speed = NULL;
704 		break;
705 	}
706 	if (speed != NULL)
707 		printf("%s: port %d: %s\n", PORTNAME(ap), port, speed);
708 
709 	/* Enable command transfers on port */
710 	if (ahci_port_start(ap, 0)) {
711 		printf("%s: failed to start command DMA on port %d, "
712 		    "disabling\n", DEVNAME(sc), port);
713 		rc = ENXIO;	/* couldn't start port */
714 	}
715 
716 	/* Flush interrupts for port */
717 	ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
718 	ahci_write(sc, AHCI_REG_IS, 1 << port);
719 
720 	ahci_enable_interrupts(ap);
721 freeport:
722 	if (rc != 0)
723 		ahci_port_free(sc, port);
724 }
725 
726 void
727 ahci_port_free(struct ahci_softc *sc, u_int port)
728 {
729 	struct ahci_port		*ap = sc->sc_ports[port];
730 	struct ahci_ccb			*ccb;
731 
732 	/* Ensure port is disabled and its interrupts are flushed */
733 	if (ap->ap_sc) {
734 		ahci_pwrite(ap, AHCI_PREG_CMD, 0);
735 		ahci_pwrite(ap, AHCI_PREG_IE, 0);
736 		ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
737 		ahci_write(sc, AHCI_REG_IS, 1 << port);
738 	}
739 
740 	if (ap->ap_ccb_err)
741 		ahci_put_ccb(ap->ap_ccb_err);
742 
743 	if (ap->ap_ccbs) {
744 		while ((ccb = ahci_get_ccb(ap)) != NULL)
745 			bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
746 		free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb));
747 	}
748 
749 	if (ap->ap_dmamem_cmd_list)
750 		ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
751 	if (ap->ap_dmamem_rfis)
752 		ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
753 	if (ap->ap_dmamem_cmd_table)
754 		ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
755 	if (ap->ap_err_scratch)
756 		dma_free(ap->ap_err_scratch, DEV_BSIZE);
757 
758 	/* bus_space(9) says we dont free the subregions handle */
759 
760 	free(ap, M_DEVBUF, sizeof(*ap));
761 	sc->sc_ports[port] = NULL;
762 }
763 
764 int
765 ahci_port_init(struct ahci_softc *sc, u_int port)
766 {
767 	struct ahci_port		*ap;
768 	u_int64_t			dva;
769 	u_int32_t			cmd;
770 	int				rc = ENOMEM;
771 
772 	ap = sc->sc_ports[port];
773 #ifdef AHCI_DEBUG
774 	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
775 	    DEVNAME(sc), port);
776 #endif
777 
778 	/* Disable port interrupts */
779 	ahci_pwrite(ap, AHCI_PREG_IE, 0);
780 
781 	/* Sec 10.1.2 - deinitialise port if it is already running */
782 	cmd = ahci_pread(ap, AHCI_PREG_CMD);
783 	if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
784 	    AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
785 	    ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
786 		int r;
787 
788 		r = ahci_port_stop(ap, 1);
789 		if (r) {
790 			printf("%s: unable to disable %s, ignoring port %d\n",
791 			    DEVNAME(sc), r == 2 ? "CR" : "FR", port);
792 			rc = ENXIO;
793 			goto reterr;
794 		}
795 
796 		/* Write DET to zero */
797 		ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
798 	}
799 
800 	/* Setup RFIS base address */
801 	ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
802 	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
803 	ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
804 	ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
805 
806 	/* Enable FIS reception and activate port. */
807 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
808 	cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
809 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
810 
811 	/* Check whether port activated.  Skip it if not. */
812 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
813 	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
814 		rc = ENXIO;
815 		goto reterr;
816 	}
817 
818 	/* Setup command list base address */
819 	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
820 	ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
821 	ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
822 
823 	/* Wait for ICC change to complete */
824 	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
825 
826 	/* Reset port */
827 	rc = ahci_port_portreset(ap, 1);
828 	switch (rc) {
829 	case ENODEV:
830 		switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
831 		case AHCI_PREG_SSTS_DET_DEV_NE:
832 			printf("%s: device not communicating on port %d\n",
833 			    DEVNAME(sc), port);
834 			break;
835 		case AHCI_PREG_SSTS_DET_PHYOFFLINE:
836 			printf("%s: PHY offline on port %d\n", DEVNAME(sc),
837 			    port);
838 			break;
839 		default:
840 			DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
841 			    "on port %d\n", DEVNAME(sc), port);
842 			break;
843 		}
844 		goto reterr;
845 
846 	case EBUSY:
847 		printf("%s: device on port %d didn't come ready, "
848 		    "TFD: 0x%b\n", DEVNAME(sc), port,
849 		    ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
850 
851 		/* Try a soft reset to clear busy */
852 		rc = ahci_port_softreset(ap);
853 		if (rc) {
854 			printf("%s: unable to communicate "
855 			    "with device on port %d\n", DEVNAME(sc), port);
856 			goto reterr;
857 		}
858 		break;
859 
860 	default:
861 		break;
862 	}
863 	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
864 	    DEVNAME(sc), port);
865 
866 	if (ap->ap_pmp_ports > 0) {
867 		int p;
868 
869 		for (p = 0; p < ap->ap_pmp_ports; p++) {
870 			int sig;
871 
872 			/* might need to do a portreset first here? */
873 
874 			/* softreset the port */
875 			if (ahci_pmp_port_softreset(ap, p)) {
876 				printf("%s.%d: unable to probe PMP port due to"
877 				    " softreset failure\n", PORTNAME(ap), p);
878 				continue;
879 			}
880 
881 			sig = ahci_port_signature(ap);
882 			printf("%s.%d: port signature returned %d\n",
883 			    PORTNAME(ap), p, sig);
884 		}
885 	}
886 
887 	/* Enable command transfers on port */
888 	if (ahci_port_start(ap, 0)) {
889 		printf("%s: failed to start command DMA on port %d, "
890 		    "disabling\n", DEVNAME(sc), port);
891 		rc = ENXIO;	/* couldn't start port */
892 	}
893 
894 	/* Flush interrupts for port */
895 	ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
896 	ahci_write(sc, AHCI_REG_IS, 1 << port);
897 
898 	ahci_enable_interrupts(ap);
899 
900 reterr:
901 	return (rc);
902 }
903 
904 int
905 ahci_default_port_start(struct ahci_port *ap, int fre_only)
906 {
907 	u_int32_t			r;
908 
909 	/* Turn on FRE (and ST) */
910 	r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
911 	r |= AHCI_PREG_CMD_FRE;
912 	if (!fre_only)
913 		r |= AHCI_PREG_CMD_ST;
914 	ahci_pwrite(ap, AHCI_PREG_CMD, r);
915 
916 #ifdef AHCI_COALESCE
917 	/* (Re-)enable coalescing on the port. */
918 	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
919 		ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
920 		ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
921 		    ap->ap_sc->sc_ccc_ports_cur);
922 	}
923 #endif
924 
925 	/* Wait for CR to come on */
926 	if (!fre_only &&
927 	    ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
928 		return (1);
929 
930 	return (0);
931 }
932 
933 int
934 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
935 {
936 	u_int32_t			r;
937 
938 #ifdef AHCI_COALESCE
939 	/* Disable coalescing on the port while it is stopped. */
940 	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
941 		ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
942 		ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
943 		    ap->ap_sc->sc_ccc_ports_cur);
944 	}
945 #endif
946 
947 	/* Turn off ST (and FRE) */
948 	r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
949 	r &= ~AHCI_PREG_CMD_ST;
950 	if (stop_fis_rx)
951 		r &= ~AHCI_PREG_CMD_FRE;
952 	ahci_pwrite(ap, AHCI_PREG_CMD, r);
953 
954 	/* Wait for CR to go off */
955 	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
956 		return (1);
957 
958 	/* Wait for FR to go off */
959 	if (stop_fis_rx &&
960 	    ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1))
961 		return (2);
962 
963 	return (0);
964 }
965 
966 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
967 int
968 ahci_port_clo(struct ahci_port *ap)
969 {
970 	struct ahci_softc		*sc = ap->ap_sc;
971 	u_int32_t			cmd;
972 
973 	/* Only attempt CLO if supported by controller */
974 	if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
975 		return (1);
976 
977 	/* Issue CLO */
978 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
979 #ifdef DIAGNOSTIC
980 	if (ISSET(cmd, AHCI_PREG_CMD_ST))
981 		printf("%s: CLO requested while port running\n", PORTNAME(ap));
982 #endif
983 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
984 
985 	/* Wait for completion */
986 	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) {
987 		printf("%s: CLO did not complete\n", PORTNAME(ap));
988 		return (1);
989 	}
990 
991 	return (0);
992 }
993 
994 /* AHCI soft reset, Section 10.4.1 */
995 int
996 ahci_port_softreset(struct ahci_port *ap)
997 {
998 	struct ahci_ccb			*ccb = NULL;
999 	struct ahci_cmd_hdr		*cmd_slot;
1000 	u_int8_t			*fis;
1001 	int				s, rc = EIO, oldstate;
1002 	u_int32_t			cmd;
1003 
1004 	DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1005 
1006 	s = splbio();
1007 	oldstate = ap->ap_state;
1008 	ap->ap_state = AP_S_ERROR_RECOVERY;
1009 
1010 	/* Save previous command register state */
1011 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1012 
1013 	/* Idle port */
1014 	if (ahci_port_stop(ap, 0)) {
1015 		printf("%s: failed to stop port, cannot softreset\n",
1016 		    PORTNAME(ap));
1017 		goto err;
1018 	}
1019 
1020 	/* Request CLO if device appears hung */
1021 	if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
1022 	    AHCI_PREG_TFD_STS_DRQ))
1023 		ahci_port_clo(ap);
1024 
1025 	/* Clear port errors to permit TFD transfer */
1026 	ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1027 
1028 	/* Restart port */
1029 	if (ahci_port_start(ap, 0)) {
1030 		printf("%s: failed to start port, cannot softreset\n",
1031 		    PORTNAME(ap));
1032 		goto err;
1033 	}
1034 
1035 	/* Check whether CLO worked */
1036 	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1037 	    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1038 		printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
1039 		    ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
1040 		    ? "failed" : "unsupported");
1041 		rc = EBUSY;
1042 		goto err;
1043 	}
1044 
1045 	/* Prep first D2H command with SRST feature & clear busy/reset flags */
1046 	ccb = ahci_get_err_ccb(ap);
1047 	cmd_slot = ccb->ccb_cmd_hdr;
1048 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1049 
1050 	fis = ccb->ccb_cmd_table->cfis;
1051 	fis[0] = ATA_FIS_TYPE_H2D;
1052 	fis[15] = ATA_FIS_CONTROL_SRST;
1053 
1054 	cmd_slot->prdtl = 0;
1055 	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1056 	    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1057 	    AHCI_CMD_LIST_FLAG_W);
1058 
1059 	ccb->ccb_xa.state = ATA_S_PENDING;
1060 	if (ahci_poll(ccb, 1000, NULL) != 0)
1061 		goto err;
1062 
1063 	/* Prep second D2H command to read status and complete reset sequence */
1064 	fis[0] = ATA_FIS_TYPE_H2D;
1065 	fis[15] = 0;
1066 
1067 	cmd_slot->prdtl = 0;
1068 	htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W);
1069 
1070 	ccb->ccb_xa.state = ATA_S_PENDING;
1071 	if (ahci_poll(ccb, 1000, NULL) != 0)
1072 		goto err;
1073 
1074 	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1075 	    AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) {
1076 		printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1077 		    PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
1078 		    AHCI_PFMT_TFD_STS);
1079 		rc = EBUSY;
1080 		goto err;
1081 	}
1082 
1083 	rc = 0;
1084 err:
1085 	if (ccb != NULL) {
1086 		/* Abort our command, if it failed, by stopping command DMA. */
1087 		if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
1088 			printf("%s: stopping the port, softreset slot %d was "
1089 			    "still active.\n", PORTNAME(ap), ccb->ccb_slot);
1090 			ahci_port_stop(ap, 0);
1091 		}
1092 		ccb->ccb_xa.state = ATA_S_ERROR;
1093 		ahci_put_err_ccb(ccb);
1094 	}
1095 
1096 	/* Restore saved CMD register state */
1097 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1098 	ap->ap_state = oldstate;
1099 
1100 	splx(s);
1101 
1102 	return (rc);
1103 }
1104 
1105 int
1106 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1107 {
1108 	struct ahci_ccb		*ccb = NULL;
1109 	u_int32_t		data;
1110 	int			count;
1111 	int			rc;
1112 	int			s;
1113 	struct ahci_cmd_hdr	*cmd_slot;
1114 	u_int8_t		*fis;
1115 
1116 	s = splbio();
1117 	/* ignore spurious IFS errors while resetting */
1118 	DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1119 	ap->ap_pmp_ignore_ifs = 1;
1120 
1121 	count = 2;
1122 	rc = 0;
1123 	do {
1124 		if (ccb != NULL) {
1125 			ahci_put_pmp_ccb(ccb);
1126 			ccb = NULL;
1127 		}
1128 
1129 		if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1130 			printf("%s.%d: unable to clear PHY status\n",
1131 			    PORTNAME(ap), pmp_port);
1132 		}
1133 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1134 		/* maybe don't do this on the first loop: */
1135 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1136 		ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1137 
1138 		/* send first softreset FIS */
1139 		ccb = ahci_get_pmp_ccb(ap);
1140 		cmd_slot = ccb->ccb_cmd_hdr;
1141 		memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1142 
1143 		fis = ccb->ccb_cmd_table->cfis;
1144 		fis[0] = ATA_FIS_TYPE_H2D;
1145 		fis[1] = pmp_port;
1146 		fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1147 
1148 		cmd_slot->prdtl = 0;
1149 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1150 		    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1151 		    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1152 
1153 		ccb->ccb_xa.state = ATA_S_PENDING;
1154 
1155 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1156 		    PORTNAME(ap), pmp_port);
1157 		if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1158 			printf("%s.%d: PMP port softreset cmd failed\n",
1159 			       PORTNAME(ap), pmp_port);
1160 			rc = EBUSY;
1161 			if (count > 0) {
1162 				/* probably delay a while to allow
1163 				 * it to settle down?
1164 				 */
1165 			}
1166 			continue;
1167 		}
1168 
1169 		/* send signature FIS */
1170 		memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1171 		fis[0] = ATA_FIS_TYPE_H2D;
1172 		fis[1] = pmp_port;
1173 		fis[15] = ATA_FIS_CONTROL_4BIT;
1174 
1175 		cmd_slot->prdtl = 0;
1176 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1177 		    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1178 
1179 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1180 		    PORTNAME(ap), pmp_port);
1181 		ccb->ccb_xa.state = ATA_S_PENDING;
1182 		if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1183 			DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1184 			    "failed\n", PORTNAME(ap), pmp_port);
1185 			rc = EBUSY;
1186 			if (count > 0) {
1187 				/* sleep a while? */
1188 			}
1189 			continue;
1190 		}
1191 
1192 		fis[15] = 0;
1193 		break;
1194 	} while (count--);
1195 
1196 	if (ccb != NULL) {
1197 		ahci_put_pmp_ccb(ccb);
1198 		ccb = NULL;
1199 	}
1200 
1201 	/* clean up a bit */
1202 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1203 	ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1204 	ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1205 	ap->ap_pmp_ignore_ifs = 0;
1206 	DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1207 	splx(s);
1208 
1209 	return (rc);
1210 }
1211 
1212 int
1213 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1214 {
1215 	int sig;
1216 
1217 	ap->ap_state = AP_S_PMP_PORT_PROBE;
1218 
1219 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1220 	    pmp_port);
1221 	if (ahci_pmp_port_portreset(ap, pmp_port)) {
1222 		printf("%s.%d: unable to probe PMP port; portreset failed\n",
1223 		    PORTNAME(ap), pmp_port);
1224 		ap->ap_state = AP_S_NORMAL;
1225 		return (ATA_PORT_T_NONE);
1226 	}
1227 
1228 	if (ahci_pmp_port_softreset(ap, pmp_port)) {
1229 		printf("%s.%d: unable to probe PMP port due to softreset "
1230 		    "failure\n", PORTNAME(ap), pmp_port);
1231 		ap->ap_state = AP_S_NORMAL;
1232 		return (ATA_PORT_T_NONE);
1233 	}
1234 
1235 	sig = ahci_port_signature(ap);
1236 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1237 	    PORTNAME(ap), pmp_port, sig);
1238 	ap->ap_state = AP_S_NORMAL;
1239 	return (sig);
1240 }
1241 
1242 
1243 void
1244 ahci_flush_tfd(struct ahci_port *ap)
1245 {
1246 	u_int32_t r;
1247 
1248 	r = ahci_pread(ap, AHCI_PREG_SERR);
1249 	if (r & AHCI_PREG_SERR_DIAG_X)
1250 		ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
1251 }
1252 
1253 u_int32_t
1254 ahci_active_mask(struct ahci_port *ap)
1255 {
1256 	u_int32_t mask;
1257 
1258 	mask = ahci_pread(ap, AHCI_PREG_CI);
1259 	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ)
1260 		mask |= ahci_pread(ap, AHCI_PREG_SACT);
1261 	return mask;
1262 }
1263 
1264 void
1265 ahci_pmp_probe_timeout(void *cookie)
1266 {
1267 	struct ahci_ccb *ccb = cookie;
1268 	struct ahci_port *ap = ccb->ccb_port;
1269 	u_int32_t mask;
1270 
1271 	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1272 	switch (ccb->ccb_xa.state) {
1273 	case ATA_S_PENDING:
1274 		TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
1275 		ccb->ccb_xa.state = ATA_S_TIMEOUT;
1276 		break;
1277 
1278 	case ATA_S_ONCHIP:
1279 	case ATA_S_ERROR:  /* currently mostly here for the ATI SBx00 quirk */
1280 		/* clear the command on-chip */
1281 		KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
1282 		    ap->ap_sactive == 0);
1283 		ahci_port_stop(ap, 0);
1284 		ahci_port_start(ap, 0);
1285 
1286 		if (ahci_active_mask(ap) != 0) {
1287 			ahci_port_stop(ap, 0);
1288 			ahci_port_start(ap, 0);
1289 			mask = ahci_active_mask(ap);
1290 			if (mask != 0) {
1291 				printf("%s: ahci_pmp_probe_timeout: failed to "
1292 				    "clear active cmds: %08x\n", PORTNAME(ap),
1293 				    mask);
1294 			}
1295 		}
1296 
1297 		ccb->ccb_xa.state = ATA_S_TIMEOUT;
1298 		ap->ap_active &= ~(1 << ccb->ccb_slot);
1299 		KASSERT(ap->ap_active_cnt > 0);
1300 		--ap->ap_active_cnt;
1301 		DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n",
1302 		    PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt);
1303 		break;
1304 
1305 	default:
1306 		panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1307 			PORTNAME(ap), ccb->ccb_xa.state);
1308 	}
1309 }
1310 
1311 int
1312 ahci_port_signature(struct ahci_port *ap)
1313 {
1314 	u_int32_t sig;
1315 
1316 	sig = ahci_pread(ap, AHCI_PREG_SIG);
1317 	if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
1318 		return (ATA_PORT_T_ATAPI);
1319 	else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER &
1320 	    0xffff0000))
1321 		return (ATA_PORT_T_PM);
1322 	else
1323 		return (ATA_PORT_T_DISK);
1324 }
1325 
1326 int
1327 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1328 {
1329 	u_int32_t cmd, data;
1330 	int loop;
1331 	int rc = 1;
1332 	int s;
1333 
1334 	s = splbio();
1335 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1336 	    pmp_port);
1337 
1338 	/* Save previous command register state */
1339 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1340 
1341 	/* turn off power management and disable the PHY */
1342 	data = AHCI_PREG_SCTL_IPM_DISABLED;
1343 	/* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1344 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1))
1345 		goto err;
1346 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1347 		goto err;
1348 	delay(10000);
1349 
1350 	/* start COMRESET */
1351 	data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1352 	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1353 		DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1354 		    pmp_port);
1355 		data |= AHCI_PREG_SCTL_SPD_GEN1;
1356 	} else
1357 		data |= AHCI_PREG_SCTL_SPD_ANY;
1358 
1359 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1360 		goto err;
1361 
1362 	/* give it a while to settle down */
1363 	delay(100000);
1364 
1365 	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1366 		printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap),
1367 		    pmp_port);
1368 	}
1369 
1370 	/* start trying to negotiate */
1371 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1372 	data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE;
1373 	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1374 		goto err;
1375 
1376 	/* give it a while to detect */
1377 	for (loop = 3; loop; --loop) {
1378 		if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1379 			goto err;
1380 		if (data & AHCI_PREG_SSTS_DET)
1381 			break;
1382 		delay(100000);
1383 	}
1384 	if (loop == 0) {
1385 		printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port);
1386 		goto err;
1387 	}
1388 
1389 	/* give it even longer to fully negotiate */
1390 	for (loop = 30; loop; --loop) {
1391 		if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1392 			goto err;
1393 		if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV)
1394 			break;
1395 		delay(100000);
1396 	}
1397 
1398 	if (loop == 0) {
1399 		printf("%s.%d: device is not negotiating\n", PORTNAME(ap),
1400 		    pmp_port);
1401 		goto err;
1402 	}
1403 
1404 	/* device detected */
1405 	DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1406 	    pmp_port);
1407 
1408 	/* clean up a bit */
1409 	delay(100000);
1410 	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1411 	ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1412 	ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1413 
1414 	rc = 0;
1415 err:
1416 	/* Restore preserved port state */
1417 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1418 	splx(s);
1419 	return (rc);
1420 }
1421 
1422 /* AHCI port reset, Section 10.4.2 */
1423 
1424 void
1425 ahci_port_comreset(struct ahci_port *ap)
1426 {
1427 	u_int32_t			r;
1428 
1429 	r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1430 	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1431 		DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1432 		r |= AHCI_PREG_SCTL_SPD_GEN1;
1433 	} else
1434 		r |= AHCI_PREG_SCTL_SPD_ANY;
1435 	ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1436 	delay(10000);	/* wait at least 1ms for COMRESET to be sent */
1437 	r &= ~AHCI_PREG_SCTL_DET_INIT;
1438 	r |= AHCI_PREG_SCTL_DET_NONE;
1439 	ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1440 	delay(10000);
1441 }
1442 
1443 void
1444 ahci_port_portreset_start(struct ahci_port *ap)
1445 {
1446 	int				s;
1447 
1448 	s = splbio();
1449 	DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1450 
1451 	/* Save previous command register state */
1452 	ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1453 
1454 	/* Clear ST, ignoring failure */
1455 	ahci_port_stop(ap, 0);
1456 
1457 	/* Perform device detection */
1458 	ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
1459 	delay(10000);
1460 	ahci_port_comreset(ap);
1461 	splx(s);
1462 }
1463 
1464 int
1465 ahci_port_portreset_poll(struct ahci_port *ap)
1466 {
1467 	if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) !=
1468 	    AHCI_PREG_SSTS_DET_DEV)
1469 		return (EAGAIN);
1470 	return (0);
1471 }
1472 
1473 void
1474 ahci_port_portreset_wait(struct ahci_port *ap)
1475 {
1476 	int				i;
1477 
1478 	for (i = 0; i < 1000; i++) {
1479 		if (ahci_port_portreset_poll(ap) == 0)
1480 			break;
1481 		delay(1000);
1482 	}
1483 }
1484 
1485 int
1486 ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1487 {
1488 	int				rc, s, retries = 0;
1489 
1490 	s = splbio();
1491 retry:
1492 	if (ahci_port_portreset_poll(ap)) {
1493 		rc = ENODEV;
1494 		if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
1495 			/* this may be a port multiplier with no device
1496 			 * on port 0, so still do the pmp check if requested.
1497 			 */
1498 		} else {
1499 			goto err;
1500 		}
1501 	} else {
1502 		/* Clear SERR (incl X bit), so TFD can update */
1503 		ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1504 
1505 		/* Wait for device to become ready */
1506 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1507 		    AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) {
1508 			/* even if the device doesn't wake up, check if there's
1509 			 * a port multiplier there
1510 			 */
1511 			if (retries == 0) {
1512 				retries = 1;
1513 				ahci_port_comreset(ap);
1514 				ahci_port_portreset_wait(ap);
1515 				goto retry;
1516 			}
1517 			rc = EBUSY;
1518 		} else {
1519 			rc = 0;
1520 		}
1521 	}
1522 
1523 	if (pmp != 0) {
1524 		if (ahci_port_detect_pmp(ap)) {
1525 			/* reset again without pmp support */
1526 			pmp = 0;
1527 			retries = 0;
1528 			ahci_port_comreset(ap);
1529 			ahci_port_portreset_wait(ap);
1530 			goto retry;
1531 		}
1532 	}
1533 
1534 err:
1535 	/* Restore preserved port state */
1536 	ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd);
1537 	ap->ap_saved_cmd = 0;
1538 	splx(s);
1539 
1540 	return (rc);
1541 }
1542 
1543 int
1544 ahci_port_portreset(struct ahci_port *ap, int pmp)
1545 {
1546 	ahci_port_portreset_start(ap);
1547 	ahci_port_portreset_wait(ap);
1548 	return (ahci_port_portreset_finish(ap, pmp));
1549 }
1550 
1551 int
1552 ahci_port_detect_pmp(struct ahci_port *ap)
1553 {
1554 	int				 count, pmp_rc, rc;
1555 	u_int32_t			 r, cmd;
1556 	struct ahci_cmd_hdr		*cmd_slot;
1557 	struct ahci_ccb			*ccb = NULL;
1558 	u_int8_t			*fis = NULL;
1559 
1560 	if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) ||
1561 	    !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) {
1562 		return 0;
1563 	}
1564 
1565 	rc = 0;
1566 	pmp_rc = 0;
1567 	count = 2;
1568 	do {
1569 		DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1570 		    count);
1571 		if (ccb != NULL) {
1572 			ahci_put_pmp_ccb(ccb);
1573 			ccb = NULL;
1574 		}
1575 		ahci_port_stop(ap, 0);
1576 		ap->ap_state = AP_S_PMP_PROBE;
1577 
1578 		/* set PMA in cmd reg */
1579 		cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1580 		if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
1581 			cmd |= AHCI_PREG_CMD_PMA;
1582 			ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1583 		}
1584 
1585 		/* Flush errors and request CLO unconditionally,
1586 		 * then start the port
1587 		 */
1588 		r = ahci_pread(ap, AHCI_PREG_SERR);
1589 		if (r & AHCI_PREG_SERR_DIAG_X)
1590 			ahci_pwrite(ap, AHCI_PREG_SERR,
1591 			    AHCI_PREG_SERR_DIAG_X);
1592 
1593 		/* Request CLO */
1594 		ahci_port_clo(ap);
1595 
1596 		/* Clear port errors to permit TFD transfer */
1597 		r = ahci_pread(ap, AHCI_PREG_SERR);
1598 		ahci_pwrite(ap, AHCI_PREG_SERR, r);
1599 
1600 		/* Restart port */
1601 		if (ahci_port_start(ap, 0)) {
1602 			rc = EBUSY;
1603 			printf("%s: failed to start port, cannot probe PMP\n",
1604 			    PORTNAME(ap));
1605 			break;
1606 		}
1607 
1608 		/* Check whether CLO worked */
1609 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1610 		    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1611 			u_int32_t cap;
1612 
1613 			cap = ahci_read(ap->ap_sc, AHCI_REG_CAP);
1614 			printf("%s: CLO %s, need port reset\n",
1615 			    PORTNAME(ap),
1616 			    ISSET(cap, AHCI_REG_CAP_SCLO)
1617 			    ? "failed" : "unsupported");
1618 			pmp_rc = EBUSY;
1619 			break;
1620 		}
1621 
1622 		/* Prep first command with SRST feature &
1623 		 * clear busy/reset flags
1624 		 */
1625 		ccb = ahci_get_pmp_ccb(ap);
1626 		cmd_slot = ccb->ccb_cmd_hdr;
1627 		memset(ccb->ccb_cmd_table, 0,
1628 		    sizeof(struct ahci_cmd_table));
1629 
1630 		fis = ccb->ccb_cmd_table->cfis;
1631 		fis[0] = ATA_FIS_TYPE_H2D;
1632 		fis[1] = SATA_PMP_CONTROL_PORT;
1633 		fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1634 
1635 		cmd_slot->prdtl = 0;
1636 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1637 		    AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1638 		    AHCI_CMD_LIST_FLAG_PMP);
1639 
1640 		DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1641 		    PORTNAME(ap));
1642 		ccb->ccb_xa.state = ATA_S_PENDING;
1643 		if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1644 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1645 			    PORTNAME(ap));
1646 			pmp_rc = EBUSY;
1647 			continue;
1648 		}
1649 
1650 		if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1651 		    AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1652 			printf("%s: port busy after first PMP probe FIS\n",
1653 			    PORTNAME(ap));
1654 		}
1655 
1656 		/* clear errors in case the device
1657 		 * didn't reset cleanly
1658 		 */
1659 		ahci_flush_tfd(ap);
1660 		r = ahci_pread(ap, AHCI_PREG_SERR);
1661 		ahci_pwrite(ap, AHCI_PREG_SERR, r);
1662 
1663 		/* Prep second command to read status and
1664 		 * complete reset sequence
1665 		 */
1666 		memset(ccb->ccb_cmd_table, 0,
1667 		    sizeof(struct ahci_cmd_table));
1668 		fis[0] = ATA_FIS_TYPE_H2D;
1669 		fis[1] = SATA_PMP_CONTROL_PORT;
1670 		fis[15] = ATA_FIS_CONTROL_4BIT;
1671 
1672 		cmd_slot->prdtl = 0;
1673 		htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1674 		    AHCI_CMD_LIST_FLAG_PMP);
1675 
1676 		DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1677 		    PORTNAME(ap));
1678 		ccb->ccb_xa.state = ATA_S_PENDING;
1679 		if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1680 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1681 			    "cmd failed\n", PORTNAME(ap));
1682 			pmp_rc = EBUSY;
1683 			continue;
1684 		}
1685 
1686 		/* apparently we need to retry at least once
1687 		 * to get the right signature
1688 		 */
1689 		fis[15] = 0;
1690 		pmp_rc = 0;
1691 	} while (--count);
1692 
1693 	if (ccb != NULL) {
1694 		ahci_put_pmp_ccb(ccb);
1695 		ccb = NULL;
1696 	}
1697 
1698 	if (ap->ap_state == AP_S_PMP_PROBE) {
1699 		ap->ap_state = AP_S_NORMAL;
1700 	}
1701 
1702 	if (pmp_rc == 0) {
1703 		if (ahci_port_signature(ap) != ATA_PORT_T_PM) {
1704 			DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1705 			    PORTNAME(ap));
1706 			pmp_rc = EBUSY;
1707 		} else {
1708 			DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1709 			    PORTNAME(ap));
1710 		}
1711 	}
1712 
1713 	if (pmp_rc == 0) {
1714 		if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1715 			pmp_rc = EBUSY;
1716 		} else {
1717 			rc = 0;
1718 		}
1719 	}
1720 
1721 	/* if PMP detection failed, so turn off the PMA bit and
1722 	 * reset the port again
1723 	 */
1724 	if (pmp_rc != 0) {
1725 		DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1726 		    "the port\n", PORTNAME(ap));
1727 		ahci_port_stop(ap, 0);
1728 		ahci_port_clo(ap);
1729 		cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1730 		cmd &= ~AHCI_PREG_CMD_PMA;
1731 		ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1732 
1733 		ahci_pwrite(ap, AHCI_PREG_IE, 0);
1734 		ahci_port_stop(ap, 0);
1735 		if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
1736 			ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
1737 		ahci_flush_tfd(ap);
1738 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1739 
1740 		ahci_pwrite(ap, AHCI_PREG_IS, -1);
1741 
1742 		ahci_enable_interrupts(ap);
1743 
1744 		rc = pmp_rc;
1745 	}
1746 
1747 	return (rc);
1748 }
1749 
1750 void
1751 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1752     u_int32_t flags)
1753 {
1754 	flags |= len - 1;
1755 
1756 	htolem64(&prd->dba, addr);
1757 	htolem32(&prd->flags, flags);
1758 }
1759 
1760 int
1761 ahci_load_prdt(struct ahci_ccb *ccb)
1762 {
1763 	struct ahci_port		*ap = ccb->ccb_port;
1764 	struct ahci_softc		*sc = ap->ap_sc;
1765 	struct ata_xfer			*xa = &ccb->ccb_xa;
1766 	struct ahci_prdt		*prdt = ccb->ccb_cmd_table->prdt;
1767 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1768 	struct ahci_cmd_hdr		*cmd_slot = ccb->ccb_cmd_hdr;
1769 	int				i, error;
1770 
1771 	if (xa->datalen == 0) {
1772 		ccb->ccb_cmd_hdr->prdtl = 0;
1773 		return (0);
1774 	}
1775 
1776 	error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
1777 	    (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1778 	if (error != 0) {
1779 		printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1780 		return (1);
1781 	}
1782 
1783 	for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1784 		ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1785 		    dmap->dm_segs[i].ds_len, 0);
1786 	}
1787 
1788 	ahci_load_prdt_seg(&prdt[i],
1789 	    dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1790 	    ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0);
1791 
1792 	htolem16(&cmd_slot->prdtl, dmap->dm_nsegs);
1793 
1794 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1795 	    (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
1796 	    BUS_DMASYNC_PREWRITE);
1797 
1798 	return (0);
1799 }
1800 
1801 void
1802 ahci_unload_prdt(struct ahci_ccb *ccb)
1803 {
1804 	struct ahci_port		*ap = ccb->ccb_port;
1805 	struct ahci_softc		*sc = ap->ap_sc;
1806 	struct ata_xfer			*xa = &ccb->ccb_xa;
1807 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1808 
1809 	if (xa->datalen != 0) {
1810 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1811 		    (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
1812 		    BUS_DMASYNC_POSTWRITE);
1813 
1814 		bus_dmamap_unload(sc->sc_dmat, dmap);
1815 
1816 		if (ccb->ccb_xa.flags & ATA_F_NCQ)
1817 			xa->resid = 0;
1818 		else
1819 			xa->resid = xa->datalen -
1820 			    lemtoh32(&ccb->ccb_cmd_hdr->prdbc);
1821 	}
1822 }
1823 
1824 int
1825 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
1826 {
1827 	struct ahci_port		*ap = ccb->ccb_port;
1828 	int				s;
1829 
1830 	s = splbio();
1831 	ahci_start(ccb);
1832 	do {
1833 		if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
1834 		    1 << ccb->ccb_slot)) {
1835 			splx(s);
1836 			return (0);
1837 		}
1838 		if (ccb->ccb_xa.state == ATA_S_ERROR) {
1839 			DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1840 			    PORTNAME(ap), ccb->ccb_slot);
1841 			/* pretend it timed out? */
1842 			if (timeout_fn != NULL) {
1843 				timeout_fn(ccb);
1844 			}
1845 			splx(s);
1846 			return (1);
1847 		}
1848 
1849 		delay(1000);
1850 	} while (--timeout > 0);
1851 
1852 	/* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1853 	if (timeout_fn != NULL)
1854 		timeout_fn(ccb);
1855 
1856 	splx(s);
1857 
1858 	return (1);
1859 }
1860 
1861 void
1862 ahci_start(struct ahci_ccb *ccb)
1863 {
1864 	struct ahci_port		*ap = ccb->ccb_port;
1865 	struct ahci_softc		*sc = ap->ap_sc;
1866 
1867 	/* Zero transferred byte count before transfer */
1868 	ccb->ccb_cmd_hdr->prdbc = 0;
1869 
1870 	/* Sync command list entry and corresponding command table entry */
1871 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1872 	    ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
1873 	    sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
1874 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1875 	    ccb->ccb_slot * sizeof(struct ahci_cmd_table),
1876 	    sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
1877 
1878 	/* Prepare RFIS area for write by controller */
1879 	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
1880 	    sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
1881 
1882 	if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1883 		/* Issue NCQ commands only when there are no outstanding
1884 		 * standard commands. */
1885 		if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) ||
1886 		    (ap->ap_sactive != 0 &&
1887 		     ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1888 			TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1889 		} else {
1890 			KASSERT(ap->ap_active_cnt == 0);
1891 			ap->ap_sactive |= (1 << ccb->ccb_slot);
1892 			ccb->ccb_xa.state = ATA_S_ONCHIP;
1893 			ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1894 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1895 			ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1896 		}
1897 	} else {
1898 		/* Wait for all NCQ commands to finish before issuing standard
1899 		 * command. */
1900 		if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
1901 			TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1902 		else if (ap->ap_active_cnt < 2) {
1903 			ap->ap_active |= 1 << ccb->ccb_slot;
1904 			ccb->ccb_xa.state = ATA_S_ONCHIP;
1905 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1906 			ap->ap_active_cnt++;
1907 		}
1908 	}
1909 }
1910 
1911 void
1912 ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1913 {
1914 	struct ahci_ccb			*nextccb;
1915 	u_int32_t			sact_change = 0;
1916 
1917 	KASSERT(ap->ap_active_cnt == 0);
1918 
1919 	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1920 	if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
1921 		return;
1922 
1923 	/* Start all the NCQ commands at the head of the pending list.
1924 	 * If a port multiplier is attached to the port, we can only
1925 	 * issue commands for one of its ports at a time.
1926 	 */
1927 	if (ap->ap_sactive != 0 &&
1928 	    ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1929 		return;
1930 	}
1931 
1932 	ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1933 	do {
1934 		TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1935 		sact_change |= 1 << nextccb->ccb_slot;
1936 		nextccb->ccb_xa.state = ATA_S_ONCHIP;
1937 		nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1938 	} while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) &&
1939 	    (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1940 
1941 	ap->ap_sactive |= sact_change;
1942 	ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
1943 	ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
1944 }
1945 
1946 void
1947 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1948 {
1949 	struct ahci_ccb			*nextccb;
1950 
1951 	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1952 	if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
1953 		if (last_was_ncq) {
1954 			KASSERT(nextccb->ccb_xa.pmp_port !=
1955 			    ap->ap_pmp_ncq_port);
1956 			/* otherwise it should have been started already */
1957 		} else {
1958 			ap->ap_active_cnt--;
1959 		}
1960 
1961 		/* Issue NCQ commands only when there are no outstanding
1962 		 * standard commands, and previous NCQ commands for other
1963 		 * PMP ports have finished.
1964 		 */
1965 		if (ap->ap_active == 0)
1966 			ahci_issue_pending_ncq_commands(ap);
1967 		else
1968 			KASSERT(ap->ap_active_cnt == 1);
1969 	} else if (nextccb) {
1970 		if (ap->ap_sactive != 0 || last_was_ncq)
1971 			KASSERT(ap->ap_active_cnt == 0);
1972 
1973 		/* Wait for all NCQ commands to finish before issuing standard
1974 		 * command. */
1975 		if (ap->ap_sactive != 0)
1976 			return;
1977 
1978 		/* Keep up to 2 standard commands on-chip at a time. */
1979 		do {
1980 			TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1981 			ap->ap_active |= 1 << nextccb->ccb_slot;
1982 			nextccb->ccb_xa.state = ATA_S_ONCHIP;
1983 			ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1984 			if (last_was_ncq)
1985 				ap->ap_active_cnt++;
1986 			if (ap->ap_active_cnt == 2)
1987 				break;
1988 			KASSERT(ap->ap_active_cnt == 1);
1989 			nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1990 		} while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
1991 	} else if (!last_was_ncq) {
1992 		KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
1993 
1994 		/* Standard command finished, none waiting to start. */
1995 		ap->ap_active_cnt--;
1996 	} else {
1997 		KASSERT(ap->ap_active_cnt == 0);
1998 
1999 		/* NCQ command finished. */
2000 	}
2001 }
2002 
2003 int
2004 ahci_intr(void *arg)
2005 {
2006 	struct ahci_softc		*sc = arg;
2007 	u_int32_t			is, ack = 0;
2008 	int				port;
2009 
2010 	/* Read global interrupt status */
2011 	is = ahci_read(sc, AHCI_REG_IS);
2012 	if (is == 0 || is == 0xffffffff)
2013 		return (0);
2014 	ack = is;
2015 
2016 #ifdef AHCI_COALESCE
2017 	/* Check coalescing interrupt first */
2018 	if (is & sc->sc_ccc_mask) {
2019 		DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2020 		    DEVNAME(sc));
2021 		is &= ~sc->sc_ccc_mask;
2022 		is |= sc->sc_ccc_ports_cur;
2023 	}
2024 #endif
2025 
2026 	/* Process interrupts for each port */
2027 	while (is) {
2028 		port = ffs(is) - 1;
2029 		if (sc->sc_ports[port])
2030 			ahci_port_intr(sc->sc_ports[port],
2031 			    AHCI_PREG_CI_ALL_SLOTS);
2032 		is &= ~(1 << port);
2033 	}
2034 
2035 	/* Finally, acknowledge global interrupt */
2036 	ahci_write(sc, AHCI_REG_IS, ack);
2037 
2038 	return (1);
2039 }
2040 
2041 u_int32_t
2042 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2043 {
2044 	struct ahci_softc		*sc = ap->ap_sc;
2045 	u_int32_t			is, ci_saved, ci_masked, processed = 0;
2046 	int				slot, need_restart = 0;
2047 	int				process_error = 0;
2048 	struct ahci_ccb			*ccb;
2049 	volatile u_int32_t		*active;
2050 #ifdef DIAGNOSTIC
2051 	u_int32_t			tmp;
2052 #endif
2053 
2054 	is = ahci_pread(ap, AHCI_PREG_IS);
2055 
2056 	/* Ack port interrupt only if checking all command slots. */
2057 	if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
2058 		ahci_pwrite(ap, AHCI_PREG_IS, is);
2059 
2060 	if (is)
2061 		DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2062 		    is, AHCI_PFMT_IS);
2063 
2064 	if (ap->ap_sactive) {
2065 		/* Active NCQ commands - use SActive instead of CI */
2066 		KASSERT(ap->ap_active == 0);
2067 		KASSERT(ap->ap_active_cnt == 0);
2068 		ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
2069 		active = &ap->ap_sactive;
2070 	} else {
2071 		/* Save CI */
2072 		ci_saved = ahci_pread(ap, AHCI_PREG_CI);
2073 		active = &ap->ap_active;
2074 	}
2075 
2076 	if (is & AHCI_PREG_IS_TFES) {
2077 		process_error = 1;
2078 	} else if (is & AHCI_PREG_IS_DHRS) {
2079 		u_int32_t tfd;
2080 		u_int32_t cmd;
2081 		u_int32_t serr;
2082 
2083 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
2084 		cmd = ahci_pread(ap, AHCI_PREG_CMD);
2085 		serr = ahci_pread(ap, AHCI_PREG_SERR);
2086 		if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
2087 		    (cmd & AHCI_PREG_CMD_CR) == 0) {
2088 			DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2089 			    " %b, DIAG: %b\n", PORTNAME(ap), tfd,
2090 			    AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2091 			    AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2092 			    AHCI_PFMT_SERR_DIAG);
2093 			process_error = 1;
2094 		} else {
2095 			/* rfis copy back is in the normal execution path */
2096 			ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
2097 		}
2098 	}
2099 
2100 	/* Command failed.  See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2101 	if (process_error) {
2102 		u_int32_t		tfd, serr;
2103 		int			err_slot;
2104 
2105 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
2106 		serr = ahci_pread(ap, AHCI_PREG_SERR);
2107 
2108 		if (ap->ap_sactive == 0) {
2109 			/* Errored slot is easy to determine from CMD. */
2110 			err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
2111 			    AHCI_PREG_CMD));
2112 
2113 			if ((ci_saved & (1 << err_slot)) == 0) {
2114 				/*
2115 				 * Hardware doesn't seem to report correct
2116 				 * slot number. If there's only one
2117 				 * outstanding command we can cope,
2118 				 * otherwise fail all active commands.
2119 				 */
2120 				if (ap->ap_active_cnt == 1)
2121 					err_slot = ffs(ap->ap_active) - 1;
2122 				else
2123 					goto failall;
2124 			}
2125 
2126 			ccb = &ap->ap_ccbs[err_slot];
2127 
2128 			/* Preserve received taskfile data from the RFIS. */
2129 			memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
2130 			    sizeof(struct ata_fis_d2h));
2131 		} else
2132 			err_slot = -1;	/* Must extract error from log page */
2133 
2134 		DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2135 		    " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2136 		    AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2137 		    AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2138 		    AHCI_PFMT_SERR_DIAG);
2139 
2140 		/* Turn off ST to clear CI and SACT. */
2141 		ahci_port_stop(ap, 0);
2142 		need_restart = 1;
2143 
2144 		/* Clear SERR to enable capturing new errors. */
2145 		ahci_pwrite(ap, AHCI_PREG_SERR, serr);
2146 
2147 		/* Acknowledge the interrupts we can recover from. */
2148 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
2149 		    AHCI_PREG_IS_IFS);
2150 		is = ahci_pread(ap, AHCI_PREG_IS);
2151 
2152 		/* If device hasn't cleared its busy status, try to idle it. */
2153 		if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
2154 
2155 			if ((ap->ap_state == AP_S_PMP_PORT_PROBE) ||
2156 			    (ap->ap_state == AP_S_ERROR_RECOVERY)) {
2157 				/* can't reset the port here, just make sure
2158 				 * the operation fails and the port still works.
2159 				 */
2160 			} else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2161 				printf("%s: error on PMP port %d, idling "
2162 				    "device\n", PORTNAME(ap),
2163 				    ccb->ccb_xa.pmp_port);
2164 				if (ahci_pmp_port_softreset(ap,
2165 				        ccb->ccb_xa.pmp_port) == 0) {
2166 					printf("%s: unable to softreset port "
2167 					    "%d\n", PORTNAME(ap),
2168 					    ccb->ccb_xa.pmp_port);
2169 					if (ahci_pmp_port_portreset(ap,
2170 						ccb->ccb_xa.pmp_port)) {
2171 						printf("%s: failed to port "
2172 						    " reset %d, giving up on "
2173 						    "it\n", PORTNAME(ap),
2174 						    ccb->ccb_xa.pmp_port);
2175 						goto fatal;
2176 					}
2177 				}
2178 			} else {
2179 				printf("%s: attempting to idle device\n",
2180 				    PORTNAME(ap));
2181 				if (ahci_port_softreset(ap)) {
2182 					printf("%s: failed to soft reset "
2183 					    "device\n", PORTNAME(ap));
2184 					if (ahci_port_portreset(ap, 0)) {
2185 						printf("%s: failed to port "
2186 						    "reset device, give up on "
2187 						    "it\n", PORTNAME(ap));
2188 						goto fatal;
2189 					}
2190 				}
2191 			}
2192 
2193 			/* Had to reset device, can't gather extended info. */
2194 		} else if (ap->ap_sactive) {
2195 			/* Recover the NCQ error from log page 10h.
2196 			 * We can only have queued commands active for one port
2197 			 * at a time, so we know which device errored.
2198 			 */
2199 			ahci_port_read_ncq_error(ap, &err_slot,
2200 			    ap->ap_pmp_ncq_port);
2201 			if (err_slot < 0)
2202 				goto failall;
2203 
2204 			DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2205 				PORTNAME(ap), err_slot);
2206 
2207 			ccb = &ap->ap_ccbs[err_slot];
2208 			if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2209 				printf("%s: NCQ errored slot %d is idle"
2210 				    " (%08x active)\n", PORTNAME(ap), err_slot,
2211 				    ci_saved);
2212 				goto failall;
2213 			}
2214 		} else {
2215 			/* Didn't reset, could gather extended info from log. */
2216 		}
2217 
2218 		/*
2219 		 * If we couldn't determine the errored slot, reset the port
2220 		 * and fail all the active slots.
2221 		 */
2222 		if (err_slot == -1) {
2223 			if (ahci_port_softreset(ap) != 0 &&
2224 			    ahci_port_portreset(ap, 0) != 0) {
2225 				printf("%s: couldn't reset after NCQ error, "
2226 				    "disabling device.\n", PORTNAME(ap));
2227 				goto fatal;
2228 			}
2229 			printf("%s: couldn't recover NCQ error, failing "
2230 			    "all outstanding commands.\n", PORTNAME(ap));
2231 			goto failall;
2232 		}
2233 
2234 		/* Clear the failed command in saved CI so completion runs. */
2235 		ci_saved &= ~(1 << err_slot);
2236 
2237 		/* Note the error in the ata_xfer. */
2238 		KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2239 		ccb->ccb_xa.state = ATA_S_ERROR;
2240 
2241 #ifdef DIAGNOSTIC
2242 		/* There may only be one outstanding standard command now. */
2243 		if (ap->ap_sactive == 0) {
2244 			tmp = ci_saved;
2245 			if (tmp) {
2246 				slot = ffs(tmp) - 1;
2247 				tmp &= ~(1 << slot);
2248 				KASSERT(tmp == 0);
2249 			}
2250 		}
2251 #endif
2252 	}
2253 
2254 	/* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2255 	 * when there's a normal SATA device attached.
2256 	 */
2257 	if ((ap->ap_state == AP_S_PMP_PROBE) &&
2258 	    (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) &&
2259 	    (is & AHCI_PREG_IS_IPMS)) {
2260 		slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD));
2261 		DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2262 		    PORTNAME(ap), slot);
2263 
2264 		ccb = &ap->ap_ccbs[slot];
2265 		ccb->ccb_xa.state = ATA_S_ERROR;
2266 
2267 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS);
2268 		is &= ~AHCI_PREG_IS_IPMS;
2269 	}
2270 
2271 	/* ignore IFS errors while resetting a PMP port */
2272 	if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) {
2273 		DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2274 		    "port\n", PORTNAME(ap));
2275 
2276 		need_restart = 1;
2277 		ahci_pwrite(ap, AHCI_PREG_SERR, -1);
2278 		ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
2279 		is &= ~AHCI_PREG_IS_IFS;
2280 		goto failall;
2281 	}
2282 
2283 	/* Check for remaining errors - they are fatal. */
2284 	if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
2285 	    AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
2286 		printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2287 		    PORTNAME(ap), is, AHCI_PFMT_IS);
2288 
2289 		/* XXX try recovery first */
2290 		goto fatal;
2291 	}
2292 
2293 	/* Fail all outstanding commands if we know the port won't recover. */
2294 	if (ap->ap_state == AP_S_FATAL_ERROR) {
2295 fatal:
2296 		ap->ap_state = AP_S_FATAL_ERROR;
2297 failall:
2298 
2299 		/* Ensure port is shut down. */
2300 		ahci_port_stop(ap, 1);
2301 
2302 		/* Error all the active slots. */
2303 		ci_masked = ci_saved & *active;
2304 		while (ci_masked) {
2305 			slot = ffs(ci_masked) - 1;
2306 			ccb = &ap->ap_ccbs[slot];
2307 			ci_masked &= ~(1 << slot);
2308 			ccb->ccb_xa.state = ATA_S_ERROR;
2309 		}
2310 
2311 		/* Run completion for all active slots. */
2312 		ci_saved &= ~*active;
2313 
2314 		/* Don't restart the port if our problems were deemed fatal. */
2315 		if (ap->ap_state == AP_S_FATAL_ERROR)
2316 			need_restart = 0;
2317 	}
2318 
2319 	/*
2320 	 * CCB completion is detected by noticing its slot's bit in CI has
2321 	 * changed to zero some time after we activated it.
2322 	 * If we are polling, we may only be interested in particular slot(s).
2323 	 */
2324 	ci_masked = ~ci_saved & *active & ci_mask;
2325 	while (ci_masked) {
2326 		slot = ffs(ci_masked) - 1;
2327 		ccb = &ap->ap_ccbs[slot];
2328 		ci_masked &= ~(1 << slot);
2329 
2330 		DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2331 		    PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2332 		    " (error)" : "");
2333 
2334 		bus_dmamap_sync(sc->sc_dmat,
2335 		    AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
2336 		    ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
2337 		    sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
2338 
2339 		bus_dmamap_sync(sc->sc_dmat,
2340 		    AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
2341 		    ccb->ccb_slot * sizeof(struct ahci_cmd_table),
2342 		    sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
2343 
2344 		bus_dmamap_sync(sc->sc_dmat,
2345 		    AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
2346 		    sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
2347 
2348 		*active &= ~(1 << ccb->ccb_slot);
2349 		/* Copy the rfis into the ccb if we were asked for it */
2350 		if (ccb->ccb_xa.state == ATA_S_ONCHIP &&
2351 		    ccb->ccb_xa.flags & ATA_F_GET_RFIS) {
2352 			memcpy(&ccb->ccb_xa.rfis,
2353 			       ap->ap_rfis->rfis,
2354 			       sizeof(struct ata_fis_d2h));
2355 		}
2356 
2357 		ccb->ccb_done(ccb);
2358 
2359 		processed |= 1 << ccb->ccb_slot;
2360 	}
2361 
2362 	if (need_restart) {
2363 		/* Restart command DMA on the port */
2364 		ahci_port_start(ap, 0);
2365 
2366 		/* Re-enable outstanding commands on port. */
2367 		if (ci_saved) {
2368 #ifdef DIAGNOSTIC
2369 			tmp = ci_saved;
2370 			while (tmp) {
2371 				slot = ffs(tmp) - 1;
2372 				tmp &= ~(1 << slot);
2373 				ccb = &ap->ap_ccbs[slot];
2374 				KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2375 				KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
2376 				    (!!ap->ap_sactive));
2377 			}
2378 #endif
2379 			DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2380 			    "re-enabling%s slots %08x\n", PORTNAME(ap),
2381 			    ap->ap_sactive ? " NCQ" : "", ci_saved);
2382 
2383 			if (ap->ap_sactive)
2384 				ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
2385 			ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
2386 		}
2387 	}
2388 
2389 	return (processed);
2390 }
2391 
2392 struct ahci_ccb *
2393 ahci_get_ccb(struct ahci_port *ap)
2394 {
2395 	struct ahci_ccb			*ccb;
2396 
2397 	mtx_enter(&ap->ap_ccb_mtx);
2398 	ccb = TAILQ_FIRST(&ap->ap_ccb_free);
2399 	if (ccb != NULL) {
2400 		KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2401 		TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2402 		ccb->ccb_xa.state = ATA_S_SETUP;
2403 	}
2404 	mtx_leave(&ap->ap_ccb_mtx);
2405 
2406 	return (ccb);
2407 }
2408 
2409 void
2410 ahci_put_ccb(struct ahci_ccb *ccb)
2411 {
2412 	struct ahci_port		*ap = ccb->ccb_port;
2413 
2414 #ifdef DIAGNOSTIC
2415 	if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
2416 	    ccb->ccb_xa.state != ATA_S_TIMEOUT &&
2417 	    ccb->ccb_xa.state != ATA_S_ERROR) {
2418 		printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2419 		    "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
2420 		    ccb->ccb_slot);
2421 	}
2422 #endif
2423 
2424 	ccb->ccb_xa.state = ATA_S_PUT;
2425 	mtx_enter(&ap->ap_ccb_mtx);
2426 	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2427 	mtx_leave(&ap->ap_ccb_mtx);
2428 }
2429 
2430 struct ahci_ccb *
2431 ahci_get_err_ccb(struct ahci_port *ap)
2432 {
2433 	struct ahci_ccb *err_ccb;
2434 	u_int32_t sact;
2435 
2436 	splassert(IPL_BIO);
2437 
2438 	/* No commands may be active on the chip. */
2439 	sact = ahci_pread(ap, AHCI_PREG_SACT);
2440 	if (sact != 0)
2441 		printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2442 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2443 
2444 #ifdef DIAGNOSTIC
2445 	KASSERT(ap->ap_err_busy == 0);
2446 	ap->ap_err_busy = 1;
2447 #endif
2448 	/* Save outstanding command state. */
2449 	ap->ap_err_saved_active = ap->ap_active;
2450 	ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2451 	ap->ap_err_saved_sactive = ap->ap_sactive;
2452 
2453 	/*
2454 	 * Pretend we have no commands outstanding, so that completions won't
2455 	 * run prematurely.
2456 	 */
2457 	ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2458 
2459 	/*
2460 	 * Grab a CCB to use for error recovery.  This should never fail, as
2461 	 * we ask atascsi to reserve one for us at init time.
2462 	 */
2463 	err_ccb = ap->ap_ccb_err;
2464 	err_ccb->ccb_xa.flags = 0;
2465 	err_ccb->ccb_xa.state = ATA_S_SETUP;
2466 	err_ccb->ccb_done = ahci_empty_done;
2467 
2468 	return (err_ccb);
2469 }
2470 
2471 void
2472 ahci_put_err_ccb(struct ahci_ccb *ccb)
2473 {
2474 	struct ahci_port *ap = ccb->ccb_port;
2475 	u_int32_t sact;
2476 
2477 	splassert(IPL_BIO);
2478 
2479 #ifdef DIAGNOSTIC
2480 	KASSERT(ap->ap_err_busy);
2481 #endif
2482 	/* No commands may be active on the chip */
2483 	sact = ahci_pread(ap, AHCI_PREG_SACT);
2484 	if (sact != 0)
2485 		printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
2486 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2487 
2488 #ifdef DIAGNOSTIC
2489 	/* Done with the CCB */
2490 	KASSERT(ccb == ap->ap_ccb_err);
2491 #endif
2492 
2493 	/* Restore outstanding command state */
2494 	ap->ap_sactive = ap->ap_err_saved_sactive;
2495 	ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2496 	ap->ap_active = ap->ap_err_saved_active;
2497 
2498 #ifdef DIAGNOSTIC
2499 	ap->ap_err_busy = 0;
2500 #endif
2501 }
2502 
2503 struct ahci_ccb *
2504 ahci_get_pmp_ccb(struct ahci_port *ap)
2505 {
2506 	struct ahci_ccb *ccb;
2507 	u_int32_t sact;
2508 
2509 	/* some PMP commands need to be issued on slot 1,
2510 	 * particularly the command that clears SRST and
2511 	 * fetches the device signature.
2512 	 *
2513 	 * ensure the chip is idle and ccb 1 is available.
2514 	 */
2515 	splassert(IPL_BIO);
2516 
2517 	sact = ahci_pread(ap, AHCI_PREG_SACT);
2518 	if (sact != 0)
2519 		printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2520 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2521 
2522 	ccb = &ap->ap_ccbs[1];
2523 	KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2524 	ccb->ccb_xa.flags = 0;
2525 	ccb->ccb_done = ahci_pmp_cmd_done;
2526 
2527 	mtx_enter(&ap->ap_ccb_mtx);
2528 	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2529 	mtx_leave(&ap->ap_ccb_mtx);
2530 
2531 	return ccb;
2532 }
2533 
2534 void
2535 ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2536 {
2537 	struct ahci_port *ap = ccb->ccb_port;
2538 	u_int32_t sact;
2539 
2540 	/* make sure this is the right ccb */
2541 	KASSERT(ccb == &ap->ap_ccbs[1]);
2542 
2543 	/* No commands may be active on the chip */
2544 	sact = ahci_pread(ap, AHCI_PREG_SACT);
2545 	if (sact != 0)
2546 		printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
2547 	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2548 
2549 	ccb->ccb_xa.state = ATA_S_PUT;
2550 	mtx_enter(&ap->ap_ccb_mtx);
2551 	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2552 	mtx_leave(&ap->ap_ccb_mtx);
2553 }
2554 
2555 int
2556 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
2557 {
2558 	struct ahci_ccb			*ccb;
2559 	struct ahci_cmd_hdr		*cmd_slot;
2560 	u_int32_t			cmd;
2561 	struct ata_fis_h2d		*fis;
2562 	int				rc = EIO, oldstate;
2563 
2564 	DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2565 	oldstate = ap->ap_state;
2566 	ap->ap_state = AP_S_ERROR_RECOVERY;
2567 
2568 	/* Save command register state. */
2569 	cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
2570 
2571 	/* Port should have been idled already.  Start it. */
2572 	KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
2573 	ahci_port_start(ap, 0);
2574 
2575 	/* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2576 	ccb = ahci_get_err_ccb(ap);
2577 	ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
2578 	ccb->ccb_xa.data = ap->ap_err_scratch;
2579 	ccb->ccb_xa.datalen = 512;
2580 	cmd_slot = ccb->ccb_cmd_hdr;
2581 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
2582 
2583 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2584 	fis->type = ATA_FIS_TYPE_H2D;
2585 	fis->flags = ATA_H2D_FLAGS_CMD | pmp_port;
2586 	fis->command = ATA_C_READ_LOG_EXT;
2587 	fis->lba_low = 0x10;		/* queued error log page (10h) */
2588 	fis->sector_count = 1;		/* number of sectors (1) */
2589 	fis->sector_count_exp = 0;
2590 	fis->lba_mid = 0;		/* starting offset */
2591 	fis->lba_mid_exp = 0;
2592 	fis->device = 0;
2593 
2594 	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
2595 	    (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
2596 
2597 	if (ahci_load_prdt(ccb) != 0) {
2598 		rc = ENOMEM;	/* XXX caller must abort all commands */
2599 		goto err;
2600 	}
2601 
2602 	ccb->ccb_xa.state = ATA_S_PENDING;
2603 	if (ahci_poll(ccb, 1000, NULL) != 0 ||
2604 	    ccb->ccb_xa.state == ATA_S_ERROR)
2605 		goto err;
2606 
2607 	rc = 0;
2608 err:
2609 	/* Abort our command, if it failed, by stopping command DMA. */
2610 	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
2611 		printf("%s: log page read failed, slot %d was still active.\n",
2612 		    PORTNAME(ap), ccb->ccb_slot);
2613 		ahci_port_stop(ap, 0);
2614 	}
2615 
2616 	/* Done with the error CCB now. */
2617 	ahci_unload_prdt(ccb);
2618 	ahci_put_err_ccb(ccb);
2619 
2620 	/* Extract failed register set and tags from the scratch space. */
2621 	if (rc == 0) {
2622 		struct ata_log_page_10h		*log;
2623 		int				err_slot;
2624 
2625 		log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2626 		if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
2627 			/* Not queued bit was set - wasn't an NCQ error? */
2628 			printf("%s: read NCQ error page, but not an NCQ "
2629 			    "error?\n", PORTNAME(ap));
2630 			rc = ESRCH;
2631 		} else {
2632 			/* Copy back the log record as a D2H register FIS. */
2633 			*err_slotp = err_slot = log->err_regs.type &
2634 			    ATA_LOG_10H_TYPE_TAG_MASK;
2635 
2636 			ccb = &ap->ap_ccbs[err_slot];
2637 			memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2638 			    sizeof(struct ata_fis_d2h));
2639 			ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2640 			ccb->ccb_xa.rfis.flags = 0;
2641 		}
2642 	}
2643 
2644 	/* Restore saved CMD register state */
2645 	ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2646 	ap->ap_state = oldstate;
2647 
2648 	return (rc);
2649 }
2650 
2651 struct ahci_dmamem *
2652 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2653 {
2654 	struct ahci_dmamem		*adm;
2655 	int				nsegs;
2656 
2657 	adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO);
2658 	if (adm == NULL)
2659 		return (NULL);
2660 
2661 	adm->adm_size = size;
2662 
2663 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2664 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
2665 		goto admfree;
2666 
2667 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
2668 	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
2669 		goto destroy;
2670 
2671 	if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
2672 	    &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0)
2673 		goto free;
2674 
2675 	if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
2676 	    NULL, BUS_DMA_NOWAIT) != 0)
2677 		goto unmap;
2678 
2679 	return (adm);
2680 
2681 unmap:
2682 	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
2683 free:
2684 	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2685 destroy:
2686 	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2687 admfree:
2688 	free(adm, M_DEVBUF, sizeof(*adm));
2689 
2690 	return (NULL);
2691 }
2692 
2693 void
2694 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2695 {
2696 	bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
2697 	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
2698 	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2699 	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2700 	free(adm, M_DEVBUF, sizeof(*adm));
2701 }
2702 
2703 u_int32_t
2704 ahci_read(struct ahci_softc *sc, bus_size_t r)
2705 {
2706 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2707 	    BUS_SPACE_BARRIER_READ);
2708 	return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2709 }
2710 
2711 void
2712 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2713 {
2714 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2715 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2716 	    BUS_SPACE_BARRIER_WRITE);
2717 }
2718 
2719 int
2720 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2721     u_int32_t target)
2722 {
2723 	int				i;
2724 
2725 	for (i = 0; i < 1000; i++) {
2726 		if ((ahci_read(sc, r) & mask) != target)
2727 			return (0);
2728 		delay(1000);
2729 	}
2730 
2731 	return (1);
2732 }
2733 
2734 u_int32_t
2735 ahci_pread(struct ahci_port *ap, bus_size_t r)
2736 {
2737 	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2738 	    BUS_SPACE_BARRIER_READ);
2739 	return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
2740 }
2741 
2742 void
2743 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2744 {
2745 	bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
2746 	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2747 	    BUS_SPACE_BARRIER_WRITE);
2748 }
2749 
2750 int
2751 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2752     u_int32_t target, int n)
2753 {
2754 	int				i;
2755 
2756 	for (i = 0; i < n * 1000; i++) {
2757 		if ((ahci_pread(ap, r) & mask) == target)
2758 			return (0);
2759 		delay(1000);
2760 	}
2761 
2762 	return (1);
2763 }
2764 
2765 int
2766 ahci_ata_probe(void *xsc, int port, int lun)
2767 {
2768 	struct ahci_softc		*sc = xsc;
2769 	struct ahci_port		*ap = sc->sc_ports[port];
2770 
2771 	if (ap == NULL)
2772 		return (ATA_PORT_T_NONE);
2773 
2774 	if (lun != 0) {
2775 		int pmp_port = lun - 1;
2776 		if (pmp_port >= ap->ap_pmp_ports) {
2777 			return (ATA_PORT_T_NONE);
2778 		}
2779 		return (ahci_pmp_port_probe(ap, pmp_port));
2780 	} else {
2781 		return (ahci_port_signature(ap));
2782 	}
2783 }
2784 
2785 void
2786 ahci_ata_free(void *xsc, int port, int lun)
2787 {
2788 
2789 }
2790 
2791 struct ata_xfer *
2792 ahci_ata_get_xfer(void *aaa_cookie, int port)
2793 {
2794 	struct ahci_softc		*sc = aaa_cookie;
2795 	struct ahci_port		*ap = sc->sc_ports[port];
2796 	struct ahci_ccb			*ccb;
2797 
2798 	ccb = ahci_get_ccb(ap);
2799 	if (ccb == NULL) {
2800 		DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2801 		    PORTNAME(ap));
2802 		return (NULL);
2803 	}
2804 
2805 	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2806 	    PORTNAME(ap), ccb->ccb_slot);
2807 
2808 	return ((struct ata_xfer *)ccb);
2809 }
2810 
2811 void
2812 ahci_ata_put_xfer(struct ata_xfer *xa)
2813 {
2814 	struct ahci_ccb			*ccb = (struct ahci_ccb *)xa;
2815 
2816 	DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2817 
2818 	ahci_put_ccb(ccb);
2819 }
2820 
2821 void
2822 ahci_ata_cmd(struct ata_xfer *xa)
2823 {
2824 	struct ahci_ccb			*ccb = (struct ahci_ccb *)xa;
2825 	struct ahci_cmd_hdr		*cmd_slot;
2826 	int				s;
2827 	u_int16_t			flags;
2828 
2829 	if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
2830 		goto failcmd;
2831 
2832 	ccb->ccb_done = ahci_ata_cmd_done;
2833 
2834 	cmd_slot = ccb->ccb_cmd_hdr;
2835 	flags = 5 /* FIS length (in DWORDs) */;
2836 	flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
2837 
2838 	if (xa->flags & ATA_F_WRITE)
2839 		flags |= AHCI_CMD_LIST_FLAG_W;
2840 
2841 	if (xa->flags & ATA_F_PACKET)
2842 		flags |= AHCI_CMD_LIST_FLAG_A;
2843 
2844 	htolem16(&cmd_slot->flags, flags);
2845 
2846 	if (ahci_load_prdt(ccb) != 0)
2847 		goto failcmd;
2848 
2849 	timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2850 
2851 	xa->state = ATA_S_PENDING;
2852 
2853 	if (xa->flags & ATA_F_POLL)
2854 		ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2855 	else {
2856 		s = splbio();
2857 		timeout_add_msec(&xa->stimeout, xa->timeout);
2858 		ahci_start(ccb);
2859 		splx(s);
2860 	}
2861 
2862 	return;
2863 
2864 failcmd:
2865 	s = splbio();
2866 	xa->state = ATA_S_ERROR;
2867 	ata_complete(xa);
2868 	splx(s);
2869 }
2870 
2871 void
2872 ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2873 {
2874 	struct ata_xfer			*xa = &ccb->ccb_xa;
2875 
2876 	if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2877 		ahci_issue_pending_commands(ccb->ccb_port,
2878 		    xa->flags & ATA_F_NCQ);
2879 
2880 	xa->state = ATA_S_COMPLETE;
2881 }
2882 
2883 
2884 void
2885 ahci_ata_cmd_done(struct ahci_ccb *ccb)
2886 {
2887 	struct ata_xfer			*xa = &ccb->ccb_xa;
2888 
2889 	timeout_del(&xa->stimeout);
2890 
2891 	if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2892 		ahci_issue_pending_commands(ccb->ccb_port,
2893 		    xa->flags & ATA_F_NCQ);
2894 
2895 	ahci_unload_prdt(ccb);
2896 
2897 	if (xa->state == ATA_S_ONCHIP)
2898 		xa->state = ATA_S_COMPLETE;
2899 #ifdef DIAGNOSTIC
2900 	else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
2901 		printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2902 		    "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
2903 		    ccb->ccb_slot);
2904 #endif
2905 	if (xa->state != ATA_S_TIMEOUT)
2906 		ata_complete(xa);
2907 }
2908 
2909 void
2910 ahci_ata_cmd_timeout(void *arg)
2911 {
2912 	struct ahci_ccb			*ccb = arg;
2913 	struct ata_xfer			*xa = &ccb->ccb_xa;
2914 	struct ahci_port		*ap = ccb->ccb_port;
2915 	int				s, ccb_was_started, ncq_cmd;
2916 	volatile u_int32_t		*active;
2917 
2918 	s = splbio();
2919 
2920 	ncq_cmd = (xa->flags & ATA_F_NCQ);
2921 	active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2922 
2923 	if (ccb->ccb_xa.state == ATA_S_PENDING) {
2924 		DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2925 		    "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2926 		TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
2927 		ccb_was_started = 0;
2928 	} else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
2929 	    1 << ccb->ccb_slot)) {
2930 		DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2931 		    "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2932 		goto ret;
2933 	} else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2934 		DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2935 		    "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2936 		    ISSET(*active, 1 << ccb->ccb_slot) ?
2937 		    " but slot is still active?" : ".");
2938 		goto ret;
2939 	} else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
2940 	    AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
2941 	    1 << ccb->ccb_slot)) {
2942 		DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2943 		    "IRQ handler didn't detect it.  Why?\n", PORTNAME(ap),
2944 		    ccb->ccb_slot);
2945 		*active &= ~(1 << ccb->ccb_slot);
2946 		ccb->ccb_done(ccb);
2947 		goto ret;
2948 	} else {
2949 		ccb_was_started = 1;
2950 	}
2951 
2952 	/* Complete the slot with a timeout error. */
2953 	ccb->ccb_xa.state = ATA_S_TIMEOUT;
2954 	*active &= ~(1 << ccb->ccb_slot);
2955 	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2956 	ccb->ccb_done(ccb);	/* This won't issue pending commands or run the
2957 				   atascsi completion. */
2958 
2959 	/* Reset port to abort running command. */
2960 	if (ccb_was_started) {
2961 		DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2962 		    "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2963 		    ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2964 		if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2965 		    != 0) {
2966 			printf("%s: failed to reset port during timeout "
2967 			    "handling, disabling it\n", PORTNAME(ap));
2968 			ap->ap_state = AP_S_FATAL_ERROR;
2969 		}
2970 
2971 		/* Restart any other commands that were aborted by the reset. */
2972 		if (*active) {
2973 			DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2974 			    "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2975 			    *active);
2976 			if (ncq_cmd)
2977 				ahci_pwrite(ap, AHCI_PREG_SACT, *active);
2978 			ahci_pwrite(ap, AHCI_PREG_CI, *active);
2979 		}
2980 	}
2981 
2982 	/* Issue any pending commands now. */
2983 	DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2984 	if (ccb_was_started)
2985 		ahci_issue_pending_commands(ap, ncq_cmd);
2986 	else if (ap->ap_active == 0)
2987 		ahci_issue_pending_ncq_commands(ap);
2988 
2989 	/* Complete the timed out ata_xfer I/O (may generate new I/O). */
2990 	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2991 	ata_complete(xa);
2992 
2993 	DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2994 ret:
2995 	splx(s);
2996 }
2997 
2998 void
2999 ahci_empty_done(struct ahci_ccb *ccb)
3000 {
3001 	if (ccb->ccb_xa.state != ATA_S_ERROR)
3002 		ccb->ccb_xa.state = ATA_S_COMPLETE;
3003 }
3004 
3005 int
3006 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
3007 {
3008 	struct ahci_ccb	*ccb;
3009 	struct ata_fis_h2d *fis;
3010 	int error;
3011 
3012 	ccb = ahci_get_pmp_ccb(ap);
3013 	if (ccb == NULL) {
3014 		printf("%s: NULL ccb!\n", PORTNAME(ap));
3015 		return (1);
3016 	}
3017 	ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS;
3018 	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3019 	ccb->ccb_xa.state = ATA_S_PENDING;
3020 
3021 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3022 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3023 	fis->type = ATA_FIS_TYPE_H2D;
3024 	fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3025 	fis->command = ATA_C_READ_PM;
3026 	fis->features = which;
3027 	fis->device = target | ATA_H2D_DEVICE_LBA;
3028 	fis->control = ATA_FIS_CONTROL_4BIT;
3029 
3030 	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3031 		error = 1;
3032 	} else {
3033 		*datap = ccb->ccb_xa.rfis.sector_count |
3034 		    (ccb->ccb_xa.rfis.lba_low << 8) |
3035 		    (ccb->ccb_xa.rfis.lba_mid << 16) |
3036 		    (ccb->ccb_xa.rfis.lba_high << 24);
3037 		error = 0;
3038 	}
3039 	ahci_put_pmp_ccb(ccb);
3040 	return (error);
3041 }
3042 
3043 int
3044 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3045 {
3046 	struct ahci_ccb	*ccb;
3047 	struct ata_fis_h2d *fis;
3048 	int error;
3049 
3050 	ccb = ahci_get_pmp_ccb(ap);
3051 	if (ccb == NULL) {
3052 		printf("%s: NULL ccb!\n", PORTNAME(ap));
3053 		return (1);
3054 	}
3055 	ccb->ccb_xa.flags = ATA_F_POLL;
3056 	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3057 	ccb->ccb_xa.state = ATA_S_PENDING;
3058 
3059 	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3060 	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3061 	fis->type = ATA_FIS_TYPE_H2D;
3062 	fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3063 	fis->command = ATA_C_WRITE_PM;
3064 	fis->features = which;
3065 	fis->device = target | ATA_H2D_DEVICE_LBA;
3066 	fis->sector_count = (u_int8_t)data;
3067 	fis->lba_low = (u_int8_t)(data >> 8);
3068 	fis->lba_mid = (u_int8_t)(data >> 16);
3069 	fis->lba_high = (u_int8_t)(data >> 24);
3070 	fis->control = ATA_FIS_CONTROL_4BIT;
3071 
3072 	error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3073 	ahci_put_pmp_ccb(ccb);
3074 	return (error);
3075 }
3076 
3077 int
3078 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
3079 {
3080 	int error;
3081 
3082 	error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap);
3083 	if (error == 0)
3084 		error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1);
3085 	if (error)
3086 		*datap = 0;
3087 
3088 	return (error);
3089 }
3090 
3091 int
3092 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
3093 {
3094 	u_int32_t chipid;
3095 	u_int32_t rev;
3096 	u_int32_t nports;
3097 	u_int32_t features;
3098 	u_int32_t enabled;
3099 	int s;
3100 
3101 	s = splbio();
3102 
3103 	if (ahci_pmp_read(ap, 15, 0, &chipid) ||
3104 	    ahci_pmp_read(ap, 15, 1, &rev) ||
3105 	    ahci_pmp_read(ap, 15, 2, &nports) ||
3106 	    ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) ||
3107 	    ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) {
3108 		printf("%s: port multiplier identification failed\n",
3109 		    PORTNAME(ap));
3110 		splx(s);
3111 		return (1);
3112 	}
3113 	splx(s);
3114 
3115 	nports &= 0x0F;
3116 
3117 	/* ignore SEMB port on SiI3726 port multiplier chips */
3118 	if (chipid == 0x37261095) {
3119 		nports--;
3120 	}
3121 
3122 	printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3123 	    "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev,
3124 	    SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled,
3125 	    SATA_PFMT_PM_FEA);
3126 
3127 	*ret_nports = nports;
3128 	return (0);
3129 }
3130 
3131 
3132 #ifdef HIBERNATE
3133 void
3134 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
3135 {
3136 	ccb->ccb_cmd_hdr->prdbc = 0;
3137 	ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
3138 }
3139 
3140 int
3141 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
3142 {
3143 	u_int32_t			is, ci_saved;
3144 	int				process_error = 0;
3145 
3146 	is = ahci_pread(ap, AHCI_PREG_IS);
3147 
3148 	ci_saved = ahci_pread(ap, AHCI_PREG_CI);
3149 
3150 	if (is & AHCI_PREG_IS_DHRS) {
3151 		u_int32_t tfd;
3152 		u_int32_t cmd;
3153 		u_int32_t serr;
3154 
3155 		tfd = ahci_pread(ap, AHCI_PREG_TFD);
3156 		cmd = ahci_pread(ap, AHCI_PREG_CMD);
3157 		serr = ahci_pread(ap, AHCI_PREG_SERR);
3158 		if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
3159 		    (cmd & AHCI_PREG_CMD_CR) == 0) {
3160 			process_error = 1;
3161 		} else {
3162 			ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
3163 		}
3164 	} else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
3165 	    AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
3166 		process_error = 1;
3167 	}
3168 
3169 	/* Command failed.  See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3170 	if (process_error) {
3171 
3172 		/* Turn off ST to clear CI and SACT. */
3173 		ahci_port_stop(ap, 0);
3174 
3175 		/* just return an error indicator?  we can't meaningfully
3176 		 * recover, and on the way back out we'll DVACT_RESUME which
3177 		 * resets and reinits the port.
3178 		 */
3179 		return (EIO);
3180 	}
3181 
3182 	/* command is finished when the bit in CI for the slot goes to 0 */
3183 	if (ci_saved & (1 << ccb->ccb_slot)) {
3184 		return (EAGAIN);
3185 	}
3186 
3187 	return (0);
3188 }
3189 
3190 void
3191 ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3192 {
3193 	struct ata_xfer			*xa = &ccb->ccb_xa;
3194 	struct ahci_prdt		*prdt = ccb->ccb_cmd_table->prdt, *prd;
3195 	struct ahci_cmd_hdr		*cmd_slot = ccb->ccb_cmd_hdr;
3196 	int				i;
3197 	paddr_t				data_phys;
3198 	u_int64_t			data_bus_phys;
3199 	vaddr_t				data_addr;
3200 	size_t				seglen;
3201 	size_t				buflen;
3202 
3203 	if (xa->datalen == 0) {
3204 		ccb->ccb_cmd_hdr->prdtl = 0;
3205 		return;
3206 	}
3207 
3208 	/* derived from i386/amd64 _bus_dma_load_buffer;
3209 	 * for amd64 the buffer will always be dma safe.
3210 	 */
3211 
3212 	buflen = xa->datalen;
3213 	data_addr = (vaddr_t)xa->data;
3214 	for (i = 0; buflen > 0; i++) {
3215 		prd = &prdt[i];
3216 
3217 		pmap_extract(pmap_kernel(), data_addr, &data_phys);
3218 		data_bus_phys = data_phys;
3219 
3220 		seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET);
3221 		if (buflen < seglen)
3222 			seglen = buflen;
3223 
3224 		ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3225 
3226 		data_addr += seglen;
3227 		buflen -= seglen;
3228 	}
3229 
3230 	htolem16(&cmd_slot->prdtl, i);
3231 }
3232 
3233 int
3234 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3235     int op, void *page)
3236 {
3237 	/* we use the 'real' ahci_port and ahci_softc here, but
3238 	 * never write to them
3239 	 */
3240 	struct {
3241 		struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3242 		struct ahci_rfis rfis;		 /* 1k aligned, 256 bytes */
3243 		/* cmd table isn't actually used because of mysteries */
3244 		struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3245 		struct ahci_port *ap;
3246 		struct ahci_ccb ccb_buf;
3247 		struct ahci_ccb *ccb;
3248 		struct ahci_cmd_hdr *hdr_buf;
3249 		int pmp_port;
3250 		daddr_t poffset;
3251 		size_t psize;
3252 	} *my = page;
3253 	struct ata_fis_h2d *fis;
3254 	u_int32_t sector_count;
3255 	struct ahci_cmd_hdr *cmd_slot;
3256 	int rc;
3257 	int timeout;
3258 	u_int16_t flags;
3259 
3260 	if (op == HIB_INIT) {
3261 		struct device *disk;
3262 		struct device *scsibus;
3263 		struct ahci_softc *sc;
3264 		extern struct cfdriver sd_cd;
3265 		struct scsi_link *link;
3266 		struct scsibus_softc *bus_sc;
3267 		int port;
3268 		paddr_t page_phys;
3269 		u_int64_t item_phys;
3270 		u_int32_t cmd;
3271 
3272 		my->poffset = blkno;
3273 		my->psize = size;
3274 
3275 		/* map dev to an ahci port */
3276 		disk = disk_lookup(&sd_cd, DISKUNIT(dev));
3277 		scsibus = disk->dv_parent;
3278 		sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3279 
3280 		/* find the scsi_link for the device, which has the port */
3281 		port = -1;
3282 		bus_sc = (struct scsibus_softc *)scsibus;
3283 		SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) {
3284 			if (link->device_softc == disk) {
3285 				/* link->adapter_softc == sc->sc_atascsi */
3286 				port = link->target;
3287 				if (link->lun > 0)
3288 					my->pmp_port = link->lun - 1;
3289 				else
3290 					my->pmp_port = 0;
3291 
3292 				break;
3293 			}
3294 		}
3295 		if (port == -1) {
3296 			/* don't know where the disk is */
3297 			return (EIO);
3298 		}
3299 
3300 		my->ap = sc->sc_ports[port];
3301 
3302 		/* we're going to use the first command slot,
3303 		 * so ensure it's not already in use
3304 		 */
3305 		if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) {
3306 			/* this shouldn't happen, we should be idle */
3307 			return (EIO);
3308 		}
3309 
3310 		/* stop the port so we can relocate to the hibernate page */
3311 		if (ahci_port_stop(my->ap, 1)) {
3312 			return (EIO);
3313 		}
3314 		ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0);
3315 
3316 		pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
3317 
3318 		/* Setup RFIS base address */
3319 		item_phys = page_phys + ((void *)&my->rfis - page);
3320 		ahci_pwrite(my->ap, AHCI_PREG_FBU,
3321 		    (u_int32_t)(item_phys >> 32));
3322 		ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys);
3323 
3324 		/* Enable FIS reception and activate port. */
3325 		cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3326 		cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD |
3327 		    AHCI_PREG_CMD_SUD;
3328 		ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd |
3329 		    AHCI_PREG_CMD_ICC_ACTIVE);
3330 
3331 		/* Check whether port activated.  */
3332 		cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3333 		if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
3334 			return (EIO);
3335 		}
3336 
3337 		/* Set up the single CCB */
3338 		my->ccb = &my->ccb_buf;
3339 		my->ccb->ccb_slot = 0;
3340 		my->ccb->ccb_port = my->ap;
3341 
3342 		/* Setup command list base address */
3343 		item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3344 		ahci_pwrite(my->ap, AHCI_PREG_CLBU,
3345 		    (u_int32_t)(item_phys >> 32));
3346 		ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys);
3347 
3348 		my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3349 
3350 		/* use existing cmd table - don't know why moving to a new one fails */
3351 		my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3352 		pmap_extract(pmap_kernel(),
3353 		    (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table),
3354 		    &page_phys);
3355 		item_phys = page_phys;
3356 #if 0
3357 		/* use cmd table in hibernate page (doesn't work) */
3358 		my->ccb->ccb_cmd_table = &my->cmd_table;
3359 		item_phys = page_phys + ((void *)&my->cmd_table - page);
3360 #endif
3361 		htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys);
3362 
3363 		my->ccb->ccb_xa.fis =
3364 		    (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3365 		my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3366 		my->ccb->ccb_xa.tag = 0;
3367 
3368 		/* Wait for ICC change to complete */
3369 		ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
3370 
3371 		if (ahci_port_start(my->ap, 0)) {
3372 			return (EIO);
3373 		}
3374 
3375 		/* Flush interrupts for port */
3376 		ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap,
3377 		    AHCI_PREG_IS));
3378 		ahci_write(sc, AHCI_REG_IS, 1 << port);
3379 
3380 		ahci_enable_interrupts(my->ap);
3381 		return (0);
3382 	} else if (op == HIB_DONE) {
3383 		ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME);
3384 		return (0);
3385 	}
3386 
3387 	if (blkno > my->psize)
3388 		return (E2BIG);
3389 	blkno += my->poffset;
3390 
3391 	/* build fis */
3392 	sector_count = size / 512;	/* dlg promises this is okay */
3393 	my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ;
3394 	fis = my->ccb->ccb_xa.fis;
3395 	fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port;
3396 	fis->lba_low = blkno & 0xff;
3397 	fis->lba_mid = (blkno >> 8) & 0xff;
3398 	fis->lba_high = (blkno >> 16) & 0xff;
3399 
3400 	if (sector_count > 0x100 || blkno > 0xfffffff) {
3401 		/* Use LBA48 */
3402 		fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT :
3403 		    ATA_C_READDMA_EXT;
3404 		fis->device = ATA_H2D_DEVICE_LBA;
3405 		fis->lba_low_exp = (blkno >> 24) & 0xff;
3406 		fis->lba_mid_exp = (blkno >> 32) & 0xff;
3407 		fis->lba_high_exp = (blkno >> 40) & 0xff;
3408 		fis->sector_count = sector_count & 0xff;
3409 		fis->sector_count_exp = (sector_count >> 8) & 0xff;
3410 	} else {
3411 		/* Use LBA */
3412 		fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA;
3413 		fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f);
3414 		fis->sector_count = sector_count & 0xff;
3415 	}
3416 
3417 	my->ccb->ccb_xa.data = (void *)addr;
3418 	my->ccb->ccb_xa.datalen = size;
3419 	my->ccb->ccb_xa.pmp_port = my->pmp_port;
3420 	my->ccb->ccb_xa.flags |= ATA_F_POLL;
3421 
3422 	cmd_slot = my->ccb->ccb_cmd_hdr;
3423 	flags = 5; /* FIS length (in DWORDs) */
3424 	flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
3425 
3426 	if (op == HIB_W)
3427 		flags |= AHCI_CMD_LIST_FLAG_W;
3428 
3429 	htolem16(&cmd_slot->flags, flags);
3430 
3431 	ahci_hibernate_load_prdt(my->ccb);
3432 
3433 	ahci_hibernate_io_start(my->ap, my->ccb);
3434 	timeout = 1000000;
3435 	while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) {
3436 		delay(1);
3437 		timeout--;
3438 		if (timeout == 0) {
3439 			return (EIO);
3440 		}
3441 	}
3442 
3443 	return (0);
3444 }
3445 
3446 #endif
3447