xref: /openbsd-src/sys/dev/ic/mpi.c (revision 34213ea4bc89f28dfc7c298fa6fea44ae2d93e3e)
1 /*	$OpenBSD: mpi.c,v 1.96 2008/09/30 23:40:16 dlg Exp $ */
2 
3 /*
4  * Copyright (c) 2005, 2006 David Gwynne <dlg@openbsd.org>
5  * Copyright (c) 2005 Marco Peereboom <marco@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/param.h>
21 #include <sys/systm.h>
22 #include <sys/buf.h>
23 #include <sys/device.h>
24 #include <sys/proc.h>
25 #include <sys/malloc.h>
26 #include <sys/kernel.h>
27 
28 #include <machine/bus.h>
29 
30 #include <scsi/scsi_all.h>
31 #include <scsi/scsiconf.h>
32 
33 #include <dev/ic/mpireg.h>
34 #include <dev/ic/mpivar.h>
35 
36 #ifdef MPI_DEBUG
37 uint32_t	mpi_debug = 0
38 /*		    | MPI_D_CMD */
39 /*		    | MPI_D_INTR */
40 /*		    | MPI_D_MISC */
41 /*		    | MPI_D_DMA */
42 /*		    | MPI_D_IOCTL */
43 /*		    | MPI_D_RW */
44 /*		    | MPI_D_MEM */
45 /*		    | MPI_D_CCB */
46 /*		    | MPI_D_PPR */
47 /*		    | MPI_D_RAID */
48 /*		    | MPI_D_EVT */
49 		;
50 #endif
51 
52 struct cfdriver mpi_cd = {
53 	NULL,
54 	"mpi",
55 	DV_DULL
56 };
57 
58 int			mpi_scsi_cmd(struct scsi_xfer *);
59 void			mpi_scsi_cmd_done(struct mpi_ccb *);
60 void			mpi_minphys(struct buf *bp);
61 int			mpi_scsi_probe(struct scsi_link *);
62 int			mpi_scsi_ioctl(struct scsi_link *, u_long, caddr_t,
63 			    int, struct proc *);
64 
65 struct scsi_adapter mpi_switch = {
66 	mpi_scsi_cmd,
67 	mpi_minphys,
68 	mpi_scsi_probe,
69 	NULL,
70 	mpi_scsi_ioctl
71 };
72 
73 struct scsi_device mpi_dev = {
74 	NULL,
75 	NULL,
76 	NULL,
77 	NULL
78 };
79 
80 struct mpi_dmamem	*mpi_dmamem_alloc(struct mpi_softc *, size_t);
81 void			mpi_dmamem_free(struct mpi_softc *,
82 			    struct mpi_dmamem *);
83 int			mpi_alloc_ccbs(struct mpi_softc *);
84 struct mpi_ccb		*mpi_get_ccb(struct mpi_softc *);
85 void			mpi_put_ccb(struct mpi_softc *, struct mpi_ccb *);
86 int			mpi_alloc_replies(struct mpi_softc *);
87 void			mpi_push_replies(struct mpi_softc *);
88 
89 void			mpi_start(struct mpi_softc *, struct mpi_ccb *);
90 int			mpi_complete(struct mpi_softc *, struct mpi_ccb *, int);
91 int			mpi_poll(struct mpi_softc *, struct mpi_ccb *, int);
92 int			mpi_reply(struct mpi_softc *, u_int32_t);
93 
94 int			mpi_cfg_spi_port(struct mpi_softc *);
95 void			mpi_squash_ppr(struct mpi_softc *);
96 void			mpi_run_ppr(struct mpi_softc *);
97 int			mpi_ppr(struct mpi_softc *, struct scsi_link *,
98 			    struct mpi_cfg_raid_physdisk *, int, int, int);
99 int			mpi_inq(struct mpi_softc *, u_int16_t, int);
100 
101 void			mpi_timeout_xs(void *);
102 int			mpi_load_xs(struct mpi_ccb *);
103 
104 u_int32_t		mpi_read(struct mpi_softc *, bus_size_t);
105 void			mpi_write(struct mpi_softc *, bus_size_t, u_int32_t);
106 int			mpi_wait_eq(struct mpi_softc *, bus_size_t, u_int32_t,
107 			    u_int32_t);
108 int			mpi_wait_ne(struct mpi_softc *, bus_size_t, u_int32_t,
109 			    u_int32_t);
110 
111 int			mpi_init(struct mpi_softc *);
112 int			mpi_reset_soft(struct mpi_softc *);
113 int			mpi_reset_hard(struct mpi_softc *);
114 
115 int			mpi_handshake_send(struct mpi_softc *, void *, size_t);
116 int			mpi_handshake_recv_dword(struct mpi_softc *,
117 			    u_int32_t *);
118 int			mpi_handshake_recv(struct mpi_softc *, void *, size_t);
119 
120 void			mpi_empty_done(struct mpi_ccb *);
121 
122 int			mpi_iocinit(struct mpi_softc *);
123 int			mpi_iocfacts(struct mpi_softc *);
124 int			mpi_portfacts(struct mpi_softc *);
125 int			mpi_portenable(struct mpi_softc *);
126 void			mpi_get_raid(struct mpi_softc *);
127 int			mpi_fwupload(struct mpi_softc *);
128 
129 int			mpi_eventnotify(struct mpi_softc *);
130 void			mpi_eventnotify_done(struct mpi_ccb *);
131 void			mpi_eventack(struct mpi_softc *,
132 			    struct mpi_msg_event_reply *);
133 void			mpi_eventack_done(struct mpi_ccb *);
134 void			mpi_evt_sas(void *, void *);
135 
136 int			mpi_req_cfg_header(struct mpi_softc *, u_int8_t,
137 			    u_int8_t, u_int32_t, int, void *);
138 int			mpi_req_cfg_page(struct mpi_softc *, u_int32_t, int,
139 			    void *, int, void *, size_t);
140 
141 #define DEVNAME(s)		((s)->sc_dev.dv_xname)
142 
143 #define	dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
144 #define	sizeofa(s)		(sizeof(s) / sizeof((s)[0]))
145 
146 #define mpi_read_db(s)		mpi_read((s), MPI_DOORBELL)
147 #define mpi_write_db(s, v)	mpi_write((s), MPI_DOORBELL, (v))
148 #define mpi_read_intr(s)	mpi_read((s), MPI_INTR_STATUS)
149 #define mpi_write_intr(s, v)	mpi_write((s), MPI_INTR_STATUS, (v))
150 #define mpi_pop_reply(s)	mpi_read((s), MPI_REPLY_QUEUE)
151 #define mpi_push_reply(s, v)	mpi_write((s), MPI_REPLY_QUEUE, (v))
152 
153 #define mpi_wait_db_int(s)	mpi_wait_ne((s), MPI_INTR_STATUS, \
154 				    MPI_INTR_STATUS_DOORBELL, 0)
155 #define mpi_wait_db_ack(s)	mpi_wait_eq((s), MPI_INTR_STATUS, \
156 				    MPI_INTR_STATUS_IOCDOORBELL, 0)
157 
158 #define mpi_cfg_header(_s, _t, _n, _a, _h) \
159 	mpi_req_cfg_header((_s), (_t), (_n), (_a), 0, (_h))
160 #define mpi_ecfg_header(_s, _t, _n, _a, _h) \
161 	mpi_req_cfg_header((_s), (_t), (_n), (_a), 1, (_h))
162 
163 #define mpi_cfg_page(_s, _a, _h, _r, _p, _l) \
164 	mpi_req_cfg_page((_s), (_a), 0, (_h), (_r), (_p), (_l))
165 #define mpi_ecfg_page(_s, _a, _h, _r, _p, _l) \
166 	mpi_req_cfg_page((_s), (_a), 1, (_h), (_r), (_p), (_l))
167 
168 int
169 mpi_attach(struct mpi_softc *sc)
170 {
171 	struct scsibus_attach_args	saa;
172 	struct mpi_ccb			*ccb;
173 
174 	printf("\n");
175 
176 	/* disable interrupts */
177 	mpi_write(sc, MPI_INTR_MASK,
178 	    MPI_INTR_MASK_REPLY | MPI_INTR_MASK_DOORBELL);
179 
180 	if (mpi_init(sc) != 0) {
181 		printf("%s: unable to initialise\n", DEVNAME(sc));
182 		return (1);
183 	}
184 
185 	if (mpi_iocfacts(sc) != 0) {
186 		printf("%s: unable to get iocfacts\n", DEVNAME(sc));
187 		return (1);
188 	}
189 
190 	if (mpi_alloc_ccbs(sc) != 0) {
191 		/* error already printed */
192 		return (1);
193 	}
194 
195 	if (mpi_alloc_replies(sc) != 0) {
196 		printf("%s: unable to allocate reply space\n", DEVNAME(sc));
197 		goto free_ccbs;
198 	}
199 
200 	if (mpi_iocinit(sc) != 0) {
201 		printf("%s: unable to send iocinit\n", DEVNAME(sc));
202 		goto free_ccbs;
203 	}
204 
205 	/* spin until we're operational */
206 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
207 	    MPI_DOORBELL_STATE_OPER) != 0) {
208 		printf("%s: state: 0x%08x\n", DEVNAME(sc),
209 		    mpi_read_db(sc) & MPI_DOORBELL_STATE);
210 		printf("%s: operational state timeout\n", DEVNAME(sc));
211 		goto free_ccbs;
212 	}
213 
214 	mpi_push_replies(sc);
215 
216 	if (mpi_portfacts(sc) != 0) {
217 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
218 		goto free_replies;
219 	}
220 
221 #ifdef notyet
222 	if (mpi_eventnotify(sc) != 0) {
223 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
224 		goto free_replies;
225 	}
226 #endif
227 
228 	if (mpi_portenable(sc) != 0) {
229 		printf("%s: unable to enable port\n", DEVNAME(sc));
230 		goto free_replies;
231 	}
232 
233 	if (mpi_fwupload(sc) != 0) {
234 		printf("%s: unable to upload firmware\n", DEVNAME(sc));
235 		goto free_replies;
236 	}
237 
238 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI) {
239 		if (mpi_cfg_spi_port(sc) != 0)
240 			goto free_replies;
241 		mpi_squash_ppr(sc);
242 	}
243 
244 	/* we should be good to go now, attach scsibus */
245 	sc->sc_link.device = &mpi_dev;
246 	sc->sc_link.adapter = &mpi_switch;
247 	sc->sc_link.adapter_softc = sc;
248 	sc->sc_link.adapter_target = sc->sc_target;
249 	sc->sc_link.adapter_buswidth = sc->sc_buswidth;
250 	sc->sc_link.openings = sc->sc_maxcmds / sc->sc_buswidth;
251 
252 	bzero(&saa, sizeof(saa));
253 	saa.saa_sc_link = &sc->sc_link;
254 
255 	/* config_found() returns the scsibus attached to us */
256 	sc->sc_scsibus = (struct scsibus_softc *) config_found(&sc->sc_dev,
257 	    &saa, scsiprint);
258 
259 	/* get raid pages */
260 	mpi_get_raid(sc);
261 
262 	/* do domain validation */
263 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI)
264 		mpi_run_ppr(sc);
265 
266 	/* enable interrupts */
267 	mpi_write(sc, MPI_INTR_MASK, MPI_INTR_MASK_DOORBELL);
268 
269 	return (0);
270 
271 free_replies:
272 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
273 	    0, PAGE_SIZE, BUS_DMASYNC_POSTREAD);
274 	mpi_dmamem_free(sc, sc->sc_replies);
275 free_ccbs:
276 	while ((ccb = mpi_get_ccb(sc)) != NULL)
277 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
278 	mpi_dmamem_free(sc, sc->sc_requests);
279 	free(sc->sc_ccbs, M_DEVBUF);
280 
281 	return(1);
282 }
283 
284 int
285 mpi_cfg_spi_port(struct mpi_softc *sc)
286 {
287 	struct mpi_cfg_hdr		hdr;
288 	struct mpi_cfg_spi_port_pg1	port;
289 
290 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 1, 0x0,
291 	    &hdr) != 0)
292 		return (1);
293 
294 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port, sizeof(port)) != 0)
295 		return (1);
296 
297 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_spi_port_pg1\n", DEVNAME(sc));
298 	DNPRINTF(MPI_D_MISC, "%s:  port_scsi_id: %d port_resp_ids 0x%04x\n",
299 	    DEVNAME(sc), port.port_scsi_id, letoh16(port.port_resp_ids));
300 	DNPRINTF(MPI_D_MISC, "%s:  on_bus_timer_value: 0x%08x\n", DEVNAME(sc),
301 	    letoh32(port.port_scsi_id));
302 	DNPRINTF(MPI_D_MISC, "%s:  target_config: 0x%02x id_config: 0x%04x\n",
303 	    DEVNAME(sc), port.target_config, letoh16(port.id_config));
304 
305 	if (port.port_scsi_id == sc->sc_target &&
306 	    port.port_resp_ids == htole16(1 << sc->sc_target) &&
307 	    port.on_bus_timer_value != htole32(0x0))
308 		return (0);
309 
310 	DNPRINTF(MPI_D_MISC, "%s: setting port scsi id to %d\n", DEVNAME(sc),
311 	    sc->sc_target);
312 	port.port_scsi_id = sc->sc_target;
313 	port.port_resp_ids = htole16(1 << sc->sc_target);
314 	port.on_bus_timer_value = htole32(0x07000000); /* XXX magic */
315 
316 	if (mpi_cfg_page(sc, 0x0, &hdr, 0, &port, sizeof(port)) != 0) {
317 		printf("%s: unable to configure port scsi id\n", DEVNAME(sc));
318 		return (1);
319 	}
320 
321 	return (0);
322 }
323 
324 void
325 mpi_squash_ppr(struct mpi_softc *sc)
326 {
327 	struct mpi_cfg_hdr		hdr;
328 	struct mpi_cfg_spi_dev_pg1	page;
329 	int				i;
330 
331 	DNPRINTF(MPI_D_PPR, "%s: mpi_squash_ppr\n", DEVNAME(sc));
332 
333 	for (i = 0; i < sc->sc_buswidth; i++) {
334 		if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV,
335 		    1, i, &hdr) != 0)
336 			return;
337 
338 		if (mpi_cfg_page(sc, i, &hdr, 1, &page, sizeof(page)) != 0)
339 			return;
340 
341 		DNPRINTF(MPI_D_PPR, "%s:  target: %d req_params1: 0x%02x "
342 		    "req_offset: 0x%02x req_period: 0x%02x "
343 		    "req_params2: 0x%02x conf: 0x%08x\n", DEVNAME(sc), i,
344 		    page.req_params1, page.req_offset, page.req_period,
345 		    page.req_params2, letoh32(page.configuration));
346 
347 		page.req_params1 = 0x0;
348 		page.req_offset = 0x0;
349 		page.req_period = 0x0;
350 		page.req_params2 = 0x0;
351 		page.configuration = htole32(0x0);
352 
353 		if (mpi_cfg_page(sc, i, &hdr, 0, &page, sizeof(page)) != 0)
354 			return;
355 	}
356 }
357 
358 void
359 mpi_run_ppr(struct mpi_softc *sc)
360 {
361 	struct mpi_cfg_hdr		hdr;
362 	struct mpi_cfg_spi_port_pg0	port_pg;
363 	struct mpi_cfg_ioc_pg3		*physdisk_pg;
364 	struct mpi_cfg_raid_physdisk	*physdisk_list, *physdisk;
365 	size_t				pagelen;
366 	struct scsi_link		*link;
367 	int				i, tries;
368 
369 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 0, 0x0,
370 	    &hdr) != 0) {
371 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch header\n",
372 		    DEVNAME(sc));
373 		return;
374 	}
375 
376 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port_pg, sizeof(port_pg)) != 0) {
377 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch page\n",
378 		    DEVNAME(sc));
379 		return;
380 	}
381 
382 	for (i = 0; i < sc->sc_buswidth; i++) {
383 		link = sc->sc_scsibus->sc_link[i][0];
384 		if (link == NULL)
385 			continue;
386 
387 		/* do not ppr volumes */
388 		if (link->flags & SDEV_VIRTUAL)
389 			continue;
390 
391 		tries = 0;
392 		while (mpi_ppr(sc, link, NULL, port_pg.min_period,
393 		    port_pg.max_offset, tries) == EAGAIN)
394 			tries++;
395 	}
396 
397 	if ((sc->sc_flags & MPI_F_RAID) == 0)
398 		return;
399 
400 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 3, 0x0,
401 	    &hdr) != 0) {
402 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
403 		    "fetch ioc pg 3 header\n", DEVNAME(sc));
404 		return;
405 	}
406 
407 	pagelen = hdr.page_length * 4; /* dwords to bytes */
408 	physdisk_pg = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
409 	if (physdisk_pg == NULL) {
410 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
411 		    "allocate ioc pg 3\n", DEVNAME(sc));
412 		return;
413 	}
414 	physdisk_list = (struct mpi_cfg_raid_physdisk *)(physdisk_pg + 1);
415 
416 	if (mpi_cfg_page(sc, 0, &hdr, 1, physdisk_pg, pagelen) != 0) {
417 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s: mpi_run_ppr unable to "
418 		    "fetch ioc page 3\n", DEVNAME(sc));
419 		goto out;
420 	}
421 
422 	DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  no_phys_disks: %d\n", DEVNAME(sc),
423 	    physdisk_pg->no_phys_disks);
424 
425 	for (i = 0; i < physdisk_pg->no_phys_disks; i++) {
426 		physdisk = &physdisk_list[i];
427 
428 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  id: %d bus: %d ioc: %d "
429 		    "num: %d\n", DEVNAME(sc), physdisk->phys_disk_id,
430 		    physdisk->phys_disk_bus, physdisk->phys_disk_ioc,
431 		    physdisk->phys_disk_num);
432 
433 		if (physdisk->phys_disk_ioc != sc->sc_ioc_number)
434 			continue;
435 
436 		tries = 0;
437 		while (mpi_ppr(sc, NULL, physdisk, port_pg.min_period,
438 		    port_pg.max_offset, tries) == EAGAIN)
439 			tries++;
440 	}
441 
442 out:
443 	free(physdisk_pg, M_TEMP);
444 }
445 
446 int
447 mpi_ppr(struct mpi_softc *sc, struct scsi_link *link,
448     struct mpi_cfg_raid_physdisk *physdisk, int period, int offset, int try)
449 {
450 	struct mpi_cfg_hdr		hdr0, hdr1;
451 	struct mpi_cfg_spi_dev_pg0	pg0;
452 	struct mpi_cfg_spi_dev_pg1	pg1;
453 	u_int32_t			address;
454 	int				id;
455 	int				raid = 0;
456 
457 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr period: %d offset: %d try: %d "
458 	    "link quirks: 0x%x\n", DEVNAME(sc), period, offset, try,
459 	    link->quirks);
460 
461 	if (try >= 3)
462 		return (EIO);
463 
464 	if (physdisk == NULL) {
465 		if ((link->inqdata.device & SID_TYPE) == T_PROCESSOR)
466 			return (EIO);
467 
468 		address = link->target;
469 		id = link->target;
470 	} else {
471 		raid = 1;
472 		address = (physdisk->phys_disk_bus << 8) |
473 		    (physdisk->phys_disk_id);
474 		id = physdisk->phys_disk_num;
475 	}
476 
477 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 0,
478 	    address, &hdr0) != 0) {
479 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 0\n",
480 		    DEVNAME(sc));
481 		return (EIO);
482 	}
483 
484 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 1,
485 	    address, &hdr1) != 0) {
486 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 1\n",
487 		    DEVNAME(sc));
488 		return (EIO);
489 	}
490 
491 #ifdef MPI_DEBUG
492 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
493 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 0\n",
494 		    DEVNAME(sc));
495 		return (EIO);
496 	}
497 
498 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
499 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
500 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
501 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
502 #endif
503 
504 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
505 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 1\n",
506 		    DEVNAME(sc));
507 		return (EIO);
508 	}
509 
510 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
511 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
512 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
513 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
514 
515 	pg1.req_params1 = 0;
516 	pg1.req_offset = offset;
517 	pg1.req_period = period;
518 	pg1.req_params2 &= ~MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH;
519 
520 	if (raid || !(link->quirks & SDEV_NOSYNC)) {
521 		pg1.req_params2 |= MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH_WIDE;
522 
523 		switch (try) {
524 		case 0: /* U320 */
525 			break;
526 		case 1: /* U160 */
527 			pg1.req_period = 0x09;
528 			break;
529 		case 2: /* U80 */
530 			pg1.req_period = 0x0a;
531 			break;
532 		}
533 
534 		if (pg1.req_period < 0x09) {
535 			/* Ultra320: enable QAS & PACKETIZED */
536 			pg1.req_params1 |= MPI_CFG_SPI_DEV_1_REQPARAMS_QAS |
537 			    MPI_CFG_SPI_DEV_1_REQPARAMS_PACKETIZED;
538 		}
539 		if (pg1.req_period < 0xa) {
540 			/* >= Ultra160: enable dual xfers */
541 			pg1.req_params1 |=
542 			    MPI_CFG_SPI_DEV_1_REQPARAMS_DUALXFERS;
543 		}
544 	}
545 
546 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
547 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
548 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
549 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
550 
551 	if (mpi_cfg_page(sc, address, &hdr1, 0, &pg1, sizeof(pg1)) != 0) {
552 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to write page 1\n",
553 		    DEVNAME(sc));
554 		return (EIO);
555 	}
556 
557 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
558 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 1\n",
559 		    DEVNAME(sc));
560 		return (EIO);
561 	}
562 
563 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
564 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
565 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
566 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
567 
568 	if (mpi_inq(sc, id, raid) != 0) {
569 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to do inquiry against "
570 		    "target %d\n", DEVNAME(sc), link->target);
571 		return (EIO);
572 	}
573 
574 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
575 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 0 after "
576 		    "inquiry\n", DEVNAME(sc));
577 		return (EIO);
578 	}
579 
580 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
581 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
582 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
583 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
584 
585 	if (!(letoh32(pg0.information) & 0x07) && (try == 0)) {
586 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U320 ppr rejected\n",
587 		    DEVNAME(sc));
588 		return (EAGAIN);
589 	}
590 
591 	if ((((letoh32(pg0.information) >> 8) & 0xff) > 0x09) && (try == 1)) {
592 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U160 ppr rejected\n",
593 		    DEVNAME(sc));
594 		return (EAGAIN);
595 	}
596 
597 	if (letoh32(pg0.information) & 0x0e) {
598 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr ppr rejected: %0x\n",
599 		    DEVNAME(sc), letoh32(pg0.information));
600 		return (EAGAIN);
601 	}
602 
603 	switch(pg0.neg_period) {
604 	case 0x08:
605 		period = 160;
606 		break;
607 	case 0x09:
608 		period = 80;
609 		break;
610 	case 0x0a:
611 		period = 40;
612 		break;
613 	case 0x0b:
614 		period = 20;
615 		break;
616 	case 0x0c:
617 		period = 10;
618 		break;
619 	default:
620 		period = 0;
621 		break;
622 	}
623 
624 	printf("%s: %s %d %s at %dMHz width %dbit offset %d "
625 	    "QAS %d DT %d IU %d\n", DEVNAME(sc), raid ? "phys disk" : "target",
626 	    id, period ? "Sync" : "Async", period,
627 	    (pg0.neg_params2 & MPI_CFG_SPI_DEV_0_NEGPARAMS_WIDTH_WIDE) ? 16 : 8,
628 	    pg0.neg_offset,
629 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_QAS) ? 1 : 0,
630 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_DUALXFERS) ? 1 : 0,
631 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_PACKETIZED) ? 1 : 0);
632 
633 	return (0);
634 }
635 
636 int
637 mpi_inq(struct mpi_softc *sc, u_int16_t target, int physdisk)
638 {
639 	struct mpi_ccb			*ccb;
640 	struct scsi_inquiry		inq;
641 	struct {
642 		struct mpi_msg_scsi_io		io;
643 		struct mpi_sge			sge;
644 		struct scsi_inquiry_data	inqbuf;
645 		struct scsi_sense_data		sense;
646 	} __packed			*bundle;
647 	struct mpi_msg_scsi_io		*io;
648 	struct mpi_sge			*sge;
649 	u_int64_t			addr;
650 
651 	DNPRINTF(MPI_D_PPR, "%s: mpi_inq\n", DEVNAME(sc));
652 
653 	bzero(&inq, sizeof(inq));
654 	inq.opcode = INQUIRY;
655 	_lto2b(sizeof(struct scsi_inquiry_data), inq.length);
656 
657 	ccb = mpi_get_ccb(sc);
658 	if (ccb == NULL)
659 		return (1);
660 
661 	ccb->ccb_done = mpi_empty_done;
662 
663 	bundle = ccb->ccb_cmd;
664 	io = &bundle->io;
665 	sge = &bundle->sge;
666 
667 	io->function = physdisk ? MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH :
668 	    MPI_FUNCTION_SCSI_IO_REQUEST;
669 	/*
670 	 * bus is always 0
671 	 * io->bus = htole16(sc->sc_bus);
672 	 */
673 	io->target_id = target;
674 
675 	io->cdb_length = sizeof(inq);
676 	io->sense_buf_len = sizeof(struct scsi_sense_data);
677 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
678 
679 	io->msg_context = htole32(ccb->ccb_id);
680 
681 	/*
682 	 * always lun 0
683 	 * io->lun[0] = htobe16(link->lun);
684 	 */
685 
686 	io->direction = MPI_SCSIIO_DIR_READ;
687 	io->tagging = MPI_SCSIIO_ATTR_NO_DISCONNECT;
688 
689 	bcopy(&inq, io->cdb, sizeof(inq));
690 
691 	io->data_length = htole32(sizeof(struct scsi_inquiry_data));
692 
693 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
694 	    ((u_int8_t *)&bundle->sense - (u_int8_t *)bundle));
695 
696 	sge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64 |
697 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
698 	    (u_int32_t)sizeof(inq));
699 
700 	addr = ccb->ccb_cmd_dva +
701 	    ((u_int8_t *)&bundle->inqbuf - (u_int8_t *)bundle);
702 	sge->sg_hi_addr = htole32((u_int32_t)(addr >> 32));
703 	sge->sg_lo_addr = htole32((u_int32_t)addr);
704 
705 	if (mpi_poll(sc, ccb, 5000) != 0)
706 		return (1);
707 
708 	if (ccb->ccb_rcb != NULL)
709 		mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
710 
711 	mpi_put_ccb(sc, ccb);
712 
713 	return (0);
714 }
715 
716 void
717 mpi_detach(struct mpi_softc *sc)
718 {
719 
720 }
721 
722 int
723 mpi_intr(void *arg)
724 {
725 	struct mpi_softc		*sc = arg;
726 	u_int32_t			reg;
727 	int				rv = 0;
728 
729 	while ((reg = mpi_pop_reply(sc)) != 0xffffffff) {
730 		mpi_reply(sc, reg);
731 		rv = 1;
732 	}
733 
734 	return (rv);
735 }
736 
737 int
738 mpi_reply(struct mpi_softc *sc, u_int32_t reg)
739 {
740 	struct mpi_ccb			*ccb;
741 	struct mpi_rcb			*rcb = NULL;
742 	struct mpi_msg_reply		*reply = NULL;
743 	u_int32_t			reply_dva;
744 	int				id;
745 	int				i;
746 
747 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply reg: 0x%08x\n", DEVNAME(sc), reg);
748 
749 	if (reg & MPI_REPLY_QUEUE_ADDRESS) {
750 		bus_dmamap_sync(sc->sc_dmat,
751 		    MPI_DMA_MAP(sc->sc_replies), 0, PAGE_SIZE,
752 		    BUS_DMASYNC_POSTREAD);
753 
754 		reply_dva = (reg & MPI_REPLY_QUEUE_ADDRESS_MASK) << 1;
755 
756 		i = (reply_dva - (u_int32_t)MPI_DMA_DVA(sc->sc_replies)) /
757 		    MPI_REPLY_SIZE;
758 		rcb = &sc->sc_rcbs[i];
759 		reply = rcb->rcb_reply;
760 
761 		id = letoh32(reply->msg_context);
762 
763 		bus_dmamap_sync(sc->sc_dmat,
764 		    MPI_DMA_MAP(sc->sc_replies), 0, PAGE_SIZE,
765 		    BUS_DMASYNC_PREREAD);
766 	} else {
767 		switch (reg & MPI_REPLY_QUEUE_TYPE_MASK) {
768 		case MPI_REPLY_QUEUE_TYPE_INIT:
769 			id = reg & MPI_REPLY_QUEUE_CONTEXT;
770 			break;
771 
772 		default:
773 			panic("%s: unsupported context reply\n",
774 			    DEVNAME(sc));
775 		}
776 	}
777 
778 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply id: %d reply: %p\n",
779 	    DEVNAME(sc), id, reply);
780 
781 	ccb = &sc->sc_ccbs[id];
782 
783 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
784 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
785 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
786 	ccb->ccb_state = MPI_CCB_READY;
787 	ccb->ccb_rcb = rcb;
788 
789 	ccb->ccb_done(ccb);
790 
791 	return (id);
792 }
793 
794 struct mpi_dmamem *
795 mpi_dmamem_alloc(struct mpi_softc *sc, size_t size)
796 {
797 	struct mpi_dmamem		*mdm;
798 	int				nsegs;
799 
800 	mdm = malloc(sizeof(struct mpi_dmamem), M_DEVBUF, M_NOWAIT | M_ZERO);
801 	if (mdm == NULL)
802 		return (NULL);
803 
804 	mdm->mdm_size = size;
805 
806 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
807 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mdm->mdm_map) != 0)
808 		goto mdmfree;
809 
810 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &mdm->mdm_seg,
811 	    1, &nsegs, BUS_DMA_NOWAIT) != 0)
812 		goto destroy;
813 
814 	if (bus_dmamem_map(sc->sc_dmat, &mdm->mdm_seg, nsegs, size,
815 	    &mdm->mdm_kva, BUS_DMA_NOWAIT) != 0)
816 		goto free;
817 
818 	if (bus_dmamap_load(sc->sc_dmat, mdm->mdm_map, mdm->mdm_kva, size,
819 	    NULL, BUS_DMA_NOWAIT) != 0)
820 		goto unmap;
821 
822 	bzero(mdm->mdm_kva, size);
823 
824 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_alloc size: %d mdm: %#x "
825 	    "map: %#x nsegs: %d segs: %#x kva: %x\n",
826 	    DEVNAME(sc), size, mdm->mdm_map, nsegs, mdm->mdm_seg, mdm->mdm_kva);
827 
828 	return (mdm);
829 
830 unmap:
831 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, size);
832 free:
833 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
834 destroy:
835 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
836 mdmfree:
837 	free(mdm, M_DEVBUF);
838 
839 	return (NULL);
840 }
841 
842 void
843 mpi_dmamem_free(struct mpi_softc *sc, struct mpi_dmamem *mdm)
844 {
845 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_free %#x\n", DEVNAME(sc), mdm);
846 
847 	bus_dmamap_unload(sc->sc_dmat, mdm->mdm_map);
848 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, mdm->mdm_size);
849 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
850 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
851 	free(mdm, M_DEVBUF);
852 }
853 
854 int
855 mpi_alloc_ccbs(struct mpi_softc *sc)
856 {
857 	struct mpi_ccb			*ccb;
858 	u_int8_t			*cmd;
859 	int				i;
860 
861 	TAILQ_INIT(&sc->sc_ccb_free);
862 
863 	sc->sc_ccbs = malloc(sizeof(struct mpi_ccb) * sc->sc_maxcmds,
864 	    M_DEVBUF, M_WAITOK | M_CANFAIL | M_ZERO);
865 	if (sc->sc_ccbs == NULL) {
866 		printf("%s: unable to allocate ccbs\n", DEVNAME(sc));
867 		return (1);
868 	}
869 
870 	sc->sc_requests = mpi_dmamem_alloc(sc,
871 	    MPI_REQUEST_SIZE * sc->sc_maxcmds);
872 	if (sc->sc_requests == NULL) {
873 		printf("%s: unable to allocate ccb dmamem\n", DEVNAME(sc));
874 		goto free_ccbs;
875 	}
876 	cmd = MPI_DMA_KVA(sc->sc_requests);
877 	bzero(cmd, MPI_REQUEST_SIZE * sc->sc_maxcmds);
878 
879 	for (i = 0; i < sc->sc_maxcmds; i++) {
880 		ccb = &sc->sc_ccbs[i];
881 
882 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
883 		    sc->sc_max_sgl_len, MAXPHYS, 0,
884 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
885 		    &ccb->ccb_dmamap) != 0) {
886 			printf("%s: unable to create dma map\n", DEVNAME(sc));
887 			goto free_maps;
888 		}
889 
890 		ccb->ccb_sc = sc;
891 		ccb->ccb_id = i;
892 		ccb->ccb_offset = MPI_REQUEST_SIZE * i;
893 
894 		ccb->ccb_cmd = &cmd[ccb->ccb_offset];
895 		ccb->ccb_cmd_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_requests) +
896 		    ccb->ccb_offset;
897 
898 		DNPRINTF(MPI_D_CCB, "%s: mpi_alloc_ccbs(%d) ccb: %#x map: %#x "
899 		    "sc: %#x id: %#x offs: %#x cmd: %#x dva: %#x\n",
900 		    DEVNAME(sc), i, ccb, ccb->ccb_dmamap, ccb->ccb_sc,
901 		    ccb->ccb_id, ccb->ccb_offset, ccb->ccb_cmd,
902 		    ccb->ccb_cmd_dva);
903 
904 		mpi_put_ccb(sc, ccb);
905 	}
906 
907 	return (0);
908 
909 free_maps:
910 	while ((ccb = mpi_get_ccb(sc)) != NULL)
911 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
912 
913 	mpi_dmamem_free(sc, sc->sc_requests);
914 free_ccbs:
915 	free(sc->sc_ccbs, M_DEVBUF);
916 
917 	return (1);
918 }
919 
920 struct mpi_ccb *
921 mpi_get_ccb(struct mpi_softc *sc)
922 {
923 	struct mpi_ccb			*ccb;
924 
925 	ccb = TAILQ_FIRST(&sc->sc_ccb_free);
926 	if (ccb == NULL) {
927 		DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb == NULL\n", DEVNAME(sc));
928 		return (NULL);
929 	}
930 
931 	TAILQ_REMOVE(&sc->sc_ccb_free, ccb, ccb_link);
932 
933 	ccb->ccb_state = MPI_CCB_READY;
934 
935 	DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb %#x\n", DEVNAME(sc), ccb);
936 
937 	return (ccb);
938 }
939 
940 void
941 mpi_put_ccb(struct mpi_softc *sc, struct mpi_ccb *ccb)
942 {
943 	DNPRINTF(MPI_D_CCB, "%s: mpi_put_ccb %#x\n", DEVNAME(sc), ccb);
944 
945 	ccb->ccb_state = MPI_CCB_FREE;
946 	ccb->ccb_xs = NULL;
947 	ccb->ccb_done = NULL;
948 	bzero(ccb->ccb_cmd, MPI_REQUEST_SIZE);
949 	TAILQ_INSERT_TAIL(&sc->sc_ccb_free, ccb, ccb_link);
950 }
951 
952 int
953 mpi_alloc_replies(struct mpi_softc *sc)
954 {
955 	DNPRINTF(MPI_D_MISC, "%s: mpi_alloc_replies\n", DEVNAME(sc));
956 
957 	sc->sc_rcbs = malloc(MPI_REPLY_COUNT * sizeof(struct mpi_rcb),
958 	    M_DEVBUF, M_WAITOK|M_CANFAIL);
959 	if (sc->sc_rcbs == NULL)
960 		return (1);
961 
962 	sc->sc_replies = mpi_dmamem_alloc(sc, PAGE_SIZE);
963 	if (sc->sc_replies == NULL) {
964 		free(sc->sc_rcbs, M_DEVBUF);
965 		return (1);
966 	}
967 
968 	return (0);
969 }
970 
971 void
972 mpi_push_replies(struct mpi_softc *sc)
973 {
974 	struct mpi_rcb			*rcb;
975 	char				*kva = MPI_DMA_KVA(sc->sc_replies);
976 	int				i;
977 
978 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
979 	    0, PAGE_SIZE, BUS_DMASYNC_PREREAD);
980 
981 	for (i = 0; i < MPI_REPLY_COUNT; i++) {
982 		rcb = &sc->sc_rcbs[i];
983 
984 		rcb->rcb_reply = kva + MPI_REPLY_SIZE * i;
985 		rcb->rcb_reply_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_replies) +
986 		    MPI_REPLY_SIZE * i;
987 		mpi_push_reply(sc, rcb->rcb_reply_dva);
988 	}
989 }
990 
991 void
992 mpi_start(struct mpi_softc *sc, struct mpi_ccb *ccb)
993 {
994 	DNPRINTF(MPI_D_RW, "%s: mpi_start %#x\n", DEVNAME(sc),
995 	    ccb->ccb_cmd_dva);
996 
997 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
998 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
999 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1000 
1001 	ccb->ccb_state = MPI_CCB_QUEUED;
1002 	mpi_write(sc, MPI_REQ_QUEUE, ccb->ccb_cmd_dva);
1003 }
1004 
1005 int
1006 mpi_complete(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1007 {
1008 	u_int32_t			reg;
1009 	int				id = -1;
1010 
1011 	DNPRINTF(MPI_D_INTR, "%s: mpi_complete timeout %d\n", DEVNAME(sc),
1012 	    timeout);
1013 
1014 	do {
1015 		reg = mpi_pop_reply(sc);
1016 		if (reg == 0xffffffff) {
1017 			if (timeout-- == 0)
1018 				return (1);
1019 
1020 			delay(1000);
1021 			continue;
1022 		}
1023 
1024 		id = mpi_reply(sc, reg);
1025 
1026 	} while (ccb->ccb_id != id);
1027 
1028 	return (0);
1029 }
1030 
1031 int
1032 mpi_poll(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1033 {
1034 	int				error;
1035 	int				s;
1036 
1037 	DNPRINTF(MPI_D_CMD, "%s: mpi_poll\n", DEVNAME(sc));
1038 
1039 	s = splbio();
1040 	mpi_start(sc, ccb);
1041 	error = mpi_complete(sc, ccb, timeout);
1042 	splx(s);
1043 
1044 	return (error);
1045 }
1046 
1047 int
1048 mpi_scsi_cmd(struct scsi_xfer *xs)
1049 {
1050 	struct scsi_link		*link = xs->sc_link;
1051 	struct mpi_softc		*sc = link->adapter_softc;
1052 	struct mpi_ccb			*ccb;
1053 	struct mpi_ccb_bundle		*mcb;
1054 	struct mpi_msg_scsi_io		*io;
1055 	int				s;
1056 
1057 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd\n", DEVNAME(sc));
1058 
1059 	if (xs->cmdlen > MPI_CDB_LEN) {
1060 		DNPRINTF(MPI_D_CMD, "%s: CBD too big %d\n",
1061 		    DEVNAME(sc), xs->cmdlen);
1062 		bzero(&xs->sense, sizeof(xs->sense));
1063 		xs->sense.error_code = SSD_ERRCODE_VALID | 0x70;
1064 		xs->sense.flags = SKEY_ILLEGAL_REQUEST;
1065 		xs->sense.add_sense_code = 0x20;
1066 		xs->error = XS_SENSE;
1067 		s = splbio();
1068 		scsi_done(xs);
1069 		splx(s);
1070 		return (COMPLETE);
1071 	}
1072 
1073 	s = splbio();
1074 	ccb = mpi_get_ccb(sc);
1075 	splx(s);
1076 	if (ccb == NULL) {
1077 		xs->error = XS_DRIVER_STUFFUP;
1078 		s = splbio();
1079 		scsi_done(xs);
1080 		splx(s);
1081 		return (COMPLETE);
1082 	}
1083 	DNPRINTF(MPI_D_CMD, "%s: ccb_id: %d xs->flags: 0x%x\n",
1084 	    DEVNAME(sc), ccb->ccb_id, xs->flags);
1085 
1086 	ccb->ccb_xs = xs;
1087 	ccb->ccb_done = mpi_scsi_cmd_done;
1088 
1089 	mcb = ccb->ccb_cmd;
1090 	io = &mcb->mcb_io;
1091 
1092 	io->function = MPI_FUNCTION_SCSI_IO_REQUEST;
1093 	/*
1094 	 * bus is always 0
1095 	 * io->bus = htole16(sc->sc_bus);
1096 	 */
1097 	io->target_id = link->target;
1098 
1099 	io->cdb_length = xs->cmdlen;
1100 	io->sense_buf_len = sizeof(xs->sense);
1101 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
1102 
1103 	io->msg_context = htole32(ccb->ccb_id);
1104 
1105 	io->lun[0] = htobe16(link->lun);
1106 
1107 	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
1108 	case SCSI_DATA_IN:
1109 		io->direction = MPI_SCSIIO_DIR_READ;
1110 		break;
1111 	case SCSI_DATA_OUT:
1112 		io->direction = MPI_SCSIIO_DIR_WRITE;
1113 		break;
1114 	default:
1115 		io->direction = MPI_SCSIIO_DIR_NONE;
1116 		break;
1117 	}
1118 
1119 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SCSI &&
1120 	    (link->quirks & SDEV_NOTAGS))
1121 		io->tagging = MPI_SCSIIO_ATTR_UNTAGGED;
1122 	else
1123 		io->tagging = MPI_SCSIIO_ATTR_SIMPLE_Q;
1124 
1125 	bcopy(xs->cmd, io->cdb, xs->cmdlen);
1126 
1127 	io->data_length = htole32(xs->datalen);
1128 
1129 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
1130 	    ((u_int8_t *)&mcb->mcb_sense - (u_int8_t *)mcb));
1131 
1132 	if (mpi_load_xs(ccb) != 0) {
1133 		xs->error = XS_DRIVER_STUFFUP;
1134 		s = splbio();
1135 		mpi_put_ccb(sc, ccb);
1136 		scsi_done(xs);
1137 		splx(s);
1138 		return (COMPLETE);
1139 	}
1140 
1141 	timeout_set(&xs->stimeout, mpi_timeout_xs, ccb);
1142 
1143 	if (xs->flags & SCSI_POLL) {
1144 		if (mpi_poll(sc, ccb, xs->timeout) != 0)
1145 			xs->error = XS_DRIVER_STUFFUP;
1146 		return (COMPLETE);
1147 	}
1148 
1149 	s = splbio();
1150 	mpi_start(sc, ccb);
1151 	splx(s);
1152 	return (SUCCESSFULLY_QUEUED);
1153 }
1154 
1155 void
1156 mpi_scsi_cmd_done(struct mpi_ccb *ccb)
1157 {
1158 	struct mpi_softc		*sc = ccb->ccb_sc;
1159 	struct scsi_xfer		*xs = ccb->ccb_xs;
1160 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1161 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1162 	struct mpi_msg_scsi_io_error	*sie;
1163 
1164 	if (xs->datalen != 0) {
1165 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1166 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1167 		    BUS_DMASYNC_POSTWRITE);
1168 
1169 		bus_dmamap_unload(sc->sc_dmat, dmap);
1170 	}
1171 
1172 	/* timeout_del */
1173 	xs->error = XS_NOERROR;
1174 	xs->resid = 0;
1175 	xs->flags |= ITSDONE;
1176 
1177 	if (ccb->ccb_rcb == NULL) {
1178 		/* no scsi error, we're ok so drop out early */
1179 		xs->status = SCSI_OK;
1180 		mpi_put_ccb(sc, ccb);
1181 		scsi_done(xs);
1182 		return;
1183 	}
1184 
1185 	sie = ccb->ccb_rcb->rcb_reply;
1186 
1187 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd_done xs cmd: 0x%02x len: %d "
1188 	    "flags 0x%x\n", DEVNAME(sc), xs->cmd->opcode, xs->datalen,
1189 	    xs->flags);
1190 	DNPRINTF(MPI_D_CMD, "%s:  target_id: %d bus: %d msg_length: %d "
1191 	    "function: 0x%02x\n", DEVNAME(sc), sie->target_id, sie->bus,
1192 	    sie->msg_length, sie->function);
1193 	DNPRINTF(MPI_D_CMD, "%s:  cdb_length: %d sense_buf_length: %d "
1194 	    "msg_flags: 0x%02x\n", DEVNAME(sc), sie->cdb_length,
1195 	    sie->sense_buf_len, sie->msg_flags);
1196 	DNPRINTF(MPI_D_CMD, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1197 	    letoh32(sie->msg_context));
1198 	DNPRINTF(MPI_D_CMD, "%s:  scsi_status: 0x%02x scsi_state: 0x%02x "
1199 	    "ioc_status: 0x%04x\n", DEVNAME(sc), sie->scsi_status,
1200 	    sie->scsi_state, letoh16(sie->ioc_status));
1201 	DNPRINTF(MPI_D_CMD, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1202 	    letoh32(sie->ioc_loginfo));
1203 	DNPRINTF(MPI_D_CMD, "%s:  transfer_count: %d\n", DEVNAME(sc),
1204 	    letoh32(sie->transfer_count));
1205 	DNPRINTF(MPI_D_CMD, "%s:  sense_count: %d\n", DEVNAME(sc),
1206 	    letoh32(sie->sense_count));
1207 	DNPRINTF(MPI_D_CMD, "%s:  response_info: 0x%08x\n", DEVNAME(sc),
1208 	    letoh32(sie->response_info));
1209 	DNPRINTF(MPI_D_CMD, "%s:  tag: 0x%04x\n", DEVNAME(sc),
1210 	    letoh16(sie->tag));
1211 
1212 	xs->status = sie->scsi_status;
1213 	switch (letoh16(sie->ioc_status)) {
1214 	case MPI_IOCSTATUS_SCSI_DATA_UNDERRUN:
1215 		xs->resid = xs->datalen - letoh32(sie->transfer_count);
1216 		if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_NO_SCSI_STATUS) {
1217 			xs->error = XS_DRIVER_STUFFUP;
1218 			break;
1219 		}
1220 		/* FALLTHROUGH */
1221 	case MPI_IOCSTATUS_SUCCESS:
1222 	case MPI_IOCSTATUS_SCSI_RECOVERED_ERROR:
1223 		switch (xs->status) {
1224 		case SCSI_OK:
1225 			xs->resid = 0;
1226 			break;
1227 
1228 		case SCSI_CHECK:
1229 			xs->error = XS_SENSE;
1230 			break;
1231 
1232 		case SCSI_BUSY:
1233 		case SCSI_QUEUE_FULL:
1234 			xs->error = XS_BUSY;
1235 			break;
1236 
1237 		default:
1238 			xs->error = XS_DRIVER_STUFFUP;
1239 			break;
1240 		}
1241 		break;
1242 
1243 	case MPI_IOCSTATUS_BUSY:
1244 	case MPI_IOCSTATUS_INSUFFICIENT_RESOURCES:
1245 		xs->error = XS_BUSY;
1246 		break;
1247 
1248 	case MPI_IOCSTATUS_SCSI_INVALID_BUS:
1249 	case MPI_IOCSTATUS_SCSI_INVALID_TARGETID:
1250 	case MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1251 		xs->error = XS_SELTIMEOUT;
1252 		break;
1253 
1254 	default:
1255 		xs->error = XS_DRIVER_STUFFUP;
1256 		break;
1257 	}
1258 
1259 	if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_AUTOSENSE_VALID)
1260 		bcopy(&mcb->mcb_sense, &xs->sense, sizeof(xs->sense));
1261 
1262 	DNPRINTF(MPI_D_CMD, "%s:  xs err: 0x%02x status: %d\n", DEVNAME(sc),
1263 	    xs->error, xs->status);
1264 
1265 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
1266 	mpi_put_ccb(sc, ccb);
1267 	scsi_done(xs);
1268 }
1269 
1270 void
1271 mpi_timeout_xs(void *arg)
1272 {
1273 	/* XXX */
1274 }
1275 
1276 int
1277 mpi_load_xs(struct mpi_ccb *ccb)
1278 {
1279 	struct mpi_softc		*sc = ccb->ccb_sc;
1280 	struct scsi_xfer		*xs = ccb->ccb_xs;
1281 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1282 	struct mpi_msg_scsi_io		*io = &mcb->mcb_io;
1283 	struct mpi_sge			*sge, *nsge = &mcb->mcb_sgl[0];
1284 	struct mpi_sge			*ce = NULL, *nce;
1285 	u_int64_t			ce_dva;
1286 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1287 	u_int32_t			addr, flags;
1288 	int				i, error;
1289 
1290 	if (xs->datalen == 0) {
1291 		nsge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
1292 		    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
1293 		return (0);
1294 	}
1295 
1296 	error = bus_dmamap_load(sc->sc_dmat, dmap,
1297 	    xs->data, xs->datalen, NULL,
1298 	    (xs->flags & SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1299 	if (error) {
1300 		printf("%s: error %d loading dmamap\n", DEVNAME(sc), error);
1301 		return (1);
1302 	}
1303 
1304 	flags = MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64;
1305 	if (xs->flags & SCSI_DATA_OUT)
1306 		flags |= MPI_SGE_FL_DIR_OUT;
1307 
1308 	if (dmap->dm_nsegs > sc->sc_first_sgl_len) {
1309 		ce = &mcb->mcb_sgl[sc->sc_first_sgl_len - 1];
1310 		io->chain_offset = ((u_int8_t *)ce - (u_int8_t *)io) / 4;
1311 	}
1312 
1313 	for (i = 0; i < dmap->dm_nsegs; i++) {
1314 
1315 		if (nsge == ce) {
1316 			nsge++;
1317 			sge->sg_hdr |= htole32(MPI_SGE_FL_LAST);
1318 
1319 			DNPRINTF(MPI_D_DMA, "%s:   - 0x%08x 0x%08x 0x%08x\n",
1320 			    DEVNAME(sc), sge->sg_hdr,
1321 			    sge->sg_hi_addr, sge->sg_lo_addr);
1322 
1323 			if ((dmap->dm_nsegs - i) > sc->sc_chain_len) {
1324 				nce = &nsge[sc->sc_chain_len - 1];
1325 				addr = ((u_int8_t *)nce - (u_int8_t *)nsge) / 4;
1326 				addr = addr << 16 |
1327 				    sizeof(struct mpi_sge) * sc->sc_chain_len;
1328 			} else {
1329 				nce = NULL;
1330 				addr = sizeof(struct mpi_sge) *
1331 				    (dmap->dm_nsegs - i);
1332 			}
1333 
1334 			ce->sg_hdr = htole32(MPI_SGE_FL_TYPE_CHAIN |
1335 			    MPI_SGE_FL_SIZE_64 | addr);
1336 
1337 			ce_dva = ccb->ccb_cmd_dva +
1338 			    ((u_int8_t *)nsge - (u_int8_t *)mcb);
1339 
1340 			addr = (u_int32_t)(ce_dva >> 32);
1341 			ce->sg_hi_addr = htole32(addr);
1342 			addr = (u_int32_t)ce_dva;
1343 			ce->sg_lo_addr = htole32(addr);
1344 
1345 			DNPRINTF(MPI_D_DMA, "%s:  ce: 0x%08x 0x%08x 0x%08x\n",
1346 			    DEVNAME(sc), ce->sg_hdr, ce->sg_hi_addr,
1347 			    ce->sg_lo_addr);
1348 
1349 			ce = nce;
1350 		}
1351 
1352 		DNPRINTF(MPI_D_DMA, "%s:  %d: %d 0x%016llx\n", DEVNAME(sc),
1353 		    i, dmap->dm_segs[i].ds_len,
1354 		    (u_int64_t)dmap->dm_segs[i].ds_addr);
1355 
1356 		sge = nsge;
1357 
1358 		sge->sg_hdr = htole32(flags | dmap->dm_segs[i].ds_len);
1359 		addr = (u_int32_t)((u_int64_t)dmap->dm_segs[i].ds_addr >> 32);
1360 		sge->sg_hi_addr = htole32(addr);
1361 		addr = (u_int32_t)dmap->dm_segs[i].ds_addr;
1362 		sge->sg_lo_addr = htole32(addr);
1363 
1364 		DNPRINTF(MPI_D_DMA, "%s:  %d: 0x%08x 0x%08x 0x%08x\n",
1365 		    DEVNAME(sc), i, sge->sg_hdr, sge->sg_hi_addr,
1366 		    sge->sg_lo_addr);
1367 
1368 		nsge = sge + 1;
1369 	}
1370 
1371 	/* terminate list */
1372 	sge->sg_hdr |= htole32(MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
1373 	    MPI_SGE_FL_EOL);
1374 
1375 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1376 	    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
1377 	    BUS_DMASYNC_PREWRITE);
1378 
1379 	return (0);
1380 }
1381 
1382 void
1383 mpi_minphys(struct buf *bp)
1384 {
1385 	/* XXX */
1386 	if (bp->b_bcount > MAXPHYS)
1387 		bp->b_bcount = MAXPHYS;
1388 	minphys(bp);
1389 }
1390 
1391 int
1392 mpi_scsi_probe(struct scsi_link *link)
1393 {
1394 	struct mpi_softc		*sc = link->adapter_softc;
1395 	struct mpi_ecfg_hdr		ehdr;
1396 	struct mpi_cfg_sas_dev_pg0	pg0;
1397 	u_int32_t			address;
1398 
1399 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SAS || link->lun != 0)
1400 		return (0);
1401 
1402 	address = MPI_CFG_SAS_DEV_ADDR_BUS | link->target;
1403 
1404 	if (mpi_ecfg_header(sc, MPI_CONFIG_REQ_EXTPAGE_TYPE_SAS_DEVICE, 0,
1405 	    address, &ehdr) != 0)
1406 		return (EIO);
1407 
1408 	if (mpi_ecfg_page(sc, address, &ehdr, 1, &pg0, sizeof(pg0)) != 0) {
1409 		/* the device probably doesnt exist if the page fetch fails */
1410 		return (ENXIO);
1411 	}
1412 
1413 	DPRINTF(MPI_D_MISC, "%s: mpi_scsi_probe sas dev pg 0 for target %d:\n",
1414 	    DEVNAME(sc), link->target);
1415 	DPRINTF(MPI_D_MISC, "%s:  slot: 0x%04x enc_handle: 0x%04x\n",
1416 	    DEVNAME(sc), letoh16(pg0.slot), letoh16(pg0.enc_handle));
1417 	DPRINTF(MPI_D_MISC, "%s:  sas_addr: 0x%016llx\n", DEVNAME(sc),
1418 	    letoh64(pg0.sas_addr));
1419 	DPRINTF(MPI_D_MISC, "%s:  parent_dev_handle: 0x%04x phy_num: 0x%02x "
1420 	    "access_status: 0x%02x\n", DEVNAME(sc),
1421 	    letoh16(pg0.parent_dev_handle), pg0.phy_num, pg0.access_status);
1422 	DPRINTF(MPI_D_MISC, "%s:  dev_handle: 0x%04x "
1423 	    "bus: 0x%02x target: 0x%02x\n", DEVNAME(sc),
1424 	    letoh16(pg0.dev_handle), pg0.bus, pg0.target);
1425 	DPRINTF(MPI_D_MISC, "%s:  device_info: 0x%08x\n", DEVNAME(sc),
1426 	    letoh32(pg0.device_info));
1427 	DPRINTF(MPI_D_MISC, "%s:  flags: 0x%04x physical_port: 0x%02x\n",
1428 	    DEVNAME(sc), letoh16(pg0.flags), pg0.physical_port);
1429 
1430 	if (ISSET(letoh32(pg0.device_info),
1431 	    MPI_CFG_SAS_DEV_0_DEVINFO_ATAPI_DEVICE)) {
1432 		DPRINTF(MPI_D_MISC, "%s: target %d is an ATAPI device\n",
1433 		    DEVNAME(sc), link->target);
1434 		link->flags |= SDEV_ATAPI;
1435 		link->quirks |= SDEV_ONLYBIG;
1436 	}
1437 
1438 	return (0);
1439 }
1440 
1441 int
1442 mpi_scsi_ioctl(struct scsi_link *a, u_long b, caddr_t c, int d, struct proc *e)
1443 {
1444 	return (ENOTTY);
1445 }
1446 
1447 u_int32_t
1448 mpi_read(struct mpi_softc *sc, bus_size_t r)
1449 {
1450 	u_int32_t			rv;
1451 
1452 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1453 	    BUS_SPACE_BARRIER_READ);
1454 	rv = bus_space_read_4(sc->sc_iot, sc->sc_ioh, r);
1455 
1456 	DNPRINTF(MPI_D_RW, "%s: mpi_read %#x %#x\n", DEVNAME(sc), r, rv);
1457 
1458 	return (rv);
1459 }
1460 
1461 void
1462 mpi_write(struct mpi_softc *sc, bus_size_t r, u_int32_t v)
1463 {
1464 	DNPRINTF(MPI_D_RW, "%s: mpi_write %#x %#x\n", DEVNAME(sc), r, v);
1465 
1466 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
1467 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1468 	    BUS_SPACE_BARRIER_WRITE);
1469 }
1470 
1471 int
1472 mpi_wait_eq(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1473     u_int32_t target)
1474 {
1475 	int				i;
1476 
1477 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_eq %#x %#x %#x\n", DEVNAME(sc), r,
1478 	    mask, target);
1479 
1480 	for (i = 0; i < 10000; i++) {
1481 		if ((mpi_read(sc, r) & mask) == target)
1482 			return (0);
1483 		delay(1000);
1484 	}
1485 
1486 	return (1);
1487 }
1488 
1489 int
1490 mpi_wait_ne(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1491     u_int32_t target)
1492 {
1493 	int				i;
1494 
1495 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_ne %#x %#x %#x\n", DEVNAME(sc), r,
1496 	    mask, target);
1497 
1498 	for (i = 0; i < 10000; i++) {
1499 		if ((mpi_read(sc, r) & mask) != target)
1500 			return (0);
1501 		delay(1000);
1502 	}
1503 
1504 	return (1);
1505 }
1506 
1507 int
1508 mpi_init(struct mpi_softc *sc)
1509 {
1510 	u_int32_t			db;
1511 	int				i;
1512 
1513 	/* spin until the IOC leaves the RESET state */
1514 	if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1515 	    MPI_DOORBELL_STATE_RESET) != 0) {
1516 		DNPRINTF(MPI_D_MISC, "%s: mpi_init timeout waiting to leave "
1517 		    "reset state\n", DEVNAME(sc));
1518 		return (1);
1519 	}
1520 
1521 	/* check current ownership */
1522 	db = mpi_read_db(sc);
1523 	if ((db & MPI_DOORBELL_WHOINIT) == MPI_DOORBELL_WHOINIT_PCIPEER) {
1524 		DNPRINTF(MPI_D_MISC, "%s: mpi_init initialised by pci peer\n",
1525 		    DEVNAME(sc));
1526 		return (0);
1527 	}
1528 
1529 	for (i = 0; i < 5; i++) {
1530 		switch (db & MPI_DOORBELL_STATE) {
1531 		case MPI_DOORBELL_STATE_READY:
1532 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is ready\n",
1533 			    DEVNAME(sc));
1534 			return (0);
1535 
1536 		case MPI_DOORBELL_STATE_OPER:
1537 		case MPI_DOORBELL_STATE_FAULT:
1538 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is being "
1539 			    "reset\n" , DEVNAME(sc));
1540 			if (mpi_reset_soft(sc) != 0)
1541 				mpi_reset_hard(sc);
1542 			break;
1543 
1544 		case MPI_DOORBELL_STATE_RESET:
1545 			DNPRINTF(MPI_D_MISC, "%s: mpi_init waiting to come "
1546 			    "out of reset\n", DEVNAME(sc));
1547 			if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1548 			    MPI_DOORBELL_STATE_RESET) != 0)
1549 				return (1);
1550 			break;
1551 		}
1552 		db = mpi_read_db(sc);
1553 	}
1554 
1555 	return (1);
1556 }
1557 
1558 int
1559 mpi_reset_soft(struct mpi_softc *sc)
1560 {
1561 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_soft\n", DEVNAME(sc));
1562 
1563 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1564 		return (1);
1565 
1566 	mpi_write_db(sc,
1567 	    MPI_DOORBELL_FUNCTION(MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET));
1568 	if (mpi_wait_eq(sc, MPI_INTR_STATUS,
1569 	    MPI_INTR_STATUS_IOCDOORBELL, 0) != 0)
1570 		return (1);
1571 
1572 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1573 	    MPI_DOORBELL_STATE_READY) != 0)
1574 		return (1);
1575 
1576 	return (0);
1577 }
1578 
1579 int
1580 mpi_reset_hard(struct mpi_softc *sc)
1581 {
1582 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_hard\n", DEVNAME(sc));
1583 
1584 	/* enable diagnostic register */
1585 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1586 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_1);
1587 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_2);
1588 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_3);
1589 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_4);
1590 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_5);
1591 
1592 	/* reset ioc */
1593 	mpi_write(sc, MPI_HOSTDIAG, MPI_HOSTDIAG_RESET_ADAPTER);
1594 
1595 	delay(10000);
1596 
1597 	/* disable diagnostic register */
1598 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1599 
1600 	/* restore pci bits? */
1601 
1602 	/* firmware bits? */
1603 	return (0);
1604 }
1605 
1606 int
1607 mpi_handshake_send(struct mpi_softc *sc, void *buf, size_t dwords)
1608 {
1609 	u_int32_t				*query = buf;
1610 	int					i;
1611 
1612 	/* make sure the doorbell is not in use. */
1613 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1614 		return (1);
1615 
1616 	/* clear pending doorbell interrupts */
1617 	if (mpi_read_intr(sc) & MPI_INTR_STATUS_DOORBELL)
1618 		mpi_write_intr(sc, 0);
1619 
1620 	/*
1621 	 * first write the doorbell with the handshake function and the
1622 	 * dword count.
1623 	 */
1624 	mpi_write_db(sc, MPI_DOORBELL_FUNCTION(MPI_FUNCTION_HANDSHAKE) |
1625 	    MPI_DOORBELL_DWORDS(dwords));
1626 
1627 	/*
1628 	 * the doorbell used bit will be set because a doorbell function has
1629 	 * started. Wait for the interrupt and then ack it.
1630 	 */
1631 	if (mpi_wait_db_int(sc) != 0)
1632 		return (1);
1633 	mpi_write_intr(sc, 0);
1634 
1635 	/* poll for the acknowledgement. */
1636 	if (mpi_wait_db_ack(sc) != 0)
1637 		return (1);
1638 
1639 	/* write the query through the doorbell. */
1640 	for (i = 0; i < dwords; i++) {
1641 		mpi_write_db(sc, htole32(query[i]));
1642 		if (mpi_wait_db_ack(sc) != 0)
1643 			return (1);
1644 	}
1645 
1646 	return (0);
1647 }
1648 
1649 int
1650 mpi_handshake_recv_dword(struct mpi_softc *sc, u_int32_t *dword)
1651 {
1652 	u_int16_t				*words = (u_int16_t *)dword;
1653 	int					i;
1654 
1655 	for (i = 0; i < 2; i++) {
1656 		if (mpi_wait_db_int(sc) != 0)
1657 			return (1);
1658 		words[i] = letoh16(mpi_read_db(sc) & MPI_DOORBELL_DATA_MASK);
1659 		mpi_write_intr(sc, 0);
1660 	}
1661 
1662 	return (0);
1663 }
1664 
1665 int
1666 mpi_handshake_recv(struct mpi_softc *sc, void *buf, size_t dwords)
1667 {
1668 	struct mpi_msg_reply			*reply = buf;
1669 	u_int32_t				*dbuf = buf, dummy;
1670 	int					i;
1671 
1672 	/* get the first dword so we can read the length out of the header. */
1673 	if (mpi_handshake_recv_dword(sc, &dbuf[0]) != 0)
1674 		return (1);
1675 
1676 	DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dwords: %d reply: %d\n",
1677 	    DEVNAME(sc), dwords, reply->msg_length);
1678 
1679 	/*
1680 	 * the total length, in dwords, is in the message length field of the
1681 	 * reply header.
1682 	 */
1683 	for (i = 1; i < MIN(dwords, reply->msg_length); i++) {
1684 		if (mpi_handshake_recv_dword(sc, &dbuf[i]) != 0)
1685 			return (1);
1686 	}
1687 
1688 	/* if there's extra stuff to come off the ioc, discard it */
1689 	while (i++ < reply->msg_length) {
1690 		if (mpi_handshake_recv_dword(sc, &dummy) != 0)
1691 			return (1);
1692 		DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dummy read: "
1693 		    "0x%08x\n", DEVNAME(sc), dummy);
1694 	}
1695 
1696 	/* wait for the doorbell used bit to be reset and clear the intr */
1697 	if (mpi_wait_db_int(sc) != 0)
1698 		return (1);
1699 	mpi_write_intr(sc, 0);
1700 
1701 	return (0);
1702 }
1703 
1704 void
1705 mpi_empty_done(struct mpi_ccb *ccb)
1706 {
1707 	/* nothing to do */
1708 }
1709 
1710 int
1711 mpi_iocfacts(struct mpi_softc *sc)
1712 {
1713 	struct mpi_msg_iocfacts_request		ifq;
1714 	struct mpi_msg_iocfacts_reply		ifp;
1715 
1716 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts\n", DEVNAME(sc));
1717 
1718 	bzero(&ifq, sizeof(ifq));
1719 	bzero(&ifp, sizeof(ifp));
1720 
1721 	ifq.function = MPI_FUNCTION_IOC_FACTS;
1722 	ifq.chain_offset = 0;
1723 	ifq.msg_flags = 0;
1724 	ifq.msg_context = htole32(0xdeadbeef);
1725 
1726 	if (mpi_handshake_send(sc, &ifq, dwordsof(ifq)) != 0) {
1727 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts send failed\n",
1728 		    DEVNAME(sc));
1729 		return (1);
1730 	}
1731 
1732 	if (mpi_handshake_recv(sc, &ifp, dwordsof(ifp)) != 0) {
1733 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts recv failed\n",
1734 		    DEVNAME(sc));
1735 		return (1);
1736 	}
1737 
1738 	DNPRINTF(MPI_D_MISC, "%s:  func: 0x%02x len: %d msgver: %d.%d\n",
1739 	    DEVNAME(sc), ifp.function, ifp.msg_length,
1740 	    ifp.msg_version_maj, ifp.msg_version_min);
1741 	DNPRINTF(MPI_D_MISC, "%s:  msgflags: 0x%02x iocnumber: 0x%02x "
1742 	    "hdrver: %d.%d\n", DEVNAME(sc), ifp.msg_flags,
1743 	    ifp.ioc_number, ifp.header_version_maj,
1744 	    ifp.header_version_min);
1745 	DNPRINTF(MPI_D_MISC, "%s:  message context: 0x%08x\n", DEVNAME(sc),
1746 	    letoh32(ifp.msg_context));
1747 	DNPRINTF(MPI_D_MISC, "%s:  iocstatus: 0x%04x ioexcept: 0x%04x\n",
1748 	    DEVNAME(sc), letoh16(ifp.ioc_status),
1749 	    letoh16(ifp.ioc_exceptions));
1750 	DNPRINTF(MPI_D_MISC, "%s:  iocloginfo: 0x%08x\n", DEVNAME(sc),
1751 	    letoh32(ifp.ioc_loginfo));
1752 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%02x blocksize: %d whoinit: 0x%02x "
1753 	    "maxchdepth: %d\n", DEVNAME(sc), ifp.flags,
1754 	    ifp.block_size, ifp.whoinit, ifp.max_chain_depth);
1755 	DNPRINTF(MPI_D_MISC, "%s:  reqfrsize: %d replyqdepth: %d\n",
1756 	    DEVNAME(sc), letoh16(ifp.request_frame_size),
1757 	    letoh16(ifp.reply_queue_depth));
1758 	DNPRINTF(MPI_D_MISC, "%s:  productid: 0x%04x\n", DEVNAME(sc),
1759 	    letoh16(ifp.product_id));
1760 	DNPRINTF(MPI_D_MISC, "%s:  hostmfahiaddr: 0x%08x\n", DEVNAME(sc),
1761 	    letoh32(ifp.current_host_mfa_hi_addr));
1762 	DNPRINTF(MPI_D_MISC, "%s:  event_state: 0x%02x number_of_ports: %d "
1763 	    "global_credits: %d\n",
1764 	    DEVNAME(sc), ifp.event_state, ifp.number_of_ports,
1765 	    letoh16(ifp.global_credits));
1766 	DNPRINTF(MPI_D_MISC, "%s:  sensebufhiaddr: 0x%08x\n", DEVNAME(sc),
1767 	    letoh32(ifp.current_sense_buffer_hi_addr));
1768 	DNPRINTF(MPI_D_MISC, "%s:  maxbus: %d maxdev: %d replyfrsize: %d\n",
1769 	    DEVNAME(sc), ifp.max_buses, ifp.max_devices,
1770 	    letoh16(ifp.current_reply_frame_size));
1771 	DNPRINTF(MPI_D_MISC, "%s:  fw_image_size: %d\n", DEVNAME(sc),
1772 	    letoh32(ifp.fw_image_size));
1773 	DNPRINTF(MPI_D_MISC, "%s:  ioc_capabilities: 0x%08x\n", DEVNAME(sc),
1774 	    letoh32(ifp.ioc_capabilities));
1775 	DNPRINTF(MPI_D_MISC, "%s:  fw_version: %d.%d fw_version_unit: 0x%02x "
1776 	    "fw_version_dev: 0x%02x\n", DEVNAME(sc),
1777 	    ifp.fw_version_maj, ifp.fw_version_min,
1778 	    ifp.fw_version_unit, ifp.fw_version_dev);
1779 	DNPRINTF(MPI_D_MISC, "%s:  hi_priority_queue_depth: 0x%04x\n",
1780 	    DEVNAME(sc), letoh16(ifp.hi_priority_queue_depth));
1781 	DNPRINTF(MPI_D_MISC, "%s:  host_page_buffer_sge: hdr: 0x%08x "
1782 	    "addr 0x%08x %08x\n", DEVNAME(sc),
1783 	    letoh32(ifp.host_page_buffer_sge.sg_hdr),
1784 	    letoh32(ifp.host_page_buffer_sge.sg_hi_addr),
1785 	    letoh32(ifp.host_page_buffer_sge.sg_lo_addr));
1786 
1787 	sc->sc_maxcmds = letoh16(ifp.global_credits);
1788 	sc->sc_maxchdepth = ifp.max_chain_depth;
1789 	sc->sc_ioc_number = ifp.ioc_number;
1790 	if (sc->sc_flags & MPI_F_SPI)
1791 		sc->sc_buswidth = 16;
1792 	else
1793 		sc->sc_buswidth =
1794 		    (ifp.max_devices == 0) ? 256 : ifp.max_devices;
1795 	if (ifp.flags & MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT)
1796 		sc->sc_fw_len = letoh32(ifp.fw_image_size);
1797 
1798 	/*
1799 	 * you can fit sg elements on the end of the io cmd if they fit in the
1800 	 * request frame size.
1801 	 */
1802 	sc->sc_first_sgl_len = ((letoh16(ifp.request_frame_size) * 4) -
1803 	    sizeof(struct mpi_msg_scsi_io)) / sizeof(struct mpi_sge);
1804 	DNPRINTF(MPI_D_MISC, "%s:   first sgl len: %d\n", DEVNAME(sc),
1805 	    sc->sc_first_sgl_len);
1806 
1807 	sc->sc_chain_len = (letoh16(ifp.request_frame_size) * 4) /
1808 	    sizeof(struct mpi_sge);
1809 	DNPRINTF(MPI_D_MISC, "%s:   chain len: %d\n", DEVNAME(sc),
1810 	    sc->sc_chain_len);
1811 
1812 	/* the sgl tailing the io cmd loses an entry to the chain element. */
1813 	sc->sc_max_sgl_len = MPI_MAX_SGL - 1;
1814 	/* the sgl chains lose an entry for each chain element */
1815 	sc->sc_max_sgl_len -= (MPI_MAX_SGL - sc->sc_first_sgl_len) /
1816 	    sc->sc_chain_len;
1817 	DNPRINTF(MPI_D_MISC, "%s:   max sgl len: %d\n", DEVNAME(sc),
1818 	    sc->sc_max_sgl_len);
1819 
1820 	/* XXX we're ignoring the max chain depth */
1821 
1822 	return (0);
1823 }
1824 
1825 int
1826 mpi_iocinit(struct mpi_softc *sc)
1827 {
1828 	struct mpi_msg_iocinit_request		iiq;
1829 	struct mpi_msg_iocinit_reply		iip;
1830 	u_int32_t				hi_addr;
1831 
1832 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit\n", DEVNAME(sc));
1833 
1834 	bzero(&iiq, sizeof(iiq));
1835 	bzero(&iip, sizeof(iip));
1836 
1837 	iiq.function = MPI_FUNCTION_IOC_INIT;
1838 	iiq.whoinit = MPI_WHOINIT_HOST_DRIVER;
1839 
1840 	iiq.max_devices = (sc->sc_buswidth == 256) ? 0 : sc->sc_buswidth;
1841 	iiq.max_buses = 1;
1842 
1843 	iiq.msg_context = htole32(0xd00fd00f);
1844 
1845 	iiq.reply_frame_size = htole16(MPI_REPLY_SIZE);
1846 
1847 	hi_addr = (u_int32_t)((u_int64_t)MPI_DMA_DVA(sc->sc_requests) >> 32);
1848 	iiq.host_mfa_hi_addr = htole32(hi_addr);
1849 	iiq.sense_buffer_hi_addr = htole32(hi_addr);
1850 
1851 	hi_addr = (u_int32_t)((u_int64_t)MPI_DMA_DVA(sc->sc_replies) >> 32);
1852 	iiq.reply_fifo_host_signalling_addr = htole32(hi_addr);
1853 
1854 	iiq.msg_version_maj = 0x01;
1855 	iiq.msg_version_min = 0x02;
1856 
1857 	iiq.hdr_version_unit = 0x0d;
1858 	iiq.hdr_version_dev = 0x00;
1859 
1860 	if (mpi_handshake_send(sc, &iiq, dwordsof(iiq)) != 0) {
1861 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit send failed\n",
1862 		    DEVNAME(sc));
1863 		return (1);
1864 	}
1865 
1866 	if (mpi_handshake_recv(sc, &iip, dwordsof(iip)) != 0) {
1867 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit recv failed\n",
1868 		    DEVNAME(sc));
1869 		return (1);
1870 	}
1871 
1872 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d "
1873 	    "whoinit: 0x%02x\n", DEVNAME(sc), iip.function,
1874 	    iip.msg_length, iip.whoinit);
1875 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x max_buses: %d "
1876 	    "max_devices: %d flags: 0x%02x\n", DEVNAME(sc), iip.msg_flags,
1877 	    iip.max_buses, iip.max_devices, iip.flags);
1878 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1879 	    letoh32(iip.msg_context));
1880 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
1881 	    letoh16(iip.ioc_status));
1882 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1883 	    letoh32(iip.ioc_loginfo));
1884 
1885 	return (0);
1886 }
1887 
1888 int
1889 mpi_portfacts(struct mpi_softc *sc)
1890 {
1891 	struct mpi_ccb				*ccb;
1892 	struct mpi_msg_portfacts_request	*pfq;
1893 	volatile struct mpi_msg_portfacts_reply	*pfp;
1894 	int					s, rv = 1;
1895 
1896 	DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts\n", DEVNAME(sc));
1897 
1898 	s = splbio();
1899 	ccb = mpi_get_ccb(sc);
1900 	splx(s);
1901 	if (ccb == NULL) {
1902 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts ccb_get\n",
1903 		    DEVNAME(sc));
1904 		return (rv);
1905 	}
1906 
1907 	ccb->ccb_done = mpi_empty_done;
1908 	pfq = ccb->ccb_cmd;
1909 
1910 	pfq->function = MPI_FUNCTION_PORT_FACTS;
1911 	pfq->chain_offset = 0;
1912 	pfq->msg_flags = 0;
1913 	pfq->port_number = 0;
1914 	pfq->msg_context = htole32(ccb->ccb_id);
1915 
1916 	if (mpi_poll(sc, ccb, 50000) != 0) {
1917 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts poll\n", DEVNAME(sc));
1918 		goto err;
1919 	}
1920 
1921 	if (ccb->ccb_rcb == NULL) {
1922 		DNPRINTF(MPI_D_MISC, "%s: empty portfacts reply\n",
1923 		    DEVNAME(sc));
1924 		goto err;
1925 	}
1926 	pfp = ccb->ccb_rcb->rcb_reply;
1927 
1928 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d\n",
1929 	    DEVNAME(sc), pfp->function, pfp->msg_length);
1930 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x port_number: %d\n",
1931 	    DEVNAME(sc), pfp->msg_flags, pfp->port_number);
1932 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1933 	    letoh32(pfp->msg_context));
1934 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
1935 	    letoh16(pfp->ioc_status));
1936 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1937 	    letoh32(pfp->ioc_loginfo));
1938 	DNPRINTF(MPI_D_MISC, "%s:  max_devices: %d port_type: 0x%02x\n",
1939 	    DEVNAME(sc), letoh16(pfp->max_devices), pfp->port_type);
1940 	DNPRINTF(MPI_D_MISC, "%s:  protocol_flags: 0x%04x port_scsi_id: %d\n",
1941 	    DEVNAME(sc), letoh16(pfp->protocol_flags),
1942 	    letoh16(pfp->port_scsi_id));
1943 	DNPRINTF(MPI_D_MISC, "%s:  max_persistent_ids: %d "
1944 	    "max_posted_cmd_buffers: %d\n", DEVNAME(sc),
1945 	    letoh16(pfp->max_persistent_ids),
1946 	    letoh16(pfp->max_posted_cmd_buffers));
1947 	DNPRINTF(MPI_D_MISC, "%s:  max_lan_buckets: %d\n", DEVNAME(sc),
1948 	    letoh16(pfp->max_lan_buckets));
1949 
1950 	sc->sc_porttype = pfp->port_type;
1951 	if (sc->sc_target == -1)
1952 		sc->sc_target = letoh16(pfp->port_scsi_id);
1953 
1954 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
1955 	rv = 0;
1956 err:
1957 	mpi_put_ccb(sc, ccb);
1958 
1959 	return (rv);
1960 }
1961 
1962 int
1963 mpi_eventnotify(struct mpi_softc *sc)
1964 {
1965 	struct mpi_ccb				*ccb;
1966 	struct mpi_msg_event_request		*enq;
1967 	int					s;
1968 
1969 	s = splbio();
1970 	ccb = mpi_get_ccb(sc);
1971 	splx(s);
1972 	if (ccb == NULL) {
1973 		DNPRINTF(MPI_D_MISC, "%s: mpi_eventnotify ccb_get\n",
1974 		    DEVNAME(sc));
1975 		return (1);
1976 	}
1977 
1978 	ccb->ccb_done = mpi_eventnotify_done;
1979 	enq = ccb->ccb_cmd;
1980 
1981 	enq->function = MPI_FUNCTION_EVENT_NOTIFICATION;
1982 	enq->chain_offset = 0;
1983 	enq->event_switch = MPI_EVENT_SWITCH_ON;
1984 	enq->msg_context = htole32(ccb->ccb_id);
1985 
1986 	mpi_start(sc, ccb);
1987 	return (0);
1988 }
1989 
1990 void
1991 mpi_eventnotify_done(struct mpi_ccb *ccb)
1992 {
1993 	struct mpi_softc			*sc = ccb->ccb_sc;
1994 	struct mpi_msg_event_reply		*enp = ccb->ccb_rcb->rcb_reply;
1995 	int					deferred = 0;
1996 
1997 	DNPRINTF(MPI_D_EVT, "%s: mpi_eventnotify_done\n", DEVNAME(sc));
1998 
1999 	DNPRINTF(MPI_D_EVT, "%s:  function: 0x%02x msg_length: %d "
2000 	    "data_length: %d\n", DEVNAME(sc), enp->function, enp->msg_length,
2001 	    letoh16(enp->data_length));
2002 	DNPRINTF(MPI_D_EVT, "%s:  ack_required: %d msg_flags 0x%02x\n",
2003 	    DEVNAME(sc), enp->ack_required, enp->msg_flags);
2004 	DNPRINTF(MPI_D_EVT, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2005 	    letoh32(enp->msg_context));
2006 	DNPRINTF(MPI_D_EVT, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2007 	    letoh16(enp->ioc_status));
2008 	DNPRINTF(MPI_D_EVT, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2009 	    letoh32(enp->ioc_loginfo));
2010 	DNPRINTF(MPI_D_EVT, "%s:  event: 0x%08x\n", DEVNAME(sc),
2011 	    letoh32(enp->event));
2012 	DNPRINTF(MPI_D_EVT, "%s:  event_context: 0x%08x\n", DEVNAME(sc),
2013 	    letoh32(enp->event_context));
2014 
2015 	switch (letoh32(enp->event)) {
2016 	/* ignore these */
2017 	case MPI_EVENT_EVENT_CHANGE:
2018 	case MPI_EVENT_SAS_PHY_LINK_STATUS:
2019 		break;
2020 
2021 	case MPI_EVENT_SAS_DEVICE_STATUS_CHANGE:
2022 		if (sc->sc_scsibus == NULL)
2023 			break;
2024 
2025 		if (scsi_task(mpi_evt_sas, sc, ccb->ccb_rcb, 0) != 0) {
2026 			printf("%s: unable to run SAS device status change\n",
2027 			    DEVNAME(sc));
2028 			break;
2029 		}
2030 		deferred = 1;
2031 		break;
2032 
2033 	default:
2034 		printf("%s: unhandled event 0x%02x\n", DEVNAME(sc),
2035 		    letoh32(enp->event));
2036 		break;
2037 	}
2038 
2039 	if (!deferred) {
2040 		if (enp->ack_required)
2041 			mpi_eventack(sc, enp);
2042 		mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2043 	}
2044 
2045 	if ((enp->msg_flags & MPI_EVENT_FLAGS_REPLY_KEPT) == 0) {
2046 		/* XXX this shouldnt happen till shutdown */
2047 		mpi_put_ccb(sc, ccb);
2048 	}
2049 }
2050 
2051 void
2052 mpi_evt_sas(void *xsc, void *arg)
2053 {
2054 	struct mpi_softc			*sc = xsc;
2055 	struct mpi_rcb				*rcb = arg;
2056 	struct mpi_msg_event_reply		*enp = rcb->rcb_reply;
2057 	struct mpi_evt_sas_change		*ch;
2058 	u_int8_t				*data;
2059 	int					s;
2060 
2061 	data = rcb->rcb_reply;
2062 	data += sizeof(struct mpi_msg_event_reply);
2063 	ch = (struct mpi_evt_sas_change *)data;
2064 
2065 	if (ch->bus != 0)
2066 		return;
2067 
2068 	switch (ch->reason) {
2069 	case MPI_EVT_SASCH_REASON_ADDED:
2070 	case MPI_EVT_SASCH_REASON_NO_PERSIST_ADDED:
2071 		scsi_probe_target(sc->sc_scsibus, ch->target);
2072 		break;
2073 
2074 	case MPI_EVT_SASCH_REASON_NOT_RESPONDING:
2075 		scsi_detach_target(sc->sc_scsibus, ch->target, DETACH_FORCE);
2076 		break;
2077 
2078 	case MPI_EVT_SASCH_REASON_SMART_DATA:
2079 	case MPI_EVT_SASCH_REASON_UNSUPPORTED:
2080 	case MPI_EVT_SASCH_REASON_INTERNAL_RESET:
2081 		break;
2082 	default:
2083 		printf("%s: unknown reason for SAS device status change: "
2084 		    "0x%02x\n", DEVNAME(sc), ch->reason);
2085 		break;
2086 	}
2087 
2088 	s = splbio();
2089 	mpi_push_reply(sc, rcb->rcb_reply_dva);
2090 	if (enp->ack_required)
2091 		mpi_eventack(sc, enp);
2092 	splx(s);
2093 }
2094 
2095 void
2096 mpi_eventack(struct mpi_softc *sc, struct mpi_msg_event_reply *enp)
2097 {
2098 	struct mpi_ccb				*ccb;
2099 	struct mpi_msg_eventack_request		*eaq;
2100 
2101 	ccb = mpi_get_ccb(sc);
2102 	if (ccb == NULL) {
2103 		DNPRINTF(MPI_D_EVT, "%s: mpi_eventack ccb_get\n", DEVNAME(sc));
2104 		return;
2105 	}
2106 
2107 	ccb->ccb_done = mpi_eventack_done;
2108 	eaq = ccb->ccb_cmd;
2109 
2110 	eaq->function = MPI_FUNCTION_EVENT_ACK;
2111 	eaq->msg_context = htole32(ccb->ccb_id);
2112 
2113 	eaq->event = enp->event;
2114 	eaq->event_context = enp->event_context;
2115 
2116 	mpi_start(sc, ccb);
2117 	return;
2118 }
2119 
2120 void
2121 mpi_eventack_done(struct mpi_ccb *ccb)
2122 {
2123 	struct mpi_softc			*sc = ccb->ccb_sc;
2124 
2125 	DNPRINTF(MPI_D_EVT, "%s: event ack done\n", DEVNAME(sc));
2126 
2127 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2128 	mpi_put_ccb(sc, ccb);
2129 }
2130 
2131 int
2132 mpi_portenable(struct mpi_softc *sc)
2133 {
2134 	struct mpi_ccb				*ccb;
2135 	struct mpi_msg_portenable_request	*peq;
2136 	struct mpi_msg_portenable_repy		*pep;
2137 	int					s;
2138 
2139 	DNPRINTF(MPI_D_MISC, "%s: mpi_portenable\n", DEVNAME(sc));
2140 
2141 	s = splbio();
2142 	ccb = mpi_get_ccb(sc);
2143 	splx(s);
2144 	if (ccb == NULL) {
2145 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable ccb_get\n",
2146 		    DEVNAME(sc));
2147 		return (1);
2148 	}
2149 
2150 	ccb->ccb_done = mpi_empty_done;
2151 	peq = ccb->ccb_cmd;
2152 
2153 	peq->function = MPI_FUNCTION_PORT_ENABLE;
2154 	peq->port_number = 0;
2155 	peq->msg_context = htole32(ccb->ccb_id);
2156 
2157 	if (mpi_poll(sc, ccb, 50000) != 0) {
2158 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable poll\n", DEVNAME(sc));
2159 		return (1);
2160 	}
2161 
2162 	if (ccb->ccb_rcb == NULL) {
2163 		DNPRINTF(MPI_D_MISC, "%s: empty portenable reply\n",
2164 		    DEVNAME(sc));
2165 		return (1);
2166 	}
2167 	pep = ccb->ccb_rcb->rcb_reply;
2168 
2169 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2170 	mpi_put_ccb(sc, ccb);
2171 
2172 	return (0);
2173 }
2174 
2175 int
2176 mpi_fwupload(struct mpi_softc *sc)
2177 {
2178 	struct mpi_ccb				*ccb;
2179 	struct {
2180 		struct mpi_msg_fwupload_request		req;
2181 		struct mpi_sge				sge;
2182 	} __packed				*bundle;
2183 	struct mpi_msg_fwupload_reply		*upp;
2184 	u_int64_t				addr;
2185 	int					s;
2186 	int					rv = 0;
2187 
2188 	if (sc->sc_fw_len == 0)
2189 		return (0);
2190 
2191 	DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload\n", DEVNAME(sc));
2192 
2193 	sc->sc_fw = mpi_dmamem_alloc(sc, sc->sc_fw_len);
2194 	if (sc->sc_fw == NULL) {
2195 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload unable to allocate %d\n",
2196 		    DEVNAME(sc), sc->sc_fw_len);
2197 		return (1);
2198 	}
2199 
2200 	s = splbio();
2201 	ccb = mpi_get_ccb(sc);
2202 	splx(s);
2203 	if (ccb == NULL) {
2204 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload ccb_get\n",
2205 		    DEVNAME(sc));
2206 		goto err;
2207 	}
2208 
2209 	ccb->ccb_done = mpi_empty_done;
2210 	bundle = ccb->ccb_cmd;
2211 
2212 	bundle->req.function = MPI_FUNCTION_FW_UPLOAD;
2213 	bundle->req.msg_context = htole32(ccb->ccb_id);
2214 
2215 	bundle->req.image_type = MPI_FWUPLOAD_IMAGETYPE_IOC_FW;
2216 
2217 	bundle->req.tce.details_length = 12;
2218 	bundle->req.tce.image_size = htole32(sc->sc_fw_len);
2219 
2220 	bundle->sge.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2221 	    MPI_SGE_FL_SIZE_64 | MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
2222 	    MPI_SGE_FL_EOL | (u_int32_t)sc->sc_fw_len);
2223 	addr = MPI_DMA_DVA(sc->sc_fw);
2224 	bundle->sge.sg_hi_addr = htole32((u_int32_t)(addr >> 32));
2225 	bundle->sge.sg_lo_addr = htole32((u_int32_t)addr);
2226 
2227 	if (mpi_poll(sc, ccb, 50000) != 0) {
2228 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2229 		goto err;
2230 	}
2231 
2232 	if (ccb->ccb_rcb == NULL)
2233 		panic("%s: unable to do fw upload\n", DEVNAME(sc));
2234 	upp = ccb->ccb_rcb->rcb_reply;
2235 
2236 	if (letoh16(upp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2237 		rv = 1;
2238 
2239 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2240 	mpi_put_ccb(sc, ccb);
2241 
2242 	return (rv);
2243 
2244 err:
2245 	mpi_dmamem_free(sc, sc->sc_fw);
2246 	return (1);
2247 }
2248 
2249 void
2250 mpi_get_raid(struct mpi_softc *sc)
2251 {
2252 	struct mpi_cfg_hdr		hdr;
2253 	struct mpi_cfg_ioc_pg2		*vol_page;
2254 	struct mpi_cfg_raid_vol		*vol_list, *vol;
2255 	size_t				pagelen;
2256 	u_int32_t			capabilities;
2257 	struct scsi_link		*link;
2258 	int				i;
2259 
2260 	DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid\n", DEVNAME(sc));
2261 
2262 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 2, 0, &hdr) != 0) {
2263 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch header"
2264 		    "for IOC page 2\n", DEVNAME(sc));
2265 		return;
2266 	}
2267 
2268 	pagelen = hdr.page_length * 4; /* dwords to bytes */
2269 	vol_page = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
2270 	if (vol_page == NULL) {
2271 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to allocate "
2272 		    "space for ioc config page 2\n", DEVNAME(sc));
2273 		return;
2274 	}
2275 	vol_list = (struct mpi_cfg_raid_vol *)(vol_page + 1);
2276 
2277 	if (mpi_cfg_page(sc, 0, &hdr, 1, vol_page, pagelen) != 0) {
2278 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch IOC "
2279 		    "page 2\n", DEVNAME(sc));
2280 		goto out;
2281 	}
2282 
2283 	capabilities = letoh32(vol_page->capabilities);
2284 
2285 	DNPRINTF(MPI_D_RAID, "%s:  capabilities: 0x08%x\n", DEVNAME(sc),
2286 	    letoh32(vol_page->capabilities));
2287 	DNPRINTF(MPI_D_RAID, "%s:  active_vols: %d max_vols: %d "
2288 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
2289 	    vol_page->active_vols, vol_page->max_vols,
2290 	    vol_page->active_physdisks, vol_page->max_physdisks);
2291 
2292 	/* don't walk list if there are no RAID capability */
2293 	if (capabilities == 0xdeadbeef) {
2294 		printf("%s: deadbeef in raid configuration\n", DEVNAME(sc));
2295 		goto out;
2296 	}
2297 
2298 	if ((capabilities & MPI_CFG_IOC_2_CAPABILITIES_RAID) == 0 ||
2299 	    (vol_page->active_vols == 0))
2300 		goto out;
2301 
2302 	sc->sc_flags |= MPI_F_RAID;
2303 
2304 	for (i = 0; i < vol_page->active_vols; i++) {
2305 		vol = &vol_list[i];
2306 
2307 		DNPRINTF(MPI_D_RAID, "%s:   id: %d bus: %d ioc: %d pg: %d\n",
2308 		    DEVNAME(sc), vol->vol_id, vol->vol_bus, vol->vol_ioc,
2309 		    vol->vol_page);
2310 		DNPRINTF(MPI_D_RAID, "%s:   type: 0x%02x flags: 0x%02x\n",
2311 		    DEVNAME(sc), vol->vol_type, vol->flags);
2312 
2313 		if (vol->vol_ioc != sc->sc_ioc_number || vol->vol_bus != 0)
2314 			continue;
2315 
2316 		link = sc->sc_scsibus->sc_link[vol->vol_id][0];
2317 		if (link == NULL)
2318 			continue;
2319 
2320 		link->flags |= SDEV_VIRTUAL;
2321 	}
2322 
2323 out:
2324 	free(vol_page, M_TEMP);
2325 }
2326 
2327 int
2328 mpi_req_cfg_header(struct mpi_softc *sc, u_int8_t type, u_int8_t number,
2329     u_int32_t address, int extended, void *p)
2330 {
2331 	struct mpi_ccb				*ccb;
2332 	struct mpi_msg_config_request		*cq;
2333 	struct mpi_msg_config_reply		*cp;
2334 	struct mpi_cfg_hdr			*hdr = p;
2335 	struct mpi_ecfg_hdr			*ehdr = p;
2336 	int					etype = 0;
2337 	int					rv = 0;
2338 	int					s;
2339 
2340 	DNPRINTF(MPI_D_MISC, "%s: mpi_req_cfg_header type: %#x number: %x "
2341 	    "address: 0x%08x extended: %d\n", DEVNAME(sc), type, number,
2342 	    address, extended);
2343 
2344 	s = splbio();
2345 	ccb = mpi_get_ccb(sc);
2346 	splx(s);
2347 	if (ccb == NULL) {
2348 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header ccb_get\n",
2349 		    DEVNAME(sc));
2350 		return (1);
2351 	}
2352 
2353 	if (extended) {
2354 		etype = type;
2355 		type = MPI_CONFIG_REQ_PAGE_TYPE_EXTENDED;
2356 	}
2357 
2358 	ccb->ccb_done = mpi_empty_done;
2359 	cq = ccb->ccb_cmd;
2360 
2361 	cq->function = MPI_FUNCTION_CONFIG;
2362 	cq->msg_context = htole32(ccb->ccb_id);
2363 
2364 	cq->action = MPI_CONFIG_REQ_ACTION_PAGE_HEADER;
2365 
2366 	cq->config_header.page_number = number;
2367 	cq->config_header.page_type = type;
2368 	cq->ext_page_type = etype;
2369 	cq->page_address = htole32(address);
2370 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2371 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
2372 
2373 	if (mpi_poll(sc, ccb, 50000) != 0) {
2374 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2375 		return (1);
2376 	}
2377 
2378 	if (ccb->ccb_rcb == NULL)
2379 		panic("%s: unable to fetch config header\n", DEVNAME(sc));
2380 	cp = ccb->ccb_rcb->rcb_reply;
2381 
2382 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2383 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2384 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2385 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2386 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2387 	    cp->msg_flags);
2388 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2389 	    letoh32(cp->msg_context));
2390 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2391 	    letoh16(cp->ioc_status));
2392 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2393 	    letoh32(cp->ioc_loginfo));
2394 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2395 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2396 	    cp->config_header.page_version,
2397 	    cp->config_header.page_length,
2398 	    cp->config_header.page_number,
2399 	    cp->config_header.page_type);
2400 
2401 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2402 		rv = 1;
2403 	else if (extended) {
2404 		bzero(ehdr, sizeof(*ehdr));
2405 		ehdr->page_version = cp->config_header.page_version;
2406 		ehdr->page_number = cp->config_header.page_number;
2407 		ehdr->page_type = cp->config_header.page_type;
2408 		ehdr->ext_page_length = cp->ext_page_length;
2409 		ehdr->ext_page_type = cp->ext_page_type;
2410 	} else
2411 		*hdr = cp->config_header;
2412 
2413 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2414 	mpi_put_ccb(sc, ccb);
2415 
2416 	return (rv);
2417 }
2418 
2419 int
2420 mpi_req_cfg_page(struct mpi_softc *sc, u_int32_t address, int extended,
2421     void *p, int read, void *page, size_t len)
2422 {
2423 	struct mpi_ccb				*ccb;
2424 	struct mpi_msg_config_request		*cq;
2425 	struct mpi_msg_config_reply		*cp;
2426 	struct mpi_cfg_hdr			*hdr = p;
2427 	struct mpi_ecfg_hdr			*ehdr = p;
2428 	u_int64_t				dva;
2429 	char					*kva;
2430 	int					page_length;
2431 	int					rv = 0;
2432 	int					s;
2433 
2434 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page address: %d read: %d type: %x\n",
2435 	    DEVNAME(sc), address, read, hdr->page_type);
2436 
2437 	page_length = extended ?
2438 	    letoh16(ehdr->ext_page_length) : hdr->page_length;
2439 
2440 	if (len > MPI_REQUEST_SIZE - sizeof(struct mpi_msg_config_request) ||
2441 	    len < page_length * 4)
2442 		return (1);
2443 
2444 	s = splbio();
2445 	ccb = mpi_get_ccb(sc);
2446 	splx(s);
2447 	if (ccb == NULL) {
2448 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page ccb_get\n", DEVNAME(sc));
2449 		return (1);
2450 	}
2451 
2452 	ccb->ccb_done = mpi_empty_done;
2453 	cq = ccb->ccb_cmd;
2454 
2455 	cq->function = MPI_FUNCTION_CONFIG;
2456 	cq->msg_context = htole32(ccb->ccb_id);
2457 
2458 	cq->action = (read ? MPI_CONFIG_REQ_ACTION_PAGE_READ_CURRENT :
2459 	    MPI_CONFIG_REQ_ACTION_PAGE_WRITE_CURRENT);
2460 
2461 	if (extended) {
2462 		cq->config_header.page_version = ehdr->page_version;
2463 		cq->config_header.page_number = ehdr->page_number;
2464 		cq->config_header.page_type = ehdr->page_type;
2465 		cq->ext_page_len = ehdr->ext_page_length;
2466 		cq->ext_page_type = ehdr->ext_page_type;
2467 	} else
2468 		cq->config_header = *hdr;
2469 	cq->config_header.page_type &= MPI_CONFIG_REQ_PAGE_TYPE_MASK;
2470 	cq->page_address = htole32(address);
2471 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2472 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
2473 	    (page_length * 4) |
2474 	    (read ? MPI_SGE_FL_DIR_IN : MPI_SGE_FL_DIR_OUT));
2475 
2476 	/* bounce the page via the request space to avoid more bus_dma games */
2477 	dva = ccb->ccb_cmd_dva + sizeof(struct mpi_msg_config_request);
2478 
2479 	cq->page_buffer.sg_hi_addr = htole32((u_int32_t)(dva >> 32));
2480 	cq->page_buffer.sg_lo_addr = htole32((u_int32_t)dva);
2481 
2482 	kva = ccb->ccb_cmd;
2483 	kva += sizeof(struct mpi_msg_config_request);
2484 	if (!read)
2485 		bcopy(page, kva, len);
2486 
2487 	if (mpi_poll(sc, ccb, 50000) != 0) {
2488 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page poll\n", DEVNAME(sc));
2489 		return (1);
2490 	}
2491 
2492 	if (ccb->ccb_rcb == NULL) {
2493 		mpi_put_ccb(sc, ccb);
2494 		return (1);
2495 	}
2496 	cp = ccb->ccb_rcb->rcb_reply;
2497 
2498 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2499 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2500 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2501 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2502 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2503 	    cp->msg_flags);
2504 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2505 	    letoh32(cp->msg_context));
2506 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2507 	    letoh16(cp->ioc_status));
2508 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2509 	    letoh32(cp->ioc_loginfo));
2510 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2511 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2512 	    cp->config_header.page_version,
2513 	    cp->config_header.page_length,
2514 	    cp->config_header.page_number,
2515 	    cp->config_header.page_type);
2516 
2517 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2518 		rv = 1;
2519 	else if (read)
2520 		bcopy(kva, page, len);
2521 
2522 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2523 	mpi_put_ccb(sc, ccb);
2524 
2525 	return (rv);
2526 }
2527