xref: /openbsd-src/sys/dev/ic/mpi.c (revision 2ed347c682971bc026328c899e27d2698e3370a6)
1 /*	$OpenBSD: mpi.c,v 1.95 2008/09/30 23:36:19 dlg Exp $ */
2 
3 /*
4  * Copyright (c) 2005, 2006 David Gwynne <dlg@openbsd.org>
5  * Copyright (c) 2005 Marco Peereboom <marco@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/param.h>
21 #include <sys/systm.h>
22 #include <sys/buf.h>
23 #include <sys/device.h>
24 #include <sys/proc.h>
25 #include <sys/malloc.h>
26 #include <sys/kernel.h>
27 
28 #include <machine/bus.h>
29 
30 #include <scsi/scsi_all.h>
31 #include <scsi/scsiconf.h>
32 
33 #include <dev/ic/mpireg.h>
34 #include <dev/ic/mpivar.h>
35 
36 #ifdef MPI_DEBUG
37 uint32_t	mpi_debug = 0
38 /*		    | MPI_D_CMD */
39 /*		    | MPI_D_INTR */
40 /*		    | MPI_D_MISC */
41 /*		    | MPI_D_DMA */
42 /*		    | MPI_D_IOCTL */
43 /*		    | MPI_D_RW */
44 /*		    | MPI_D_MEM */
45 /*		    | MPI_D_CCB */
46 /*		    | MPI_D_PPR */
47 /*		    | MPI_D_RAID */
48 /*		    | MPI_D_EVT */
49 		;
50 #endif
51 
52 struct cfdriver mpi_cd = {
53 	NULL,
54 	"mpi",
55 	DV_DULL
56 };
57 
58 int			mpi_scsi_cmd(struct scsi_xfer *);
59 void			mpi_scsi_cmd_done(struct mpi_ccb *);
60 void			mpi_minphys(struct buf *bp);
61 int			mpi_scsi_ioctl(struct scsi_link *, u_long, caddr_t,
62 			    int, struct proc *);
63 
64 struct scsi_adapter mpi_switch = {
65 	mpi_scsi_cmd,
66 	mpi_minphys,
67 	NULL,
68 	NULL,
69 	mpi_scsi_ioctl
70 };
71 
72 struct scsi_device mpi_dev = {
73 	NULL,
74 	NULL,
75 	NULL,
76 	NULL
77 };
78 
79 struct mpi_dmamem	*mpi_dmamem_alloc(struct mpi_softc *, size_t);
80 void			mpi_dmamem_free(struct mpi_softc *,
81 			    struct mpi_dmamem *);
82 int			mpi_alloc_ccbs(struct mpi_softc *);
83 struct mpi_ccb		*mpi_get_ccb(struct mpi_softc *);
84 void			mpi_put_ccb(struct mpi_softc *, struct mpi_ccb *);
85 int			mpi_alloc_replies(struct mpi_softc *);
86 void			mpi_push_replies(struct mpi_softc *);
87 
88 void			mpi_start(struct mpi_softc *, struct mpi_ccb *);
89 int			mpi_complete(struct mpi_softc *, struct mpi_ccb *, int);
90 int			mpi_poll(struct mpi_softc *, struct mpi_ccb *, int);
91 int			mpi_reply(struct mpi_softc *, u_int32_t);
92 
93 int			mpi_cfg_spi_port(struct mpi_softc *);
94 void			mpi_squash_ppr(struct mpi_softc *);
95 void			mpi_run_ppr(struct mpi_softc *);
96 int			mpi_ppr(struct mpi_softc *, struct scsi_link *,
97 			    struct mpi_cfg_raid_physdisk *, int, int, int);
98 int			mpi_inq(struct mpi_softc *, u_int16_t, int);
99 
100 void			mpi_timeout_xs(void *);
101 int			mpi_load_xs(struct mpi_ccb *);
102 
103 u_int32_t		mpi_read(struct mpi_softc *, bus_size_t);
104 void			mpi_write(struct mpi_softc *, bus_size_t, u_int32_t);
105 int			mpi_wait_eq(struct mpi_softc *, bus_size_t, u_int32_t,
106 			    u_int32_t);
107 int			mpi_wait_ne(struct mpi_softc *, bus_size_t, u_int32_t,
108 			    u_int32_t);
109 
110 int			mpi_init(struct mpi_softc *);
111 int			mpi_reset_soft(struct mpi_softc *);
112 int			mpi_reset_hard(struct mpi_softc *);
113 
114 int			mpi_handshake_send(struct mpi_softc *, void *, size_t);
115 int			mpi_handshake_recv_dword(struct mpi_softc *,
116 			    u_int32_t *);
117 int			mpi_handshake_recv(struct mpi_softc *, void *, size_t);
118 
119 void			mpi_empty_done(struct mpi_ccb *);
120 
121 int			mpi_iocinit(struct mpi_softc *);
122 int			mpi_iocfacts(struct mpi_softc *);
123 int			mpi_portfacts(struct mpi_softc *);
124 int			mpi_portenable(struct mpi_softc *);
125 void			mpi_get_raid(struct mpi_softc *);
126 int			mpi_fwupload(struct mpi_softc *);
127 
128 int			mpi_eventnotify(struct mpi_softc *);
129 void			mpi_eventnotify_done(struct mpi_ccb *);
130 void			mpi_eventack(struct mpi_softc *,
131 			    struct mpi_msg_event_reply *);
132 void			mpi_eventack_done(struct mpi_ccb *);
133 void			mpi_evt_sas(void *, void *);
134 
135 int			mpi_req_cfg_header(struct mpi_softc *, u_int8_t,
136 			    u_int8_t, u_int32_t, int, void *);
137 int			mpi_req_cfg_page(struct mpi_softc *, u_int32_t, int,
138 			    void *, int, void *, size_t);
139 
140 #define DEVNAME(s)		((s)->sc_dev.dv_xname)
141 
142 #define	dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
143 #define	sizeofa(s)		(sizeof(s) / sizeof((s)[0]))
144 
145 #define mpi_read_db(s)		mpi_read((s), MPI_DOORBELL)
146 #define mpi_write_db(s, v)	mpi_write((s), MPI_DOORBELL, (v))
147 #define mpi_read_intr(s)	mpi_read((s), MPI_INTR_STATUS)
148 #define mpi_write_intr(s, v)	mpi_write((s), MPI_INTR_STATUS, (v))
149 #define mpi_pop_reply(s)	mpi_read((s), MPI_REPLY_QUEUE)
150 #define mpi_push_reply(s, v)	mpi_write((s), MPI_REPLY_QUEUE, (v))
151 
152 #define mpi_wait_db_int(s)	mpi_wait_ne((s), MPI_INTR_STATUS, \
153 				    MPI_INTR_STATUS_DOORBELL, 0)
154 #define mpi_wait_db_ack(s)	mpi_wait_eq((s), MPI_INTR_STATUS, \
155 				    MPI_INTR_STATUS_IOCDOORBELL, 0)
156 
157 #define mpi_cfg_header(_s, _t, _n, _a, _h) \
158 	mpi_req_cfg_header((_s), (_t), (_n), (_a), 0, (_h))
159 #define mpi_ecfg_header(_s, _t, _n, _a, _h) \
160 	mpi_req_cfg_header((_s), (_t), (_n), (_a), 1, (_h))
161 
162 #define mpi_cfg_page(_s, _a, _h, _r, _p, _l) \
163 	mpi_req_cfg_page((_s), (_a), 0, (_h), (_r), (_p), (_l))
164 #define mpi_ecfg_page(_s, _a, _h, _r, _p, _l) \
165 	mpi_req_cfg_page((_s), (_a), 1, (_h), (_r), (_p), (_l))
166 
167 int
168 mpi_attach(struct mpi_softc *sc)
169 {
170 	struct scsibus_attach_args	saa;
171 	struct mpi_ccb			*ccb;
172 
173 	printf("\n");
174 
175 	/* disable interrupts */
176 	mpi_write(sc, MPI_INTR_MASK,
177 	    MPI_INTR_MASK_REPLY | MPI_INTR_MASK_DOORBELL);
178 
179 	if (mpi_init(sc) != 0) {
180 		printf("%s: unable to initialise\n", DEVNAME(sc));
181 		return (1);
182 	}
183 
184 	if (mpi_iocfacts(sc) != 0) {
185 		printf("%s: unable to get iocfacts\n", DEVNAME(sc));
186 		return (1);
187 	}
188 
189 	if (mpi_alloc_ccbs(sc) != 0) {
190 		/* error already printed */
191 		return (1);
192 	}
193 
194 	if (mpi_alloc_replies(sc) != 0) {
195 		printf("%s: unable to allocate reply space\n", DEVNAME(sc));
196 		goto free_ccbs;
197 	}
198 
199 	if (mpi_iocinit(sc) != 0) {
200 		printf("%s: unable to send iocinit\n", DEVNAME(sc));
201 		goto free_ccbs;
202 	}
203 
204 	/* spin until we're operational */
205 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
206 	    MPI_DOORBELL_STATE_OPER) != 0) {
207 		printf("%s: state: 0x%08x\n", DEVNAME(sc),
208 		    mpi_read_db(sc) & MPI_DOORBELL_STATE);
209 		printf("%s: operational state timeout\n", DEVNAME(sc));
210 		goto free_ccbs;
211 	}
212 
213 	mpi_push_replies(sc);
214 
215 	if (mpi_portfacts(sc) != 0) {
216 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
217 		goto free_replies;
218 	}
219 
220 #ifdef notyet
221 	if (mpi_eventnotify(sc) != 0) {
222 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
223 		goto free_replies;
224 	}
225 #endif
226 
227 	if (mpi_portenable(sc) != 0) {
228 		printf("%s: unable to enable port\n", DEVNAME(sc));
229 		goto free_replies;
230 	}
231 
232 	if (mpi_fwupload(sc) != 0) {
233 		printf("%s: unable to upload firmware\n", DEVNAME(sc));
234 		goto free_replies;
235 	}
236 
237 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI) {
238 		if (mpi_cfg_spi_port(sc) != 0)
239 			goto free_replies;
240 		mpi_squash_ppr(sc);
241 	}
242 
243 	/* we should be good to go now, attach scsibus */
244 	sc->sc_link.device = &mpi_dev;
245 	sc->sc_link.adapter = &mpi_switch;
246 	sc->sc_link.adapter_softc = sc;
247 	sc->sc_link.adapter_target = sc->sc_target;
248 	sc->sc_link.adapter_buswidth = sc->sc_buswidth;
249 	sc->sc_link.openings = sc->sc_maxcmds / sc->sc_buswidth;
250 
251 	bzero(&saa, sizeof(saa));
252 	saa.saa_sc_link = &sc->sc_link;
253 
254 	/* config_found() returns the scsibus attached to us */
255 	sc->sc_scsibus = (struct scsibus_softc *) config_found(&sc->sc_dev,
256 	    &saa, scsiprint);
257 
258 	/* get raid pages */
259 	mpi_get_raid(sc);
260 
261 	/* do domain validation */
262 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI)
263 		mpi_run_ppr(sc);
264 
265 	/* enable interrupts */
266 	mpi_write(sc, MPI_INTR_MASK, MPI_INTR_MASK_DOORBELL);
267 
268 	return (0);
269 
270 free_replies:
271 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
272 	    0, PAGE_SIZE, BUS_DMASYNC_POSTREAD);
273 	mpi_dmamem_free(sc, sc->sc_replies);
274 free_ccbs:
275 	while ((ccb = mpi_get_ccb(sc)) != NULL)
276 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
277 	mpi_dmamem_free(sc, sc->sc_requests);
278 	free(sc->sc_ccbs, M_DEVBUF);
279 
280 	return(1);
281 }
282 
283 int
284 mpi_cfg_spi_port(struct mpi_softc *sc)
285 {
286 	struct mpi_cfg_hdr		hdr;
287 	struct mpi_cfg_spi_port_pg1	port;
288 
289 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 1, 0x0,
290 	    &hdr) != 0)
291 		return (1);
292 
293 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port, sizeof(port)) != 0)
294 		return (1);
295 
296 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_spi_port_pg1\n", DEVNAME(sc));
297 	DNPRINTF(MPI_D_MISC, "%s:  port_scsi_id: %d port_resp_ids 0x%04x\n",
298 	    DEVNAME(sc), port.port_scsi_id, letoh16(port.port_resp_ids));
299 	DNPRINTF(MPI_D_MISC, "%s:  on_bus_timer_value: 0x%08x\n", DEVNAME(sc),
300 	    letoh32(port.port_scsi_id));
301 	DNPRINTF(MPI_D_MISC, "%s:  target_config: 0x%02x id_config: 0x%04x\n",
302 	    DEVNAME(sc), port.target_config, letoh16(port.id_config));
303 
304 	if (port.port_scsi_id == sc->sc_target &&
305 	    port.port_resp_ids == htole16(1 << sc->sc_target) &&
306 	    port.on_bus_timer_value != htole32(0x0))
307 		return (0);
308 
309 	DNPRINTF(MPI_D_MISC, "%s: setting port scsi id to %d\n", DEVNAME(sc),
310 	    sc->sc_target);
311 	port.port_scsi_id = sc->sc_target;
312 	port.port_resp_ids = htole16(1 << sc->sc_target);
313 	port.on_bus_timer_value = htole32(0x07000000); /* XXX magic */
314 
315 	if (mpi_cfg_page(sc, 0x0, &hdr, 0, &port, sizeof(port)) != 0) {
316 		printf("%s: unable to configure port scsi id\n", DEVNAME(sc));
317 		return (1);
318 	}
319 
320 	return (0);
321 }
322 
323 void
324 mpi_squash_ppr(struct mpi_softc *sc)
325 {
326 	struct mpi_cfg_hdr		hdr;
327 	struct mpi_cfg_spi_dev_pg1	page;
328 	int				i;
329 
330 	DNPRINTF(MPI_D_PPR, "%s: mpi_squash_ppr\n", DEVNAME(sc));
331 
332 	for (i = 0; i < sc->sc_buswidth; i++) {
333 		if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV,
334 		    1, i, &hdr) != 0)
335 			return;
336 
337 		if (mpi_cfg_page(sc, i, &hdr, 1, &page, sizeof(page)) != 0)
338 			return;
339 
340 		DNPRINTF(MPI_D_PPR, "%s:  target: %d req_params1: 0x%02x "
341 		    "req_offset: 0x%02x req_period: 0x%02x "
342 		    "req_params2: 0x%02x conf: 0x%08x\n", DEVNAME(sc), i,
343 		    page.req_params1, page.req_offset, page.req_period,
344 		    page.req_params2, letoh32(page.configuration));
345 
346 		page.req_params1 = 0x0;
347 		page.req_offset = 0x0;
348 		page.req_period = 0x0;
349 		page.req_params2 = 0x0;
350 		page.configuration = htole32(0x0);
351 
352 		if (mpi_cfg_page(sc, i, &hdr, 0, &page, sizeof(page)) != 0)
353 			return;
354 	}
355 }
356 
357 void
358 mpi_run_ppr(struct mpi_softc *sc)
359 {
360 	struct mpi_cfg_hdr		hdr;
361 	struct mpi_cfg_spi_port_pg0	port_pg;
362 	struct mpi_cfg_ioc_pg3		*physdisk_pg;
363 	struct mpi_cfg_raid_physdisk	*physdisk_list, *physdisk;
364 	size_t				pagelen;
365 	struct scsi_link		*link;
366 	int				i, tries;
367 
368 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 0, 0x0,
369 	    &hdr) != 0) {
370 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch header\n",
371 		    DEVNAME(sc));
372 		return;
373 	}
374 
375 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port_pg, sizeof(port_pg)) != 0) {
376 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch page\n",
377 		    DEVNAME(sc));
378 		return;
379 	}
380 
381 	for (i = 0; i < sc->sc_buswidth; i++) {
382 		link = sc->sc_scsibus->sc_link[i][0];
383 		if (link == NULL)
384 			continue;
385 
386 		/* do not ppr volumes */
387 		if (link->flags & SDEV_VIRTUAL)
388 			continue;
389 
390 		tries = 0;
391 		while (mpi_ppr(sc, link, NULL, port_pg.min_period,
392 		    port_pg.max_offset, tries) == EAGAIN)
393 			tries++;
394 	}
395 
396 	if ((sc->sc_flags & MPI_F_RAID) == 0)
397 		return;
398 
399 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 3, 0x0,
400 	    &hdr) != 0) {
401 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
402 		    "fetch ioc pg 3 header\n", DEVNAME(sc));
403 		return;
404 	}
405 
406 	pagelen = hdr.page_length * 4; /* dwords to bytes */
407 	physdisk_pg = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
408 	if (physdisk_pg == NULL) {
409 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
410 		    "allocate ioc pg 3\n", DEVNAME(sc));
411 		return;
412 	}
413 	physdisk_list = (struct mpi_cfg_raid_physdisk *)(physdisk_pg + 1);
414 
415 	if (mpi_cfg_page(sc, 0, &hdr, 1, physdisk_pg, pagelen) != 0) {
416 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s: mpi_run_ppr unable to "
417 		    "fetch ioc page 3\n", DEVNAME(sc));
418 		goto out;
419 	}
420 
421 	DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  no_phys_disks: %d\n", DEVNAME(sc),
422 	    physdisk_pg->no_phys_disks);
423 
424 	for (i = 0; i < physdisk_pg->no_phys_disks; i++) {
425 		physdisk = &physdisk_list[i];
426 
427 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  id: %d bus: %d ioc: %d "
428 		    "num: %d\n", DEVNAME(sc), physdisk->phys_disk_id,
429 		    physdisk->phys_disk_bus, physdisk->phys_disk_ioc,
430 		    physdisk->phys_disk_num);
431 
432 		if (physdisk->phys_disk_ioc != sc->sc_ioc_number)
433 			continue;
434 
435 		tries = 0;
436 		while (mpi_ppr(sc, NULL, physdisk, port_pg.min_period,
437 		    port_pg.max_offset, tries) == EAGAIN)
438 			tries++;
439 	}
440 
441 out:
442 	free(physdisk_pg, M_TEMP);
443 }
444 
445 int
446 mpi_ppr(struct mpi_softc *sc, struct scsi_link *link,
447     struct mpi_cfg_raid_physdisk *physdisk, int period, int offset, int try)
448 {
449 	struct mpi_cfg_hdr		hdr0, hdr1;
450 	struct mpi_cfg_spi_dev_pg0	pg0;
451 	struct mpi_cfg_spi_dev_pg1	pg1;
452 	u_int32_t			address;
453 	int				id;
454 	int				raid = 0;
455 
456 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr period: %d offset: %d try: %d "
457 	    "link quirks: 0x%x\n", DEVNAME(sc), period, offset, try,
458 	    link->quirks);
459 
460 	if (try >= 3)
461 		return (EIO);
462 
463 	if (physdisk == NULL) {
464 		if ((link->inqdata.device & SID_TYPE) == T_PROCESSOR)
465 			return (EIO);
466 
467 		address = link->target;
468 		id = link->target;
469 	} else {
470 		raid = 1;
471 		address = (physdisk->phys_disk_bus << 8) |
472 		    (physdisk->phys_disk_id);
473 		id = physdisk->phys_disk_num;
474 	}
475 
476 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 0,
477 	    address, &hdr0) != 0) {
478 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 0\n",
479 		    DEVNAME(sc));
480 		return (EIO);
481 	}
482 
483 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 1,
484 	    address, &hdr1) != 0) {
485 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 1\n",
486 		    DEVNAME(sc));
487 		return (EIO);
488 	}
489 
490 #ifdef MPI_DEBUG
491 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
492 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 0\n",
493 		    DEVNAME(sc));
494 		return (EIO);
495 	}
496 
497 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
498 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
499 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
500 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
501 #endif
502 
503 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
504 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 1\n",
505 		    DEVNAME(sc));
506 		return (EIO);
507 	}
508 
509 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
510 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
511 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
512 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
513 
514 	pg1.req_params1 = 0;
515 	pg1.req_offset = offset;
516 	pg1.req_period = period;
517 	pg1.req_params2 &= ~MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH;
518 
519 	if (raid || !(link->quirks & SDEV_NOSYNC)) {
520 		pg1.req_params2 |= MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH_WIDE;
521 
522 		switch (try) {
523 		case 0: /* U320 */
524 			break;
525 		case 1: /* U160 */
526 			pg1.req_period = 0x09;
527 			break;
528 		case 2: /* U80 */
529 			pg1.req_period = 0x0a;
530 			break;
531 		}
532 
533 		if (pg1.req_period < 0x09) {
534 			/* Ultra320: enable QAS & PACKETIZED */
535 			pg1.req_params1 |= MPI_CFG_SPI_DEV_1_REQPARAMS_QAS |
536 			    MPI_CFG_SPI_DEV_1_REQPARAMS_PACKETIZED;
537 		}
538 		if (pg1.req_period < 0xa) {
539 			/* >= Ultra160: enable dual xfers */
540 			pg1.req_params1 |=
541 			    MPI_CFG_SPI_DEV_1_REQPARAMS_DUALXFERS;
542 		}
543 	}
544 
545 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
546 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
547 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
548 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
549 
550 	if (mpi_cfg_page(sc, address, &hdr1, 0, &pg1, sizeof(pg1)) != 0) {
551 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to write page 1\n",
552 		    DEVNAME(sc));
553 		return (EIO);
554 	}
555 
556 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
557 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 1\n",
558 		    DEVNAME(sc));
559 		return (EIO);
560 	}
561 
562 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
563 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
564 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
565 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
566 
567 	if (mpi_inq(sc, id, raid) != 0) {
568 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to do inquiry against "
569 		    "target %d\n", DEVNAME(sc), link->target);
570 		return (EIO);
571 	}
572 
573 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
574 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 0 after "
575 		    "inquiry\n", DEVNAME(sc));
576 		return (EIO);
577 	}
578 
579 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
580 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
581 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
582 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
583 
584 	if (!(letoh32(pg0.information) & 0x07) && (try == 0)) {
585 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U320 ppr rejected\n",
586 		    DEVNAME(sc));
587 		return (EAGAIN);
588 	}
589 
590 	if ((((letoh32(pg0.information) >> 8) & 0xff) > 0x09) && (try == 1)) {
591 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U160 ppr rejected\n",
592 		    DEVNAME(sc));
593 		return (EAGAIN);
594 	}
595 
596 	if (letoh32(pg0.information) & 0x0e) {
597 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr ppr rejected: %0x\n",
598 		    DEVNAME(sc), letoh32(pg0.information));
599 		return (EAGAIN);
600 	}
601 
602 	switch(pg0.neg_period) {
603 	case 0x08:
604 		period = 160;
605 		break;
606 	case 0x09:
607 		period = 80;
608 		break;
609 	case 0x0a:
610 		period = 40;
611 		break;
612 	case 0x0b:
613 		period = 20;
614 		break;
615 	case 0x0c:
616 		period = 10;
617 		break;
618 	default:
619 		period = 0;
620 		break;
621 	}
622 
623 	printf("%s: %s %d %s at %dMHz width %dbit offset %d "
624 	    "QAS %d DT %d IU %d\n", DEVNAME(sc), raid ? "phys disk" : "target",
625 	    id, period ? "Sync" : "Async", period,
626 	    (pg0.neg_params2 & MPI_CFG_SPI_DEV_0_NEGPARAMS_WIDTH_WIDE) ? 16 : 8,
627 	    pg0.neg_offset,
628 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_QAS) ? 1 : 0,
629 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_DUALXFERS) ? 1 : 0,
630 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_PACKETIZED) ? 1 : 0);
631 
632 	return (0);
633 }
634 
635 int
636 mpi_inq(struct mpi_softc *sc, u_int16_t target, int physdisk)
637 {
638 	struct mpi_ccb			*ccb;
639 	struct scsi_inquiry		inq;
640 	struct {
641 		struct mpi_msg_scsi_io		io;
642 		struct mpi_sge			sge;
643 		struct scsi_inquiry_data	inqbuf;
644 		struct scsi_sense_data		sense;
645 	} __packed			*bundle;
646 	struct mpi_msg_scsi_io		*io;
647 	struct mpi_sge			*sge;
648 	u_int64_t			addr;
649 
650 	DNPRINTF(MPI_D_PPR, "%s: mpi_inq\n", DEVNAME(sc));
651 
652 	bzero(&inq, sizeof(inq));
653 	inq.opcode = INQUIRY;
654 	_lto2b(sizeof(struct scsi_inquiry_data), inq.length);
655 
656 	ccb = mpi_get_ccb(sc);
657 	if (ccb == NULL)
658 		return (1);
659 
660 	ccb->ccb_done = mpi_empty_done;
661 
662 	bundle = ccb->ccb_cmd;
663 	io = &bundle->io;
664 	sge = &bundle->sge;
665 
666 	io->function = physdisk ? MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH :
667 	    MPI_FUNCTION_SCSI_IO_REQUEST;
668 	/*
669 	 * bus is always 0
670 	 * io->bus = htole16(sc->sc_bus);
671 	 */
672 	io->target_id = target;
673 
674 	io->cdb_length = sizeof(inq);
675 	io->sense_buf_len = sizeof(struct scsi_sense_data);
676 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
677 
678 	io->msg_context = htole32(ccb->ccb_id);
679 
680 	/*
681 	 * always lun 0
682 	 * io->lun[0] = htobe16(link->lun);
683 	 */
684 
685 	io->direction = MPI_SCSIIO_DIR_READ;
686 	io->tagging = MPI_SCSIIO_ATTR_NO_DISCONNECT;
687 
688 	bcopy(&inq, io->cdb, sizeof(inq));
689 
690 	io->data_length = htole32(sizeof(struct scsi_inquiry_data));
691 
692 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
693 	    ((u_int8_t *)&bundle->sense - (u_int8_t *)bundle));
694 
695 	sge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64 |
696 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
697 	    (u_int32_t)sizeof(inq));
698 
699 	addr = ccb->ccb_cmd_dva +
700 	    ((u_int8_t *)&bundle->inqbuf - (u_int8_t *)bundle);
701 	sge->sg_hi_addr = htole32((u_int32_t)(addr >> 32));
702 	sge->sg_lo_addr = htole32((u_int32_t)addr);
703 
704 	if (mpi_poll(sc, ccb, 5000) != 0)
705 		return (1);
706 
707 	if (ccb->ccb_rcb != NULL)
708 		mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
709 
710 	mpi_put_ccb(sc, ccb);
711 
712 	return (0);
713 }
714 
715 void
716 mpi_detach(struct mpi_softc *sc)
717 {
718 
719 }
720 
721 int
722 mpi_intr(void *arg)
723 {
724 	struct mpi_softc		*sc = arg;
725 	u_int32_t			reg;
726 	int				rv = 0;
727 
728 	while ((reg = mpi_pop_reply(sc)) != 0xffffffff) {
729 		mpi_reply(sc, reg);
730 		rv = 1;
731 	}
732 
733 	return (rv);
734 }
735 
736 int
737 mpi_reply(struct mpi_softc *sc, u_int32_t reg)
738 {
739 	struct mpi_ccb			*ccb;
740 	struct mpi_rcb			*rcb = NULL;
741 	struct mpi_msg_reply		*reply = NULL;
742 	u_int32_t			reply_dva;
743 	int				id;
744 	int				i;
745 
746 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply reg: 0x%08x\n", DEVNAME(sc), reg);
747 
748 	if (reg & MPI_REPLY_QUEUE_ADDRESS) {
749 		bus_dmamap_sync(sc->sc_dmat,
750 		    MPI_DMA_MAP(sc->sc_replies), 0, PAGE_SIZE,
751 		    BUS_DMASYNC_POSTREAD);
752 
753 		reply_dva = (reg & MPI_REPLY_QUEUE_ADDRESS_MASK) << 1;
754 
755 		i = (reply_dva - (u_int32_t)MPI_DMA_DVA(sc->sc_replies)) /
756 		    MPI_REPLY_SIZE;
757 		rcb = &sc->sc_rcbs[i];
758 		reply = rcb->rcb_reply;
759 
760 		id = letoh32(reply->msg_context);
761 
762 		bus_dmamap_sync(sc->sc_dmat,
763 		    MPI_DMA_MAP(sc->sc_replies), 0, PAGE_SIZE,
764 		    BUS_DMASYNC_PREREAD);
765 	} else {
766 		switch (reg & MPI_REPLY_QUEUE_TYPE_MASK) {
767 		case MPI_REPLY_QUEUE_TYPE_INIT:
768 			id = reg & MPI_REPLY_QUEUE_CONTEXT;
769 			break;
770 
771 		default:
772 			panic("%s: unsupported context reply\n",
773 			    DEVNAME(sc));
774 		}
775 	}
776 
777 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply id: %d reply: %p\n",
778 	    DEVNAME(sc), id, reply);
779 
780 	ccb = &sc->sc_ccbs[id];
781 
782 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
783 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
784 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
785 	ccb->ccb_state = MPI_CCB_READY;
786 	ccb->ccb_rcb = rcb;
787 
788 	ccb->ccb_done(ccb);
789 
790 	return (id);
791 }
792 
793 struct mpi_dmamem *
794 mpi_dmamem_alloc(struct mpi_softc *sc, size_t size)
795 {
796 	struct mpi_dmamem		*mdm;
797 	int				nsegs;
798 
799 	mdm = malloc(sizeof(struct mpi_dmamem), M_DEVBUF, M_NOWAIT | M_ZERO);
800 	if (mdm == NULL)
801 		return (NULL);
802 
803 	mdm->mdm_size = size;
804 
805 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
806 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mdm->mdm_map) != 0)
807 		goto mdmfree;
808 
809 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &mdm->mdm_seg,
810 	    1, &nsegs, BUS_DMA_NOWAIT) != 0)
811 		goto destroy;
812 
813 	if (bus_dmamem_map(sc->sc_dmat, &mdm->mdm_seg, nsegs, size,
814 	    &mdm->mdm_kva, BUS_DMA_NOWAIT) != 0)
815 		goto free;
816 
817 	if (bus_dmamap_load(sc->sc_dmat, mdm->mdm_map, mdm->mdm_kva, size,
818 	    NULL, BUS_DMA_NOWAIT) != 0)
819 		goto unmap;
820 
821 	bzero(mdm->mdm_kva, size);
822 
823 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_alloc size: %d mdm: %#x "
824 	    "map: %#x nsegs: %d segs: %#x kva: %x\n",
825 	    DEVNAME(sc), size, mdm->mdm_map, nsegs, mdm->mdm_seg, mdm->mdm_kva);
826 
827 	return (mdm);
828 
829 unmap:
830 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, size);
831 free:
832 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
833 destroy:
834 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
835 mdmfree:
836 	free(mdm, M_DEVBUF);
837 
838 	return (NULL);
839 }
840 
841 void
842 mpi_dmamem_free(struct mpi_softc *sc, struct mpi_dmamem *mdm)
843 {
844 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_free %#x\n", DEVNAME(sc), mdm);
845 
846 	bus_dmamap_unload(sc->sc_dmat, mdm->mdm_map);
847 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, mdm->mdm_size);
848 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
849 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
850 	free(mdm, M_DEVBUF);
851 }
852 
853 int
854 mpi_alloc_ccbs(struct mpi_softc *sc)
855 {
856 	struct mpi_ccb			*ccb;
857 	u_int8_t			*cmd;
858 	int				i;
859 
860 	TAILQ_INIT(&sc->sc_ccb_free);
861 
862 	sc->sc_ccbs = malloc(sizeof(struct mpi_ccb) * sc->sc_maxcmds,
863 	    M_DEVBUF, M_WAITOK | M_CANFAIL | M_ZERO);
864 	if (sc->sc_ccbs == NULL) {
865 		printf("%s: unable to allocate ccbs\n", DEVNAME(sc));
866 		return (1);
867 	}
868 
869 	sc->sc_requests = mpi_dmamem_alloc(sc,
870 	    MPI_REQUEST_SIZE * sc->sc_maxcmds);
871 	if (sc->sc_requests == NULL) {
872 		printf("%s: unable to allocate ccb dmamem\n", DEVNAME(sc));
873 		goto free_ccbs;
874 	}
875 	cmd = MPI_DMA_KVA(sc->sc_requests);
876 	bzero(cmd, MPI_REQUEST_SIZE * sc->sc_maxcmds);
877 
878 	for (i = 0; i < sc->sc_maxcmds; i++) {
879 		ccb = &sc->sc_ccbs[i];
880 
881 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
882 		    sc->sc_max_sgl_len, MAXPHYS, 0,
883 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
884 		    &ccb->ccb_dmamap) != 0) {
885 			printf("%s: unable to create dma map\n", DEVNAME(sc));
886 			goto free_maps;
887 		}
888 
889 		ccb->ccb_sc = sc;
890 		ccb->ccb_id = i;
891 		ccb->ccb_offset = MPI_REQUEST_SIZE * i;
892 
893 		ccb->ccb_cmd = &cmd[ccb->ccb_offset];
894 		ccb->ccb_cmd_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_requests) +
895 		    ccb->ccb_offset;
896 
897 		DNPRINTF(MPI_D_CCB, "%s: mpi_alloc_ccbs(%d) ccb: %#x map: %#x "
898 		    "sc: %#x id: %#x offs: %#x cmd: %#x dva: %#x\n",
899 		    DEVNAME(sc), i, ccb, ccb->ccb_dmamap, ccb->ccb_sc,
900 		    ccb->ccb_id, ccb->ccb_offset, ccb->ccb_cmd,
901 		    ccb->ccb_cmd_dva);
902 
903 		mpi_put_ccb(sc, ccb);
904 	}
905 
906 	return (0);
907 
908 free_maps:
909 	while ((ccb = mpi_get_ccb(sc)) != NULL)
910 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
911 
912 	mpi_dmamem_free(sc, sc->sc_requests);
913 free_ccbs:
914 	free(sc->sc_ccbs, M_DEVBUF);
915 
916 	return (1);
917 }
918 
919 struct mpi_ccb *
920 mpi_get_ccb(struct mpi_softc *sc)
921 {
922 	struct mpi_ccb			*ccb;
923 
924 	ccb = TAILQ_FIRST(&sc->sc_ccb_free);
925 	if (ccb == NULL) {
926 		DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb == NULL\n", DEVNAME(sc));
927 		return (NULL);
928 	}
929 
930 	TAILQ_REMOVE(&sc->sc_ccb_free, ccb, ccb_link);
931 
932 	ccb->ccb_state = MPI_CCB_READY;
933 
934 	DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb %#x\n", DEVNAME(sc), ccb);
935 
936 	return (ccb);
937 }
938 
939 void
940 mpi_put_ccb(struct mpi_softc *sc, struct mpi_ccb *ccb)
941 {
942 	DNPRINTF(MPI_D_CCB, "%s: mpi_put_ccb %#x\n", DEVNAME(sc), ccb);
943 
944 	ccb->ccb_state = MPI_CCB_FREE;
945 	ccb->ccb_xs = NULL;
946 	ccb->ccb_done = NULL;
947 	bzero(ccb->ccb_cmd, MPI_REQUEST_SIZE);
948 	TAILQ_INSERT_TAIL(&sc->sc_ccb_free, ccb, ccb_link);
949 }
950 
951 int
952 mpi_alloc_replies(struct mpi_softc *sc)
953 {
954 	DNPRINTF(MPI_D_MISC, "%s: mpi_alloc_replies\n", DEVNAME(sc));
955 
956 	sc->sc_rcbs = malloc(MPI_REPLY_COUNT * sizeof(struct mpi_rcb),
957 	    M_DEVBUF, M_WAITOK|M_CANFAIL);
958 	if (sc->sc_rcbs == NULL)
959 		return (1);
960 
961 	sc->sc_replies = mpi_dmamem_alloc(sc, PAGE_SIZE);
962 	if (sc->sc_replies == NULL) {
963 		free(sc->sc_rcbs, M_DEVBUF);
964 		return (1);
965 	}
966 
967 	return (0);
968 }
969 
970 void
971 mpi_push_replies(struct mpi_softc *sc)
972 {
973 	struct mpi_rcb			*rcb;
974 	char				*kva = MPI_DMA_KVA(sc->sc_replies);
975 	int				i;
976 
977 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
978 	    0, PAGE_SIZE, BUS_DMASYNC_PREREAD);
979 
980 	for (i = 0; i < MPI_REPLY_COUNT; i++) {
981 		rcb = &sc->sc_rcbs[i];
982 
983 		rcb->rcb_reply = kva + MPI_REPLY_SIZE * i;
984 		rcb->rcb_reply_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_replies) +
985 		    MPI_REPLY_SIZE * i;
986 		mpi_push_reply(sc, rcb->rcb_reply_dva);
987 	}
988 }
989 
990 void
991 mpi_start(struct mpi_softc *sc, struct mpi_ccb *ccb)
992 {
993 	DNPRINTF(MPI_D_RW, "%s: mpi_start %#x\n", DEVNAME(sc),
994 	    ccb->ccb_cmd_dva);
995 
996 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
997 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
998 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
999 
1000 	ccb->ccb_state = MPI_CCB_QUEUED;
1001 	mpi_write(sc, MPI_REQ_QUEUE, ccb->ccb_cmd_dva);
1002 }
1003 
1004 int
1005 mpi_complete(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1006 {
1007 	u_int32_t			reg;
1008 	int				id = -1;
1009 
1010 	DNPRINTF(MPI_D_INTR, "%s: mpi_complete timeout %d\n", DEVNAME(sc),
1011 	    timeout);
1012 
1013 	do {
1014 		reg = mpi_pop_reply(sc);
1015 		if (reg == 0xffffffff) {
1016 			if (timeout-- == 0)
1017 				return (1);
1018 
1019 			delay(1000);
1020 			continue;
1021 		}
1022 
1023 		id = mpi_reply(sc, reg);
1024 
1025 	} while (ccb->ccb_id != id);
1026 
1027 	return (0);
1028 }
1029 
1030 int
1031 mpi_poll(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1032 {
1033 	int				error;
1034 	int				s;
1035 
1036 	DNPRINTF(MPI_D_CMD, "%s: mpi_poll\n", DEVNAME(sc));
1037 
1038 	s = splbio();
1039 	mpi_start(sc, ccb);
1040 	error = mpi_complete(sc, ccb, timeout);
1041 	splx(s);
1042 
1043 	return (error);
1044 }
1045 
1046 int
1047 mpi_scsi_cmd(struct scsi_xfer *xs)
1048 {
1049 	struct scsi_link		*link = xs->sc_link;
1050 	struct mpi_softc		*sc = link->adapter_softc;
1051 	struct mpi_ccb			*ccb;
1052 	struct mpi_ccb_bundle		*mcb;
1053 	struct mpi_msg_scsi_io		*io;
1054 	int				s;
1055 
1056 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd\n", DEVNAME(sc));
1057 
1058 	if (xs->cmdlen > MPI_CDB_LEN) {
1059 		DNPRINTF(MPI_D_CMD, "%s: CBD too big %d\n",
1060 		    DEVNAME(sc), xs->cmdlen);
1061 		bzero(&xs->sense, sizeof(xs->sense));
1062 		xs->sense.error_code = SSD_ERRCODE_VALID | 0x70;
1063 		xs->sense.flags = SKEY_ILLEGAL_REQUEST;
1064 		xs->sense.add_sense_code = 0x20;
1065 		xs->error = XS_SENSE;
1066 		s = splbio();
1067 		scsi_done(xs);
1068 		splx(s);
1069 		return (COMPLETE);
1070 	}
1071 
1072 	s = splbio();
1073 	ccb = mpi_get_ccb(sc);
1074 	splx(s);
1075 	if (ccb == NULL) {
1076 		xs->error = XS_DRIVER_STUFFUP;
1077 		s = splbio();
1078 		scsi_done(xs);
1079 		splx(s);
1080 		return (COMPLETE);
1081 	}
1082 	DNPRINTF(MPI_D_CMD, "%s: ccb_id: %d xs->flags: 0x%x\n",
1083 	    DEVNAME(sc), ccb->ccb_id, xs->flags);
1084 
1085 	ccb->ccb_xs = xs;
1086 	ccb->ccb_done = mpi_scsi_cmd_done;
1087 
1088 	mcb = ccb->ccb_cmd;
1089 	io = &mcb->mcb_io;
1090 
1091 	io->function = MPI_FUNCTION_SCSI_IO_REQUEST;
1092 	/*
1093 	 * bus is always 0
1094 	 * io->bus = htole16(sc->sc_bus);
1095 	 */
1096 	io->target_id = link->target;
1097 
1098 	io->cdb_length = xs->cmdlen;
1099 	io->sense_buf_len = sizeof(xs->sense);
1100 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
1101 
1102 	io->msg_context = htole32(ccb->ccb_id);
1103 
1104 	io->lun[0] = htobe16(link->lun);
1105 
1106 	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
1107 	case SCSI_DATA_IN:
1108 		io->direction = MPI_SCSIIO_DIR_READ;
1109 		break;
1110 	case SCSI_DATA_OUT:
1111 		io->direction = MPI_SCSIIO_DIR_WRITE;
1112 		break;
1113 	default:
1114 		io->direction = MPI_SCSIIO_DIR_NONE;
1115 		break;
1116 	}
1117 
1118 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SCSI &&
1119 	    (link->quirks & SDEV_NOTAGS))
1120 		io->tagging = MPI_SCSIIO_ATTR_UNTAGGED;
1121 	else
1122 		io->tagging = MPI_SCSIIO_ATTR_SIMPLE_Q;
1123 
1124 	bcopy(xs->cmd, io->cdb, xs->cmdlen);
1125 
1126 	io->data_length = htole32(xs->datalen);
1127 
1128 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
1129 	    ((u_int8_t *)&mcb->mcb_sense - (u_int8_t *)mcb));
1130 
1131 	if (mpi_load_xs(ccb) != 0) {
1132 		xs->error = XS_DRIVER_STUFFUP;
1133 		s = splbio();
1134 		mpi_put_ccb(sc, ccb);
1135 		scsi_done(xs);
1136 		splx(s);
1137 		return (COMPLETE);
1138 	}
1139 
1140 	timeout_set(&xs->stimeout, mpi_timeout_xs, ccb);
1141 
1142 	if (xs->flags & SCSI_POLL) {
1143 		if (mpi_poll(sc, ccb, xs->timeout) != 0)
1144 			xs->error = XS_DRIVER_STUFFUP;
1145 		return (COMPLETE);
1146 	}
1147 
1148 	s = splbio();
1149 	mpi_start(sc, ccb);
1150 	splx(s);
1151 	return (SUCCESSFULLY_QUEUED);
1152 }
1153 
1154 void
1155 mpi_scsi_cmd_done(struct mpi_ccb *ccb)
1156 {
1157 	struct mpi_softc		*sc = ccb->ccb_sc;
1158 	struct scsi_xfer		*xs = ccb->ccb_xs;
1159 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1160 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1161 	struct mpi_msg_scsi_io_error	*sie;
1162 
1163 	if (xs->datalen != 0) {
1164 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1165 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1166 		    BUS_DMASYNC_POSTWRITE);
1167 
1168 		bus_dmamap_unload(sc->sc_dmat, dmap);
1169 	}
1170 
1171 	/* timeout_del */
1172 	xs->error = XS_NOERROR;
1173 	xs->resid = 0;
1174 	xs->flags |= ITSDONE;
1175 
1176 	if (ccb->ccb_rcb == NULL) {
1177 		/* no scsi error, we're ok so drop out early */
1178 		xs->status = SCSI_OK;
1179 		mpi_put_ccb(sc, ccb);
1180 		scsi_done(xs);
1181 		return;
1182 	}
1183 
1184 	sie = ccb->ccb_rcb->rcb_reply;
1185 
1186 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd_done xs cmd: 0x%02x len: %d "
1187 	    "flags 0x%x\n", DEVNAME(sc), xs->cmd->opcode, xs->datalen,
1188 	    xs->flags);
1189 	DNPRINTF(MPI_D_CMD, "%s:  target_id: %d bus: %d msg_length: %d "
1190 	    "function: 0x%02x\n", DEVNAME(sc), sie->target_id, sie->bus,
1191 	    sie->msg_length, sie->function);
1192 	DNPRINTF(MPI_D_CMD, "%s:  cdb_length: %d sense_buf_length: %d "
1193 	    "msg_flags: 0x%02x\n", DEVNAME(sc), sie->cdb_length,
1194 	    sie->sense_buf_len, sie->msg_flags);
1195 	DNPRINTF(MPI_D_CMD, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1196 	    letoh32(sie->msg_context));
1197 	DNPRINTF(MPI_D_CMD, "%s:  scsi_status: 0x%02x scsi_state: 0x%02x "
1198 	    "ioc_status: 0x%04x\n", DEVNAME(sc), sie->scsi_status,
1199 	    sie->scsi_state, letoh16(sie->ioc_status));
1200 	DNPRINTF(MPI_D_CMD, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1201 	    letoh32(sie->ioc_loginfo));
1202 	DNPRINTF(MPI_D_CMD, "%s:  transfer_count: %d\n", DEVNAME(sc),
1203 	    letoh32(sie->transfer_count));
1204 	DNPRINTF(MPI_D_CMD, "%s:  sense_count: %d\n", DEVNAME(sc),
1205 	    letoh32(sie->sense_count));
1206 	DNPRINTF(MPI_D_CMD, "%s:  response_info: 0x%08x\n", DEVNAME(sc),
1207 	    letoh32(sie->response_info));
1208 	DNPRINTF(MPI_D_CMD, "%s:  tag: 0x%04x\n", DEVNAME(sc),
1209 	    letoh16(sie->tag));
1210 
1211 	xs->status = sie->scsi_status;
1212 	switch (letoh16(sie->ioc_status)) {
1213 	case MPI_IOCSTATUS_SCSI_DATA_UNDERRUN:
1214 		xs->resid = xs->datalen - letoh32(sie->transfer_count);
1215 		if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_NO_SCSI_STATUS) {
1216 			xs->error = XS_DRIVER_STUFFUP;
1217 			break;
1218 		}
1219 		/* FALLTHROUGH */
1220 	case MPI_IOCSTATUS_SUCCESS:
1221 	case MPI_IOCSTATUS_SCSI_RECOVERED_ERROR:
1222 		switch (xs->status) {
1223 		case SCSI_OK:
1224 			xs->resid = 0;
1225 			break;
1226 
1227 		case SCSI_CHECK:
1228 			xs->error = XS_SENSE;
1229 			break;
1230 
1231 		case SCSI_BUSY:
1232 		case SCSI_QUEUE_FULL:
1233 			xs->error = XS_BUSY;
1234 			break;
1235 
1236 		default:
1237 			xs->error = XS_DRIVER_STUFFUP;
1238 			break;
1239 		}
1240 		break;
1241 
1242 	case MPI_IOCSTATUS_BUSY:
1243 	case MPI_IOCSTATUS_INSUFFICIENT_RESOURCES:
1244 		xs->error = XS_BUSY;
1245 		break;
1246 
1247 	case MPI_IOCSTATUS_SCSI_INVALID_BUS:
1248 	case MPI_IOCSTATUS_SCSI_INVALID_TARGETID:
1249 	case MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1250 		xs->error = XS_SELTIMEOUT;
1251 		break;
1252 
1253 	default:
1254 		xs->error = XS_DRIVER_STUFFUP;
1255 		break;
1256 	}
1257 
1258 	if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_AUTOSENSE_VALID)
1259 		bcopy(&mcb->mcb_sense, &xs->sense, sizeof(xs->sense));
1260 
1261 	DNPRINTF(MPI_D_CMD, "%s:  xs err: 0x%02x status: %d\n", DEVNAME(sc),
1262 	    xs->error, xs->status);
1263 
1264 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
1265 	mpi_put_ccb(sc, ccb);
1266 	scsi_done(xs);
1267 }
1268 
1269 void
1270 mpi_timeout_xs(void *arg)
1271 {
1272 	/* XXX */
1273 }
1274 
1275 int
1276 mpi_load_xs(struct mpi_ccb *ccb)
1277 {
1278 	struct mpi_softc		*sc = ccb->ccb_sc;
1279 	struct scsi_xfer		*xs = ccb->ccb_xs;
1280 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1281 	struct mpi_msg_scsi_io		*io = &mcb->mcb_io;
1282 	struct mpi_sge			*sge, *nsge = &mcb->mcb_sgl[0];
1283 	struct mpi_sge			*ce = NULL, *nce;
1284 	u_int64_t			ce_dva;
1285 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1286 	u_int32_t			addr, flags;
1287 	int				i, error;
1288 
1289 	if (xs->datalen == 0) {
1290 		nsge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
1291 		    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
1292 		return (0);
1293 	}
1294 
1295 	error = bus_dmamap_load(sc->sc_dmat, dmap,
1296 	    xs->data, xs->datalen, NULL,
1297 	    (xs->flags & SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1298 	if (error) {
1299 		printf("%s: error %d loading dmamap\n", DEVNAME(sc), error);
1300 		return (1);
1301 	}
1302 
1303 	flags = MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64;
1304 	if (xs->flags & SCSI_DATA_OUT)
1305 		flags |= MPI_SGE_FL_DIR_OUT;
1306 
1307 	if (dmap->dm_nsegs > sc->sc_first_sgl_len) {
1308 		ce = &mcb->mcb_sgl[sc->sc_first_sgl_len - 1];
1309 		io->chain_offset = ((u_int8_t *)ce - (u_int8_t *)io) / 4;
1310 	}
1311 
1312 	for (i = 0; i < dmap->dm_nsegs; i++) {
1313 
1314 		if (nsge == ce) {
1315 			nsge++;
1316 			sge->sg_hdr |= htole32(MPI_SGE_FL_LAST);
1317 
1318 			DNPRINTF(MPI_D_DMA, "%s:   - 0x%08x 0x%08x 0x%08x\n",
1319 			    DEVNAME(sc), sge->sg_hdr,
1320 			    sge->sg_hi_addr, sge->sg_lo_addr);
1321 
1322 			if ((dmap->dm_nsegs - i) > sc->sc_chain_len) {
1323 				nce = &nsge[sc->sc_chain_len - 1];
1324 				addr = ((u_int8_t *)nce - (u_int8_t *)nsge) / 4;
1325 				addr = addr << 16 |
1326 				    sizeof(struct mpi_sge) * sc->sc_chain_len;
1327 			} else {
1328 				nce = NULL;
1329 				addr = sizeof(struct mpi_sge) *
1330 				    (dmap->dm_nsegs - i);
1331 			}
1332 
1333 			ce->sg_hdr = htole32(MPI_SGE_FL_TYPE_CHAIN |
1334 			    MPI_SGE_FL_SIZE_64 | addr);
1335 
1336 			ce_dva = ccb->ccb_cmd_dva +
1337 			    ((u_int8_t *)nsge - (u_int8_t *)mcb);
1338 
1339 			addr = (u_int32_t)(ce_dva >> 32);
1340 			ce->sg_hi_addr = htole32(addr);
1341 			addr = (u_int32_t)ce_dva;
1342 			ce->sg_lo_addr = htole32(addr);
1343 
1344 			DNPRINTF(MPI_D_DMA, "%s:  ce: 0x%08x 0x%08x 0x%08x\n",
1345 			    DEVNAME(sc), ce->sg_hdr, ce->sg_hi_addr,
1346 			    ce->sg_lo_addr);
1347 
1348 			ce = nce;
1349 		}
1350 
1351 		DNPRINTF(MPI_D_DMA, "%s:  %d: %d 0x%016llx\n", DEVNAME(sc),
1352 		    i, dmap->dm_segs[i].ds_len,
1353 		    (u_int64_t)dmap->dm_segs[i].ds_addr);
1354 
1355 		sge = nsge;
1356 
1357 		sge->sg_hdr = htole32(flags | dmap->dm_segs[i].ds_len);
1358 		addr = (u_int32_t)((u_int64_t)dmap->dm_segs[i].ds_addr >> 32);
1359 		sge->sg_hi_addr = htole32(addr);
1360 		addr = (u_int32_t)dmap->dm_segs[i].ds_addr;
1361 		sge->sg_lo_addr = htole32(addr);
1362 
1363 		DNPRINTF(MPI_D_DMA, "%s:  %d: 0x%08x 0x%08x 0x%08x\n",
1364 		    DEVNAME(sc), i, sge->sg_hdr, sge->sg_hi_addr,
1365 		    sge->sg_lo_addr);
1366 
1367 		nsge = sge + 1;
1368 	}
1369 
1370 	/* terminate list */
1371 	sge->sg_hdr |= htole32(MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
1372 	    MPI_SGE_FL_EOL);
1373 
1374 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1375 	    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
1376 	    BUS_DMASYNC_PREWRITE);
1377 
1378 	return (0);
1379 }
1380 
1381 void
1382 mpi_minphys(struct buf *bp)
1383 {
1384 	/* XXX */
1385 	if (bp->b_bcount > MAXPHYS)
1386 		bp->b_bcount = MAXPHYS;
1387 	minphys(bp);
1388 }
1389 
1390 int
1391 mpi_scsi_ioctl(struct scsi_link *a, u_long b, caddr_t c, int d, struct proc *e)
1392 {
1393 	return (ENOTTY);
1394 }
1395 
1396 u_int32_t
1397 mpi_read(struct mpi_softc *sc, bus_size_t r)
1398 {
1399 	u_int32_t			rv;
1400 
1401 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1402 	    BUS_SPACE_BARRIER_READ);
1403 	rv = bus_space_read_4(sc->sc_iot, sc->sc_ioh, r);
1404 
1405 	DNPRINTF(MPI_D_RW, "%s: mpi_read %#x %#x\n", DEVNAME(sc), r, rv);
1406 
1407 	return (rv);
1408 }
1409 
1410 void
1411 mpi_write(struct mpi_softc *sc, bus_size_t r, u_int32_t v)
1412 {
1413 	DNPRINTF(MPI_D_RW, "%s: mpi_write %#x %#x\n", DEVNAME(sc), r, v);
1414 
1415 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
1416 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1417 	    BUS_SPACE_BARRIER_WRITE);
1418 }
1419 
1420 int
1421 mpi_wait_eq(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1422     u_int32_t target)
1423 {
1424 	int				i;
1425 
1426 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_eq %#x %#x %#x\n", DEVNAME(sc), r,
1427 	    mask, target);
1428 
1429 	for (i = 0; i < 10000; i++) {
1430 		if ((mpi_read(sc, r) & mask) == target)
1431 			return (0);
1432 		delay(1000);
1433 	}
1434 
1435 	return (1);
1436 }
1437 
1438 int
1439 mpi_wait_ne(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1440     u_int32_t target)
1441 {
1442 	int				i;
1443 
1444 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_ne %#x %#x %#x\n", DEVNAME(sc), r,
1445 	    mask, target);
1446 
1447 	for (i = 0; i < 10000; i++) {
1448 		if ((mpi_read(sc, r) & mask) != target)
1449 			return (0);
1450 		delay(1000);
1451 	}
1452 
1453 	return (1);
1454 }
1455 
1456 int
1457 mpi_init(struct mpi_softc *sc)
1458 {
1459 	u_int32_t			db;
1460 	int				i;
1461 
1462 	/* spin until the IOC leaves the RESET state */
1463 	if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1464 	    MPI_DOORBELL_STATE_RESET) != 0) {
1465 		DNPRINTF(MPI_D_MISC, "%s: mpi_init timeout waiting to leave "
1466 		    "reset state\n", DEVNAME(sc));
1467 		return (1);
1468 	}
1469 
1470 	/* check current ownership */
1471 	db = mpi_read_db(sc);
1472 	if ((db & MPI_DOORBELL_WHOINIT) == MPI_DOORBELL_WHOINIT_PCIPEER) {
1473 		DNPRINTF(MPI_D_MISC, "%s: mpi_init initialised by pci peer\n",
1474 		    DEVNAME(sc));
1475 		return (0);
1476 	}
1477 
1478 	for (i = 0; i < 5; i++) {
1479 		switch (db & MPI_DOORBELL_STATE) {
1480 		case MPI_DOORBELL_STATE_READY:
1481 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is ready\n",
1482 			    DEVNAME(sc));
1483 			return (0);
1484 
1485 		case MPI_DOORBELL_STATE_OPER:
1486 		case MPI_DOORBELL_STATE_FAULT:
1487 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is being "
1488 			    "reset\n" , DEVNAME(sc));
1489 			if (mpi_reset_soft(sc) != 0)
1490 				mpi_reset_hard(sc);
1491 			break;
1492 
1493 		case MPI_DOORBELL_STATE_RESET:
1494 			DNPRINTF(MPI_D_MISC, "%s: mpi_init waiting to come "
1495 			    "out of reset\n", DEVNAME(sc));
1496 			if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1497 			    MPI_DOORBELL_STATE_RESET) != 0)
1498 				return (1);
1499 			break;
1500 		}
1501 		db = mpi_read_db(sc);
1502 	}
1503 
1504 	return (1);
1505 }
1506 
1507 int
1508 mpi_reset_soft(struct mpi_softc *sc)
1509 {
1510 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_soft\n", DEVNAME(sc));
1511 
1512 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1513 		return (1);
1514 
1515 	mpi_write_db(sc,
1516 	    MPI_DOORBELL_FUNCTION(MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET));
1517 	if (mpi_wait_eq(sc, MPI_INTR_STATUS,
1518 	    MPI_INTR_STATUS_IOCDOORBELL, 0) != 0)
1519 		return (1);
1520 
1521 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1522 	    MPI_DOORBELL_STATE_READY) != 0)
1523 		return (1);
1524 
1525 	return (0);
1526 }
1527 
1528 int
1529 mpi_reset_hard(struct mpi_softc *sc)
1530 {
1531 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_hard\n", DEVNAME(sc));
1532 
1533 	/* enable diagnostic register */
1534 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1535 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_1);
1536 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_2);
1537 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_3);
1538 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_4);
1539 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_5);
1540 
1541 	/* reset ioc */
1542 	mpi_write(sc, MPI_HOSTDIAG, MPI_HOSTDIAG_RESET_ADAPTER);
1543 
1544 	delay(10000);
1545 
1546 	/* disable diagnostic register */
1547 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1548 
1549 	/* restore pci bits? */
1550 
1551 	/* firmware bits? */
1552 	return (0);
1553 }
1554 
1555 int
1556 mpi_handshake_send(struct mpi_softc *sc, void *buf, size_t dwords)
1557 {
1558 	u_int32_t				*query = buf;
1559 	int					i;
1560 
1561 	/* make sure the doorbell is not in use. */
1562 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1563 		return (1);
1564 
1565 	/* clear pending doorbell interrupts */
1566 	if (mpi_read_intr(sc) & MPI_INTR_STATUS_DOORBELL)
1567 		mpi_write_intr(sc, 0);
1568 
1569 	/*
1570 	 * first write the doorbell with the handshake function and the
1571 	 * dword count.
1572 	 */
1573 	mpi_write_db(sc, MPI_DOORBELL_FUNCTION(MPI_FUNCTION_HANDSHAKE) |
1574 	    MPI_DOORBELL_DWORDS(dwords));
1575 
1576 	/*
1577 	 * the doorbell used bit will be set because a doorbell function has
1578 	 * started. Wait for the interrupt and then ack it.
1579 	 */
1580 	if (mpi_wait_db_int(sc) != 0)
1581 		return (1);
1582 	mpi_write_intr(sc, 0);
1583 
1584 	/* poll for the acknowledgement. */
1585 	if (mpi_wait_db_ack(sc) != 0)
1586 		return (1);
1587 
1588 	/* write the query through the doorbell. */
1589 	for (i = 0; i < dwords; i++) {
1590 		mpi_write_db(sc, htole32(query[i]));
1591 		if (mpi_wait_db_ack(sc) != 0)
1592 			return (1);
1593 	}
1594 
1595 	return (0);
1596 }
1597 
1598 int
1599 mpi_handshake_recv_dword(struct mpi_softc *sc, u_int32_t *dword)
1600 {
1601 	u_int16_t				*words = (u_int16_t *)dword;
1602 	int					i;
1603 
1604 	for (i = 0; i < 2; i++) {
1605 		if (mpi_wait_db_int(sc) != 0)
1606 			return (1);
1607 		words[i] = letoh16(mpi_read_db(sc) & MPI_DOORBELL_DATA_MASK);
1608 		mpi_write_intr(sc, 0);
1609 	}
1610 
1611 	return (0);
1612 }
1613 
1614 int
1615 mpi_handshake_recv(struct mpi_softc *sc, void *buf, size_t dwords)
1616 {
1617 	struct mpi_msg_reply			*reply = buf;
1618 	u_int32_t				*dbuf = buf, dummy;
1619 	int					i;
1620 
1621 	/* get the first dword so we can read the length out of the header. */
1622 	if (mpi_handshake_recv_dword(sc, &dbuf[0]) != 0)
1623 		return (1);
1624 
1625 	DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dwords: %d reply: %d\n",
1626 	    DEVNAME(sc), dwords, reply->msg_length);
1627 
1628 	/*
1629 	 * the total length, in dwords, is in the message length field of the
1630 	 * reply header.
1631 	 */
1632 	for (i = 1; i < MIN(dwords, reply->msg_length); i++) {
1633 		if (mpi_handshake_recv_dword(sc, &dbuf[i]) != 0)
1634 			return (1);
1635 	}
1636 
1637 	/* if there's extra stuff to come off the ioc, discard it */
1638 	while (i++ < reply->msg_length) {
1639 		if (mpi_handshake_recv_dword(sc, &dummy) != 0)
1640 			return (1);
1641 		DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dummy read: "
1642 		    "0x%08x\n", DEVNAME(sc), dummy);
1643 	}
1644 
1645 	/* wait for the doorbell used bit to be reset and clear the intr */
1646 	if (mpi_wait_db_int(sc) != 0)
1647 		return (1);
1648 	mpi_write_intr(sc, 0);
1649 
1650 	return (0);
1651 }
1652 
1653 void
1654 mpi_empty_done(struct mpi_ccb *ccb)
1655 {
1656 	/* nothing to do */
1657 }
1658 
1659 int
1660 mpi_iocfacts(struct mpi_softc *sc)
1661 {
1662 	struct mpi_msg_iocfacts_request		ifq;
1663 	struct mpi_msg_iocfacts_reply		ifp;
1664 
1665 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts\n", DEVNAME(sc));
1666 
1667 	bzero(&ifq, sizeof(ifq));
1668 	bzero(&ifp, sizeof(ifp));
1669 
1670 	ifq.function = MPI_FUNCTION_IOC_FACTS;
1671 	ifq.chain_offset = 0;
1672 	ifq.msg_flags = 0;
1673 	ifq.msg_context = htole32(0xdeadbeef);
1674 
1675 	if (mpi_handshake_send(sc, &ifq, dwordsof(ifq)) != 0) {
1676 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts send failed\n",
1677 		    DEVNAME(sc));
1678 		return (1);
1679 	}
1680 
1681 	if (mpi_handshake_recv(sc, &ifp, dwordsof(ifp)) != 0) {
1682 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts recv failed\n",
1683 		    DEVNAME(sc));
1684 		return (1);
1685 	}
1686 
1687 	DNPRINTF(MPI_D_MISC, "%s:  func: 0x%02x len: %d msgver: %d.%d\n",
1688 	    DEVNAME(sc), ifp.function, ifp.msg_length,
1689 	    ifp.msg_version_maj, ifp.msg_version_min);
1690 	DNPRINTF(MPI_D_MISC, "%s:  msgflags: 0x%02x iocnumber: 0x%02x "
1691 	    "hdrver: %d.%d\n", DEVNAME(sc), ifp.msg_flags,
1692 	    ifp.ioc_number, ifp.header_version_maj,
1693 	    ifp.header_version_min);
1694 	DNPRINTF(MPI_D_MISC, "%s:  message context: 0x%08x\n", DEVNAME(sc),
1695 	    letoh32(ifp.msg_context));
1696 	DNPRINTF(MPI_D_MISC, "%s:  iocstatus: 0x%04x ioexcept: 0x%04x\n",
1697 	    DEVNAME(sc), letoh16(ifp.ioc_status),
1698 	    letoh16(ifp.ioc_exceptions));
1699 	DNPRINTF(MPI_D_MISC, "%s:  iocloginfo: 0x%08x\n", DEVNAME(sc),
1700 	    letoh32(ifp.ioc_loginfo));
1701 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%02x blocksize: %d whoinit: 0x%02x "
1702 	    "maxchdepth: %d\n", DEVNAME(sc), ifp.flags,
1703 	    ifp.block_size, ifp.whoinit, ifp.max_chain_depth);
1704 	DNPRINTF(MPI_D_MISC, "%s:  reqfrsize: %d replyqdepth: %d\n",
1705 	    DEVNAME(sc), letoh16(ifp.request_frame_size),
1706 	    letoh16(ifp.reply_queue_depth));
1707 	DNPRINTF(MPI_D_MISC, "%s:  productid: 0x%04x\n", DEVNAME(sc),
1708 	    letoh16(ifp.product_id));
1709 	DNPRINTF(MPI_D_MISC, "%s:  hostmfahiaddr: 0x%08x\n", DEVNAME(sc),
1710 	    letoh32(ifp.current_host_mfa_hi_addr));
1711 	DNPRINTF(MPI_D_MISC, "%s:  event_state: 0x%02x number_of_ports: %d "
1712 	    "global_credits: %d\n",
1713 	    DEVNAME(sc), ifp.event_state, ifp.number_of_ports,
1714 	    letoh16(ifp.global_credits));
1715 	DNPRINTF(MPI_D_MISC, "%s:  sensebufhiaddr: 0x%08x\n", DEVNAME(sc),
1716 	    letoh32(ifp.current_sense_buffer_hi_addr));
1717 	DNPRINTF(MPI_D_MISC, "%s:  maxbus: %d maxdev: %d replyfrsize: %d\n",
1718 	    DEVNAME(sc), ifp.max_buses, ifp.max_devices,
1719 	    letoh16(ifp.current_reply_frame_size));
1720 	DNPRINTF(MPI_D_MISC, "%s:  fw_image_size: %d\n", DEVNAME(sc),
1721 	    letoh32(ifp.fw_image_size));
1722 	DNPRINTF(MPI_D_MISC, "%s:  ioc_capabilities: 0x%08x\n", DEVNAME(sc),
1723 	    letoh32(ifp.ioc_capabilities));
1724 	DNPRINTF(MPI_D_MISC, "%s:  fw_version: %d.%d fw_version_unit: 0x%02x "
1725 	    "fw_version_dev: 0x%02x\n", DEVNAME(sc),
1726 	    ifp.fw_version_maj, ifp.fw_version_min,
1727 	    ifp.fw_version_unit, ifp.fw_version_dev);
1728 	DNPRINTF(MPI_D_MISC, "%s:  hi_priority_queue_depth: 0x%04x\n",
1729 	    DEVNAME(sc), letoh16(ifp.hi_priority_queue_depth));
1730 	DNPRINTF(MPI_D_MISC, "%s:  host_page_buffer_sge: hdr: 0x%08x "
1731 	    "addr 0x%08x %08x\n", DEVNAME(sc),
1732 	    letoh32(ifp.host_page_buffer_sge.sg_hdr),
1733 	    letoh32(ifp.host_page_buffer_sge.sg_hi_addr),
1734 	    letoh32(ifp.host_page_buffer_sge.sg_lo_addr));
1735 
1736 	sc->sc_maxcmds = letoh16(ifp.global_credits);
1737 	sc->sc_maxchdepth = ifp.max_chain_depth;
1738 	sc->sc_ioc_number = ifp.ioc_number;
1739 	if (sc->sc_flags & MPI_F_SPI)
1740 		sc->sc_buswidth = 16;
1741 	else
1742 		sc->sc_buswidth =
1743 		    (ifp.max_devices == 0) ? 256 : ifp.max_devices;
1744 	if (ifp.flags & MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT)
1745 		sc->sc_fw_len = letoh32(ifp.fw_image_size);
1746 
1747 	/*
1748 	 * you can fit sg elements on the end of the io cmd if they fit in the
1749 	 * request frame size.
1750 	 */
1751 	sc->sc_first_sgl_len = ((letoh16(ifp.request_frame_size) * 4) -
1752 	    sizeof(struct mpi_msg_scsi_io)) / sizeof(struct mpi_sge);
1753 	DNPRINTF(MPI_D_MISC, "%s:   first sgl len: %d\n", DEVNAME(sc),
1754 	    sc->sc_first_sgl_len);
1755 
1756 	sc->sc_chain_len = (letoh16(ifp.request_frame_size) * 4) /
1757 	    sizeof(struct mpi_sge);
1758 	DNPRINTF(MPI_D_MISC, "%s:   chain len: %d\n", DEVNAME(sc),
1759 	    sc->sc_chain_len);
1760 
1761 	/* the sgl tailing the io cmd loses an entry to the chain element. */
1762 	sc->sc_max_sgl_len = MPI_MAX_SGL - 1;
1763 	/* the sgl chains lose an entry for each chain element */
1764 	sc->sc_max_sgl_len -= (MPI_MAX_SGL - sc->sc_first_sgl_len) /
1765 	    sc->sc_chain_len;
1766 	DNPRINTF(MPI_D_MISC, "%s:   max sgl len: %d\n", DEVNAME(sc),
1767 	    sc->sc_max_sgl_len);
1768 
1769 	/* XXX we're ignoring the max chain depth */
1770 
1771 	return (0);
1772 }
1773 
1774 int
1775 mpi_iocinit(struct mpi_softc *sc)
1776 {
1777 	struct mpi_msg_iocinit_request		iiq;
1778 	struct mpi_msg_iocinit_reply		iip;
1779 	u_int32_t				hi_addr;
1780 
1781 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit\n", DEVNAME(sc));
1782 
1783 	bzero(&iiq, sizeof(iiq));
1784 	bzero(&iip, sizeof(iip));
1785 
1786 	iiq.function = MPI_FUNCTION_IOC_INIT;
1787 	iiq.whoinit = MPI_WHOINIT_HOST_DRIVER;
1788 
1789 	iiq.max_devices = (sc->sc_buswidth == 256) ? 0 : sc->sc_buswidth;
1790 	iiq.max_buses = 1;
1791 
1792 	iiq.msg_context = htole32(0xd00fd00f);
1793 
1794 	iiq.reply_frame_size = htole16(MPI_REPLY_SIZE);
1795 
1796 	hi_addr = (u_int32_t)((u_int64_t)MPI_DMA_DVA(sc->sc_requests) >> 32);
1797 	iiq.host_mfa_hi_addr = htole32(hi_addr);
1798 	iiq.sense_buffer_hi_addr = htole32(hi_addr);
1799 
1800 	hi_addr = (u_int32_t)((u_int64_t)MPI_DMA_DVA(sc->sc_replies) >> 32);
1801 	iiq.reply_fifo_host_signalling_addr = htole32(hi_addr);
1802 
1803 	iiq.msg_version_maj = 0x01;
1804 	iiq.msg_version_min = 0x02;
1805 
1806 	iiq.hdr_version_unit = 0x0d;
1807 	iiq.hdr_version_dev = 0x00;
1808 
1809 	if (mpi_handshake_send(sc, &iiq, dwordsof(iiq)) != 0) {
1810 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit send failed\n",
1811 		    DEVNAME(sc));
1812 		return (1);
1813 	}
1814 
1815 	if (mpi_handshake_recv(sc, &iip, dwordsof(iip)) != 0) {
1816 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit recv failed\n",
1817 		    DEVNAME(sc));
1818 		return (1);
1819 	}
1820 
1821 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d "
1822 	    "whoinit: 0x%02x\n", DEVNAME(sc), iip.function,
1823 	    iip.msg_length, iip.whoinit);
1824 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x max_buses: %d "
1825 	    "max_devices: %d flags: 0x%02x\n", DEVNAME(sc), iip.msg_flags,
1826 	    iip.max_buses, iip.max_devices, iip.flags);
1827 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1828 	    letoh32(iip.msg_context));
1829 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
1830 	    letoh16(iip.ioc_status));
1831 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1832 	    letoh32(iip.ioc_loginfo));
1833 
1834 	return (0);
1835 }
1836 
1837 int
1838 mpi_portfacts(struct mpi_softc *sc)
1839 {
1840 	struct mpi_ccb				*ccb;
1841 	struct mpi_msg_portfacts_request	*pfq;
1842 	volatile struct mpi_msg_portfacts_reply	*pfp;
1843 	int					s, rv = 1;
1844 
1845 	DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts\n", DEVNAME(sc));
1846 
1847 	s = splbio();
1848 	ccb = mpi_get_ccb(sc);
1849 	splx(s);
1850 	if (ccb == NULL) {
1851 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts ccb_get\n",
1852 		    DEVNAME(sc));
1853 		return (rv);
1854 	}
1855 
1856 	ccb->ccb_done = mpi_empty_done;
1857 	pfq = ccb->ccb_cmd;
1858 
1859 	pfq->function = MPI_FUNCTION_PORT_FACTS;
1860 	pfq->chain_offset = 0;
1861 	pfq->msg_flags = 0;
1862 	pfq->port_number = 0;
1863 	pfq->msg_context = htole32(ccb->ccb_id);
1864 
1865 	if (mpi_poll(sc, ccb, 50000) != 0) {
1866 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts poll\n", DEVNAME(sc));
1867 		goto err;
1868 	}
1869 
1870 	if (ccb->ccb_rcb == NULL) {
1871 		DNPRINTF(MPI_D_MISC, "%s: empty portfacts reply\n",
1872 		    DEVNAME(sc));
1873 		goto err;
1874 	}
1875 	pfp = ccb->ccb_rcb->rcb_reply;
1876 
1877 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d\n",
1878 	    DEVNAME(sc), pfp->function, pfp->msg_length);
1879 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x port_number: %d\n",
1880 	    DEVNAME(sc), pfp->msg_flags, pfp->port_number);
1881 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1882 	    letoh32(pfp->msg_context));
1883 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
1884 	    letoh16(pfp->ioc_status));
1885 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1886 	    letoh32(pfp->ioc_loginfo));
1887 	DNPRINTF(MPI_D_MISC, "%s:  max_devices: %d port_type: 0x%02x\n",
1888 	    DEVNAME(sc), letoh16(pfp->max_devices), pfp->port_type);
1889 	DNPRINTF(MPI_D_MISC, "%s:  protocol_flags: 0x%04x port_scsi_id: %d\n",
1890 	    DEVNAME(sc), letoh16(pfp->protocol_flags),
1891 	    letoh16(pfp->port_scsi_id));
1892 	DNPRINTF(MPI_D_MISC, "%s:  max_persistent_ids: %d "
1893 	    "max_posted_cmd_buffers: %d\n", DEVNAME(sc),
1894 	    letoh16(pfp->max_persistent_ids),
1895 	    letoh16(pfp->max_posted_cmd_buffers));
1896 	DNPRINTF(MPI_D_MISC, "%s:  max_lan_buckets: %d\n", DEVNAME(sc),
1897 	    letoh16(pfp->max_lan_buckets));
1898 
1899 	sc->sc_porttype = pfp->port_type;
1900 	if (sc->sc_target == -1)
1901 		sc->sc_target = letoh16(pfp->port_scsi_id);
1902 
1903 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
1904 	rv = 0;
1905 err:
1906 	mpi_put_ccb(sc, ccb);
1907 
1908 	return (rv);
1909 }
1910 
1911 int
1912 mpi_eventnotify(struct mpi_softc *sc)
1913 {
1914 	struct mpi_ccb				*ccb;
1915 	struct mpi_msg_event_request		*enq;
1916 	int					s;
1917 
1918 	s = splbio();
1919 	ccb = mpi_get_ccb(sc);
1920 	splx(s);
1921 	if (ccb == NULL) {
1922 		DNPRINTF(MPI_D_MISC, "%s: mpi_eventnotify ccb_get\n",
1923 		    DEVNAME(sc));
1924 		return (1);
1925 	}
1926 
1927 	ccb->ccb_done = mpi_eventnotify_done;
1928 	enq = ccb->ccb_cmd;
1929 
1930 	enq->function = MPI_FUNCTION_EVENT_NOTIFICATION;
1931 	enq->chain_offset = 0;
1932 	enq->event_switch = MPI_EVENT_SWITCH_ON;
1933 	enq->msg_context = htole32(ccb->ccb_id);
1934 
1935 	mpi_start(sc, ccb);
1936 	return (0);
1937 }
1938 
1939 void
1940 mpi_eventnotify_done(struct mpi_ccb *ccb)
1941 {
1942 	struct mpi_softc			*sc = ccb->ccb_sc;
1943 	struct mpi_msg_event_reply		*enp = ccb->ccb_rcb->rcb_reply;
1944 	int					deferred = 0;
1945 
1946 	DNPRINTF(MPI_D_EVT, "%s: mpi_eventnotify_done\n", DEVNAME(sc));
1947 
1948 	DNPRINTF(MPI_D_EVT, "%s:  function: 0x%02x msg_length: %d "
1949 	    "data_length: %d\n", DEVNAME(sc), enp->function, enp->msg_length,
1950 	    letoh16(enp->data_length));
1951 	DNPRINTF(MPI_D_EVT, "%s:  ack_required: %d msg_flags 0x%02x\n",
1952 	    DEVNAME(sc), enp->ack_required, enp->msg_flags);
1953 	DNPRINTF(MPI_D_EVT, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1954 	    letoh32(enp->msg_context));
1955 	DNPRINTF(MPI_D_EVT, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
1956 	    letoh16(enp->ioc_status));
1957 	DNPRINTF(MPI_D_EVT, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1958 	    letoh32(enp->ioc_loginfo));
1959 	DNPRINTF(MPI_D_EVT, "%s:  event: 0x%08x\n", DEVNAME(sc),
1960 	    letoh32(enp->event));
1961 	DNPRINTF(MPI_D_EVT, "%s:  event_context: 0x%08x\n", DEVNAME(sc),
1962 	    letoh32(enp->event_context));
1963 
1964 	switch (letoh32(enp->event)) {
1965 	/* ignore these */
1966 	case MPI_EVENT_EVENT_CHANGE:
1967 	case MPI_EVENT_SAS_PHY_LINK_STATUS:
1968 		break;
1969 
1970 	case MPI_EVENT_SAS_DEVICE_STATUS_CHANGE:
1971 		if (sc->sc_scsibus == NULL)
1972 			break;
1973 
1974 		if (scsi_task(mpi_evt_sas, sc, ccb->ccb_rcb, 0) != 0) {
1975 			printf("%s: unable to run SAS device status change\n",
1976 			    DEVNAME(sc));
1977 			break;
1978 		}
1979 		deferred = 1;
1980 		break;
1981 
1982 	default:
1983 		printf("%s: unhandled event 0x%02x\n", DEVNAME(sc),
1984 		    letoh32(enp->event));
1985 		break;
1986 	}
1987 
1988 	if (!deferred) {
1989 		if (enp->ack_required)
1990 			mpi_eventack(sc, enp);
1991 		mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
1992 	}
1993 
1994 	if ((enp->msg_flags & MPI_EVENT_FLAGS_REPLY_KEPT) == 0) {
1995 		/* XXX this shouldnt happen till shutdown */
1996 		mpi_put_ccb(sc, ccb);
1997 	}
1998 }
1999 
2000 void
2001 mpi_evt_sas(void *xsc, void *arg)
2002 {
2003 	struct mpi_softc			*sc = xsc;
2004 	struct mpi_rcb				*rcb = arg;
2005 	struct mpi_msg_event_reply		*enp = rcb->rcb_reply;
2006 	struct mpi_evt_sas_change		*ch;
2007 	u_int8_t				*data;
2008 	int					s;
2009 
2010 	data = rcb->rcb_reply;
2011 	data += sizeof(struct mpi_msg_event_reply);
2012 	ch = (struct mpi_evt_sas_change *)data;
2013 
2014 	if (ch->bus != 0)
2015 		return;
2016 
2017 	switch (ch->reason) {
2018 	case MPI_EVT_SASCH_REASON_ADDED:
2019 	case MPI_EVT_SASCH_REASON_NO_PERSIST_ADDED:
2020 		scsi_probe_target(sc->sc_scsibus, ch->target);
2021 		break;
2022 
2023 	case MPI_EVT_SASCH_REASON_NOT_RESPONDING:
2024 		scsi_detach_target(sc->sc_scsibus, ch->target, DETACH_FORCE);
2025 		break;
2026 
2027 	case MPI_EVT_SASCH_REASON_SMART_DATA:
2028 	case MPI_EVT_SASCH_REASON_UNSUPPORTED:
2029 	case MPI_EVT_SASCH_REASON_INTERNAL_RESET:
2030 		break;
2031 	default:
2032 		printf("%s: unknown reason for SAS device status change: "
2033 		    "0x%02x\n", DEVNAME(sc), ch->reason);
2034 		break;
2035 	}
2036 
2037 	s = splbio();
2038 	mpi_push_reply(sc, rcb->rcb_reply_dva);
2039 	if (enp->ack_required)
2040 		mpi_eventack(sc, enp);
2041 	splx(s);
2042 }
2043 
2044 void
2045 mpi_eventack(struct mpi_softc *sc, struct mpi_msg_event_reply *enp)
2046 {
2047 	struct mpi_ccb				*ccb;
2048 	struct mpi_msg_eventack_request		*eaq;
2049 
2050 	ccb = mpi_get_ccb(sc);
2051 	if (ccb == NULL) {
2052 		DNPRINTF(MPI_D_EVT, "%s: mpi_eventack ccb_get\n", DEVNAME(sc));
2053 		return;
2054 	}
2055 
2056 	ccb->ccb_done = mpi_eventack_done;
2057 	eaq = ccb->ccb_cmd;
2058 
2059 	eaq->function = MPI_FUNCTION_EVENT_ACK;
2060 	eaq->msg_context = htole32(ccb->ccb_id);
2061 
2062 	eaq->event = enp->event;
2063 	eaq->event_context = enp->event_context;
2064 
2065 	mpi_start(sc, ccb);
2066 	return;
2067 }
2068 
2069 void
2070 mpi_eventack_done(struct mpi_ccb *ccb)
2071 {
2072 	struct mpi_softc			*sc = ccb->ccb_sc;
2073 
2074 	DNPRINTF(MPI_D_EVT, "%s: event ack done\n", DEVNAME(sc));
2075 
2076 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2077 	mpi_put_ccb(sc, ccb);
2078 }
2079 
2080 int
2081 mpi_portenable(struct mpi_softc *sc)
2082 {
2083 	struct mpi_ccb				*ccb;
2084 	struct mpi_msg_portenable_request	*peq;
2085 	struct mpi_msg_portenable_repy		*pep;
2086 	int					s;
2087 
2088 	DNPRINTF(MPI_D_MISC, "%s: mpi_portenable\n", DEVNAME(sc));
2089 
2090 	s = splbio();
2091 	ccb = mpi_get_ccb(sc);
2092 	splx(s);
2093 	if (ccb == NULL) {
2094 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable ccb_get\n",
2095 		    DEVNAME(sc));
2096 		return (1);
2097 	}
2098 
2099 	ccb->ccb_done = mpi_empty_done;
2100 	peq = ccb->ccb_cmd;
2101 
2102 	peq->function = MPI_FUNCTION_PORT_ENABLE;
2103 	peq->port_number = 0;
2104 	peq->msg_context = htole32(ccb->ccb_id);
2105 
2106 	if (mpi_poll(sc, ccb, 50000) != 0) {
2107 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable poll\n", DEVNAME(sc));
2108 		return (1);
2109 	}
2110 
2111 	if (ccb->ccb_rcb == NULL) {
2112 		DNPRINTF(MPI_D_MISC, "%s: empty portenable reply\n",
2113 		    DEVNAME(sc));
2114 		return (1);
2115 	}
2116 	pep = ccb->ccb_rcb->rcb_reply;
2117 
2118 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2119 	mpi_put_ccb(sc, ccb);
2120 
2121 	return (0);
2122 }
2123 
2124 int
2125 mpi_fwupload(struct mpi_softc *sc)
2126 {
2127 	struct mpi_ccb				*ccb;
2128 	struct {
2129 		struct mpi_msg_fwupload_request		req;
2130 		struct mpi_sge				sge;
2131 	} __packed				*bundle;
2132 	struct mpi_msg_fwupload_reply		*upp;
2133 	u_int64_t				addr;
2134 	int					s;
2135 	int					rv = 0;
2136 
2137 	if (sc->sc_fw_len == 0)
2138 		return (0);
2139 
2140 	DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload\n", DEVNAME(sc));
2141 
2142 	sc->sc_fw = mpi_dmamem_alloc(sc, sc->sc_fw_len);
2143 	if (sc->sc_fw == NULL) {
2144 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload unable to allocate %d\n",
2145 		    DEVNAME(sc), sc->sc_fw_len);
2146 		return (1);
2147 	}
2148 
2149 	s = splbio();
2150 	ccb = mpi_get_ccb(sc);
2151 	splx(s);
2152 	if (ccb == NULL) {
2153 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload ccb_get\n",
2154 		    DEVNAME(sc));
2155 		goto err;
2156 	}
2157 
2158 	ccb->ccb_done = mpi_empty_done;
2159 	bundle = ccb->ccb_cmd;
2160 
2161 	bundle->req.function = MPI_FUNCTION_FW_UPLOAD;
2162 	bundle->req.msg_context = htole32(ccb->ccb_id);
2163 
2164 	bundle->req.image_type = MPI_FWUPLOAD_IMAGETYPE_IOC_FW;
2165 
2166 	bundle->req.tce.details_length = 12;
2167 	bundle->req.tce.image_size = htole32(sc->sc_fw_len);
2168 
2169 	bundle->sge.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2170 	    MPI_SGE_FL_SIZE_64 | MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
2171 	    MPI_SGE_FL_EOL | (u_int32_t)sc->sc_fw_len);
2172 	addr = MPI_DMA_DVA(sc->sc_fw);
2173 	bundle->sge.sg_hi_addr = htole32((u_int32_t)(addr >> 32));
2174 	bundle->sge.sg_lo_addr = htole32((u_int32_t)addr);
2175 
2176 	if (mpi_poll(sc, ccb, 50000) != 0) {
2177 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2178 		goto err;
2179 	}
2180 
2181 	if (ccb->ccb_rcb == NULL)
2182 		panic("%s: unable to do fw upload\n", DEVNAME(sc));
2183 	upp = ccb->ccb_rcb->rcb_reply;
2184 
2185 	if (letoh16(upp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2186 		rv = 1;
2187 
2188 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2189 	mpi_put_ccb(sc, ccb);
2190 
2191 	return (rv);
2192 
2193 err:
2194 	mpi_dmamem_free(sc, sc->sc_fw);
2195 	return (1);
2196 }
2197 
2198 void
2199 mpi_get_raid(struct mpi_softc *sc)
2200 {
2201 	struct mpi_cfg_hdr		hdr;
2202 	struct mpi_cfg_ioc_pg2		*vol_page;
2203 	struct mpi_cfg_raid_vol		*vol_list, *vol;
2204 	size_t				pagelen;
2205 	u_int32_t			capabilities;
2206 	struct scsi_link		*link;
2207 	int				i;
2208 
2209 	DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid\n", DEVNAME(sc));
2210 
2211 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 2, 0, &hdr) != 0) {
2212 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch header"
2213 		    "for IOC page 2\n", DEVNAME(sc));
2214 		return;
2215 	}
2216 
2217 	pagelen = hdr.page_length * 4; /* dwords to bytes */
2218 	vol_page = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
2219 	if (vol_page == NULL) {
2220 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to allocate "
2221 		    "space for ioc config page 2\n", DEVNAME(sc));
2222 		return;
2223 	}
2224 	vol_list = (struct mpi_cfg_raid_vol *)(vol_page + 1);
2225 
2226 	if (mpi_cfg_page(sc, 0, &hdr, 1, vol_page, pagelen) != 0) {
2227 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch IOC "
2228 		    "page 2\n", DEVNAME(sc));
2229 		goto out;
2230 	}
2231 
2232 	capabilities = letoh32(vol_page->capabilities);
2233 
2234 	DNPRINTF(MPI_D_RAID, "%s:  capabilities: 0x08%x\n", DEVNAME(sc),
2235 	    letoh32(vol_page->capabilities));
2236 	DNPRINTF(MPI_D_RAID, "%s:  active_vols: %d max_vols: %d "
2237 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
2238 	    vol_page->active_vols, vol_page->max_vols,
2239 	    vol_page->active_physdisks, vol_page->max_physdisks);
2240 
2241 	/* don't walk list if there are no RAID capability */
2242 	if (capabilities == 0xdeadbeef) {
2243 		printf("%s: deadbeef in raid configuration\n", DEVNAME(sc));
2244 		goto out;
2245 	}
2246 
2247 	if ((capabilities & MPI_CFG_IOC_2_CAPABILITIES_RAID) == 0 ||
2248 	    (vol_page->active_vols == 0))
2249 		goto out;
2250 
2251 	sc->sc_flags |= MPI_F_RAID;
2252 
2253 	for (i = 0; i < vol_page->active_vols; i++) {
2254 		vol = &vol_list[i];
2255 
2256 		DNPRINTF(MPI_D_RAID, "%s:   id: %d bus: %d ioc: %d pg: %d\n",
2257 		    DEVNAME(sc), vol->vol_id, vol->vol_bus, vol->vol_ioc,
2258 		    vol->vol_page);
2259 		DNPRINTF(MPI_D_RAID, "%s:   type: 0x%02x flags: 0x%02x\n",
2260 		    DEVNAME(sc), vol->vol_type, vol->flags);
2261 
2262 		if (vol->vol_ioc != sc->sc_ioc_number || vol->vol_bus != 0)
2263 			continue;
2264 
2265 		link = sc->sc_scsibus->sc_link[vol->vol_id][0];
2266 		if (link == NULL)
2267 			continue;
2268 
2269 		link->flags |= SDEV_VIRTUAL;
2270 	}
2271 
2272 out:
2273 	free(vol_page, M_TEMP);
2274 }
2275 
2276 int
2277 mpi_req_cfg_header(struct mpi_softc *sc, u_int8_t type, u_int8_t number,
2278     u_int32_t address, int extended, void *p)
2279 {
2280 	struct mpi_ccb				*ccb;
2281 	struct mpi_msg_config_request		*cq;
2282 	struct mpi_msg_config_reply		*cp;
2283 	struct mpi_cfg_hdr			*hdr = p;
2284 	struct mpi_ecfg_hdr			*ehdr = p;
2285 	int					etype = 0;
2286 	int					rv = 0;
2287 	int					s;
2288 
2289 	DNPRINTF(MPI_D_MISC, "%s: mpi_req_cfg_header type: %#x number: %x "
2290 	    "address: 0x%08x extended: %d\n", DEVNAME(sc), type, number,
2291 	    address, extended);
2292 
2293 	s = splbio();
2294 	ccb = mpi_get_ccb(sc);
2295 	splx(s);
2296 	if (ccb == NULL) {
2297 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header ccb_get\n",
2298 		    DEVNAME(sc));
2299 		return (1);
2300 	}
2301 
2302 	if (extended) {
2303 		etype = type;
2304 		type = MPI_CONFIG_REQ_PAGE_TYPE_EXTENDED;
2305 	}
2306 
2307 	ccb->ccb_done = mpi_empty_done;
2308 	cq = ccb->ccb_cmd;
2309 
2310 	cq->function = MPI_FUNCTION_CONFIG;
2311 	cq->msg_context = htole32(ccb->ccb_id);
2312 
2313 	cq->action = MPI_CONFIG_REQ_ACTION_PAGE_HEADER;
2314 
2315 	cq->config_header.page_number = number;
2316 	cq->config_header.page_type = type;
2317 	cq->ext_page_type = etype;
2318 	cq->page_address = htole32(address);
2319 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2320 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
2321 
2322 	if (mpi_poll(sc, ccb, 50000) != 0) {
2323 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2324 		return (1);
2325 	}
2326 
2327 	if (ccb->ccb_rcb == NULL)
2328 		panic("%s: unable to fetch config header\n", DEVNAME(sc));
2329 	cp = ccb->ccb_rcb->rcb_reply;
2330 
2331 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2332 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2333 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2334 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2335 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2336 	    cp->msg_flags);
2337 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2338 	    letoh32(cp->msg_context));
2339 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2340 	    letoh16(cp->ioc_status));
2341 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2342 	    letoh32(cp->ioc_loginfo));
2343 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2344 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2345 	    cp->config_header.page_version,
2346 	    cp->config_header.page_length,
2347 	    cp->config_header.page_number,
2348 	    cp->config_header.page_type);
2349 
2350 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2351 		rv = 1;
2352 	else if (extended) {
2353 		bzero(ehdr, sizeof(*ehdr));
2354 		ehdr->page_version = cp->config_header.page_version;
2355 		ehdr->page_number = cp->config_header.page_number;
2356 		ehdr->page_type = cp->config_header.page_type;
2357 		ehdr->ext_page_length = cp->ext_page_length;
2358 		ehdr->ext_page_type = cp->ext_page_type;
2359 	} else
2360 		*hdr = cp->config_header;
2361 
2362 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2363 	mpi_put_ccb(sc, ccb);
2364 
2365 	return (rv);
2366 }
2367 
2368 int
2369 mpi_req_cfg_page(struct mpi_softc *sc, u_int32_t address, int extended,
2370     void *p, int read, void *page, size_t len)
2371 {
2372 	struct mpi_ccb				*ccb;
2373 	struct mpi_msg_config_request		*cq;
2374 	struct mpi_msg_config_reply		*cp;
2375 	struct mpi_cfg_hdr			*hdr = p;
2376 	struct mpi_ecfg_hdr			*ehdr = p;
2377 	u_int64_t				dva;
2378 	char					*kva;
2379 	int					page_length;
2380 	int					rv = 0;
2381 	int					s;
2382 
2383 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page address: %d read: %d type: %x\n",
2384 	    DEVNAME(sc), address, read, hdr->page_type);
2385 
2386 	page_length = extended ?
2387 	    letoh16(ehdr->ext_page_length) : hdr->page_length;
2388 
2389 	if (len > MPI_REQUEST_SIZE - sizeof(struct mpi_msg_config_request) ||
2390 	    len < page_length * 4)
2391 		return (1);
2392 
2393 	s = splbio();
2394 	ccb = mpi_get_ccb(sc);
2395 	splx(s);
2396 	if (ccb == NULL) {
2397 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page ccb_get\n", DEVNAME(sc));
2398 		return (1);
2399 	}
2400 
2401 	ccb->ccb_done = mpi_empty_done;
2402 	cq = ccb->ccb_cmd;
2403 
2404 	cq->function = MPI_FUNCTION_CONFIG;
2405 	cq->msg_context = htole32(ccb->ccb_id);
2406 
2407 	cq->action = (read ? MPI_CONFIG_REQ_ACTION_PAGE_READ_CURRENT :
2408 	    MPI_CONFIG_REQ_ACTION_PAGE_WRITE_CURRENT);
2409 
2410 	if (extended) {
2411 		cq->config_header.page_version = ehdr->page_version;
2412 		cq->config_header.page_number = ehdr->page_number;
2413 		cq->config_header.page_type = ehdr->page_type;
2414 		cq->ext_page_len = ehdr->ext_page_length;
2415 		cq->ext_page_type = ehdr->ext_page_type;
2416 	} else
2417 		cq->config_header = *hdr;
2418 	cq->config_header.page_type &= MPI_CONFIG_REQ_PAGE_TYPE_MASK;
2419 	cq->page_address = htole32(address);
2420 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2421 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
2422 	    (page_length * 4) |
2423 	    (read ? MPI_SGE_FL_DIR_IN : MPI_SGE_FL_DIR_OUT));
2424 
2425 	/* bounce the page via the request space to avoid more bus_dma games */
2426 	dva = ccb->ccb_cmd_dva + sizeof(struct mpi_msg_config_request);
2427 
2428 	cq->page_buffer.sg_hi_addr = htole32((u_int32_t)(dva >> 32));
2429 	cq->page_buffer.sg_lo_addr = htole32((u_int32_t)dva);
2430 
2431 	kva = ccb->ccb_cmd;
2432 	kva += sizeof(struct mpi_msg_config_request);
2433 	if (!read)
2434 		bcopy(page, kva, len);
2435 
2436 	if (mpi_poll(sc, ccb, 50000) != 0) {
2437 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page poll\n", DEVNAME(sc));
2438 		return (1);
2439 	}
2440 
2441 	if (ccb->ccb_rcb == NULL) {
2442 		mpi_put_ccb(sc, ccb);
2443 		return (1);
2444 	}
2445 	cp = ccb->ccb_rcb->rcb_reply;
2446 
2447 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2448 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2449 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2450 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2451 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2452 	    cp->msg_flags);
2453 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2454 	    letoh32(cp->msg_context));
2455 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2456 	    letoh16(cp->ioc_status));
2457 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2458 	    letoh32(cp->ioc_loginfo));
2459 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2460 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2461 	    cp->config_header.page_version,
2462 	    cp->config_header.page_length,
2463 	    cp->config_header.page_number,
2464 	    cp->config_header.page_type);
2465 
2466 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2467 		rv = 1;
2468 	else if (read)
2469 		bcopy(kva, page, len);
2470 
2471 	mpi_push_reply(sc, ccb->ccb_rcb->rcb_reply_dva);
2472 	mpi_put_ccb(sc, ccb);
2473 
2474 	return (rv);
2475 }
2476