xref: /openbsd-src/sys/dev/ic/mpi.c (revision a7997e14b2c5583c85c5c712c73af0f27d80f68b)
1 /*	$OpenBSD: mpi.c,v 1.214 2020/06/27 14:29:45 krw Exp $ */
2 
3 /*
4  * Copyright (c) 2005, 2006, 2009 David Gwynne <dlg@openbsd.org>
5  * Copyright (c) 2005, 2008, 2009 Marco Peereboom <marco@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include "bio.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/buf.h>
25 #include <sys/device.h>
26 #include <sys/malloc.h>
27 #include <sys/kernel.h>
28 #include <sys/mutex.h>
29 #include <sys/rwlock.h>
30 #include <sys/sensors.h>
31 #include <sys/dkio.h>
32 #include <sys/task.h>
33 
34 #include <machine/bus.h>
35 
36 #include <scsi/scsi_all.h>
37 #include <scsi/scsiconf.h>
38 
39 #include <dev/biovar.h>
40 #include <dev/ic/mpireg.h>
41 #include <dev/ic/mpivar.h>
42 
43 #ifdef MPI_DEBUG
44 uint32_t	mpi_debug = 0
45 /*		    | MPI_D_CMD */
46 /*		    | MPI_D_INTR */
47 /*		    | MPI_D_MISC */
48 /*		    | MPI_D_DMA */
49 /*		    | MPI_D_IOCTL */
50 /*		    | MPI_D_RW */
51 /*		    | MPI_D_MEM */
52 /*		    | MPI_D_CCB */
53 /*		    | MPI_D_PPR */
54 /*		    | MPI_D_RAID */
55 /*		    | MPI_D_EVT */
56 		;
57 #endif
58 
59 struct cfdriver mpi_cd = {
60 	NULL,
61 	"mpi",
62 	DV_DULL
63 };
64 
65 void			mpi_scsi_cmd(struct scsi_xfer *);
66 void			mpi_scsi_cmd_done(struct mpi_ccb *);
67 int			mpi_scsi_probe(struct scsi_link *);
68 int			mpi_scsi_ioctl(struct scsi_link *, u_long, caddr_t,
69 			    int);
70 
71 struct scsi_adapter mpi_switch = {
72 	mpi_scsi_cmd, NULL, mpi_scsi_probe, NULL, mpi_scsi_ioctl
73 };
74 
75 struct mpi_dmamem	*mpi_dmamem_alloc(struct mpi_softc *, size_t);
76 void			mpi_dmamem_free(struct mpi_softc *,
77 			    struct mpi_dmamem *);
78 int			mpi_alloc_ccbs(struct mpi_softc *);
79 void			*mpi_get_ccb(void *);
80 void			mpi_put_ccb(void *, void *);
81 int			mpi_alloc_replies(struct mpi_softc *);
82 void			mpi_push_replies(struct mpi_softc *);
83 void			mpi_push_reply(struct mpi_softc *, struct mpi_rcb *);
84 
85 void			mpi_start(struct mpi_softc *, struct mpi_ccb *);
86 int			mpi_poll(struct mpi_softc *, struct mpi_ccb *, int);
87 void			mpi_poll_done(struct mpi_ccb *);
88 void			mpi_reply(struct mpi_softc *, u_int32_t);
89 
90 void			mpi_wait(struct mpi_softc *sc, struct mpi_ccb *);
91 void			mpi_wait_done(struct mpi_ccb *);
92 
93 int			mpi_cfg_spi_port(struct mpi_softc *);
94 void			mpi_squash_ppr(struct mpi_softc *);
95 void			mpi_run_ppr(struct mpi_softc *);
96 int			mpi_ppr(struct mpi_softc *, struct scsi_link *,
97 			    struct mpi_cfg_raid_physdisk *, int, int, int);
98 int			mpi_inq(struct mpi_softc *, u_int16_t, int);
99 
100 int			mpi_cfg_sas(struct mpi_softc *);
101 int			mpi_cfg_fc(struct mpi_softc *);
102 
103 void			mpi_timeout_xs(void *);
104 int			mpi_load_xs(struct mpi_ccb *);
105 
106 u_int32_t		mpi_read(struct mpi_softc *, bus_size_t);
107 void			mpi_write(struct mpi_softc *, bus_size_t, u_int32_t);
108 int			mpi_wait_eq(struct mpi_softc *, bus_size_t, u_int32_t,
109 			    u_int32_t);
110 int			mpi_wait_ne(struct mpi_softc *, bus_size_t, u_int32_t,
111 			    u_int32_t);
112 
113 int			mpi_init(struct mpi_softc *);
114 int			mpi_reset_soft(struct mpi_softc *);
115 int			mpi_reset_hard(struct mpi_softc *);
116 
117 int			mpi_handshake_send(struct mpi_softc *, void *, size_t);
118 int			mpi_handshake_recv_dword(struct mpi_softc *,
119 			    u_int32_t *);
120 int			mpi_handshake_recv(struct mpi_softc *, void *, size_t);
121 
122 void			mpi_empty_done(struct mpi_ccb *);
123 
124 int			mpi_iocinit(struct mpi_softc *);
125 int			mpi_iocfacts(struct mpi_softc *);
126 int			mpi_portfacts(struct mpi_softc *);
127 int			mpi_portenable(struct mpi_softc *);
128 int			mpi_cfg_coalescing(struct mpi_softc *);
129 void			mpi_get_raid(struct mpi_softc *);
130 int			mpi_fwupload(struct mpi_softc *);
131 int			mpi_manufacturing(struct mpi_softc *);
132 int			mpi_scsi_probe_virtual(struct scsi_link *);
133 
134 int			mpi_eventnotify(struct mpi_softc *);
135 void			mpi_eventnotify_done(struct mpi_ccb *);
136 void			mpi_eventnotify_free(struct mpi_softc *,
137 			    struct mpi_rcb *);
138 void			mpi_eventack(void *, void *);
139 void			mpi_eventack_done(struct mpi_ccb *);
140 int			mpi_evt_sas(struct mpi_softc *, struct mpi_rcb *);
141 void			mpi_evt_sas_detach(void *, void *);
142 void			mpi_evt_sas_detach_done(struct mpi_ccb *);
143 void			mpi_fc_rescan(void *);
144 
145 int			mpi_req_cfg_header(struct mpi_softc *, u_int8_t,
146 			    u_int8_t, u_int32_t, int, void *);
147 int			mpi_req_cfg_page(struct mpi_softc *, u_int32_t, int,
148 			    void *, int, void *, size_t);
149 
150 int			mpi_ioctl_cache(struct scsi_link *, u_long,
151 			    struct dk_cache *);
152 
153 #if NBIO > 0
154 int		mpi_bio_get_pg0_raid(struct mpi_softc *, int);
155 int		mpi_ioctl(struct device *, u_long, caddr_t);
156 int		mpi_ioctl_inq(struct mpi_softc *, struct bioc_inq *);
157 int		mpi_ioctl_vol(struct mpi_softc *, struct bioc_vol *);
158 int		mpi_ioctl_disk(struct mpi_softc *, struct bioc_disk *);
159 int		mpi_ioctl_setstate(struct mpi_softc *, struct bioc_setstate *);
160 #ifndef SMALL_KERNEL
161 int		mpi_create_sensors(struct mpi_softc *);
162 void		mpi_refresh_sensors(void *);
163 #endif /* SMALL_KERNEL */
164 #endif /* NBIO > 0 */
165 
166 #define DEVNAME(s)		((s)->sc_dev.dv_xname)
167 
168 #define	dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
169 
170 #define mpi_read_db(s)		mpi_read((s), MPI_DOORBELL)
171 #define mpi_write_db(s, v)	mpi_write((s), MPI_DOORBELL, (v))
172 #define mpi_read_intr(s)	bus_space_read_4((s)->sc_iot, (s)->sc_ioh, \
173 				    MPI_INTR_STATUS)
174 #define mpi_write_intr(s, v)	mpi_write((s), MPI_INTR_STATUS, (v))
175 #define mpi_pop_reply(s)	bus_space_read_4((s)->sc_iot, (s)->sc_ioh, \
176 				    MPI_REPLY_QUEUE)
177 #define mpi_push_reply_db(s, v) bus_space_write_4((s)->sc_iot, (s)->sc_ioh, \
178 				    MPI_REPLY_QUEUE, (v))
179 
180 #define mpi_wait_db_int(s)	mpi_wait_ne((s), MPI_INTR_STATUS, \
181 				    MPI_INTR_STATUS_DOORBELL, 0)
182 #define mpi_wait_db_ack(s)	mpi_wait_eq((s), MPI_INTR_STATUS, \
183 				    MPI_INTR_STATUS_IOCDOORBELL, 0)
184 
185 #define MPI_PG_EXTENDED		(1<<0)
186 #define MPI_PG_POLL		(1<<1)
187 #define MPI_PG_FMT		"\020" "\002POLL" "\001EXTENDED"
188 
189 #define mpi_cfg_header(_s, _t, _n, _a, _h) \
190 	mpi_req_cfg_header((_s), (_t), (_n), (_a), \
191 	    MPI_PG_POLL, (_h))
192 #define mpi_ecfg_header(_s, _t, _n, _a, _h) \
193 	mpi_req_cfg_header((_s), (_t), (_n), (_a), \
194 	    MPI_PG_POLL|MPI_PG_EXTENDED, (_h))
195 
196 #define mpi_cfg_page(_s, _a, _h, _r, _p, _l) \
197 	mpi_req_cfg_page((_s), (_a), MPI_PG_POLL, \
198 	    (_h), (_r), (_p), (_l))
199 #define mpi_ecfg_page(_s, _a, _h, _r, _p, _l) \
200 	mpi_req_cfg_page((_s), (_a), MPI_PG_POLL|MPI_PG_EXTENDED, \
201 	    (_h), (_r), (_p), (_l))
202 
203 static inline void
204 mpi_dvatosge(struct mpi_sge *sge, u_int64_t dva)
205 {
206 	htolem32(&sge->sg_addr_lo, dva);
207 	htolem32(&sge->sg_addr_hi, dva >> 32);
208 }
209 
210 int
211 mpi_attach(struct mpi_softc *sc)
212 {
213 	struct scsibus_attach_args	saa;
214 	struct mpi_ccb			*ccb;
215 
216 	printf("\n");
217 
218 	rw_init(&sc->sc_lock, "mpi_lock");
219 	task_set(&sc->sc_evt_rescan, mpi_fc_rescan, sc);
220 
221 	/* disable interrupts */
222 	mpi_write(sc, MPI_INTR_MASK,
223 	    MPI_INTR_MASK_REPLY | MPI_INTR_MASK_DOORBELL);
224 
225 	if (mpi_init(sc) != 0) {
226 		printf("%s: unable to initialise\n", DEVNAME(sc));
227 		return (1);
228 	}
229 
230 	if (mpi_iocfacts(sc) != 0) {
231 		printf("%s: unable to get iocfacts\n", DEVNAME(sc));
232 		return (1);
233 	}
234 
235 	if (mpi_alloc_ccbs(sc) != 0) {
236 		/* error already printed */
237 		return (1);
238 	}
239 
240 	if (mpi_alloc_replies(sc) != 0) {
241 		printf("%s: unable to allocate reply space\n", DEVNAME(sc));
242 		goto free_ccbs;
243 	}
244 
245 	if (mpi_iocinit(sc) != 0) {
246 		printf("%s: unable to send iocinit\n", DEVNAME(sc));
247 		goto free_ccbs;
248 	}
249 
250 	/* spin until we're operational */
251 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
252 	    MPI_DOORBELL_STATE_OPER) != 0) {
253 		printf("%s: state: 0x%08x\n", DEVNAME(sc),
254 		    mpi_read_db(sc) & MPI_DOORBELL_STATE);
255 		printf("%s: operational state timeout\n", DEVNAME(sc));
256 		goto free_ccbs;
257 	}
258 
259 	mpi_push_replies(sc);
260 
261 	if (mpi_portfacts(sc) != 0) {
262 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
263 		goto free_replies;
264 	}
265 
266 	if (mpi_cfg_coalescing(sc) != 0) {
267 		printf("%s: unable to configure coalescing\n", DEVNAME(sc));
268 		goto free_replies;
269 	}
270 
271 	switch (sc->sc_porttype) {
272 	case MPI_PORTFACTS_PORTTYPE_SAS:
273 		SIMPLEQ_INIT(&sc->sc_evt_scan_queue);
274 		mtx_init(&sc->sc_evt_scan_mtx, IPL_BIO);
275 		scsi_ioh_set(&sc->sc_evt_scan_handler, &sc->sc_iopool,
276 		    mpi_evt_sas_detach, sc);
277 		/* FALLTHROUGH */
278 	case MPI_PORTFACTS_PORTTYPE_FC:
279 		if (mpi_eventnotify(sc) != 0) {
280 			printf("%s: unable to enable events\n", DEVNAME(sc));
281 			goto free_replies;
282 		}
283 		break;
284 	}
285 
286 	if (mpi_portenable(sc) != 0) {
287 		printf("%s: unable to enable port\n", DEVNAME(sc));
288 		goto free_replies;
289 	}
290 
291 	if (mpi_fwupload(sc) != 0) {
292 		printf("%s: unable to upload firmware\n", DEVNAME(sc));
293 		goto free_replies;
294 	}
295 
296 	if (mpi_manufacturing(sc) != 0) {
297 		printf("%s: unable to fetch manufacturing info\n", DEVNAME(sc));
298 		goto free_replies;
299 	}
300 
301 	switch (sc->sc_porttype) {
302 	case MPI_PORTFACTS_PORTTYPE_SCSI:
303 		if (mpi_cfg_spi_port(sc) != 0) {
304 			printf("%s: unable to configure spi\n", DEVNAME(sc));
305 			goto free_replies;
306 		}
307 		mpi_squash_ppr(sc);
308 		break;
309 	case MPI_PORTFACTS_PORTTYPE_SAS:
310 		if (mpi_cfg_sas(sc) != 0) {
311 			printf("%s: unable to configure sas\n", DEVNAME(sc));
312 			goto free_replies;
313 		}
314 		break;
315 	case MPI_PORTFACTS_PORTTYPE_FC:
316 		if (mpi_cfg_fc(sc) != 0) {
317 			printf("%s: unable to configure fc\n", DEVNAME(sc));
318 			goto free_replies;
319 		}
320 		break;
321 	}
322 
323 	/* get raid pages */
324 	mpi_get_raid(sc);
325 #if NBIO > 0
326 	if (sc->sc_flags & MPI_F_RAID) {
327 		if (bio_register(&sc->sc_dev, mpi_ioctl) != 0)
328 			panic("%s: controller registration failed",
329 			    DEVNAME(sc));
330 		else {
331 			if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC,
332 			    2, 0, &sc->sc_cfg_hdr) != 0) {
333 				panic("%s: can't get IOC page 2 hdr",
334 				    DEVNAME(sc));
335 			}
336 
337 			sc->sc_vol_page = mallocarray(sc->sc_cfg_hdr.page_length,
338 			    4, M_TEMP, M_WAITOK | M_CANFAIL);
339 			if (sc->sc_vol_page == NULL) {
340 				panic("%s: can't get memory for IOC page 2, "
341 				    "bio disabled", DEVNAME(sc));
342 			}
343 
344 			if (mpi_cfg_page(sc, 0, &sc->sc_cfg_hdr, 1,
345 			    sc->sc_vol_page,
346 			    sc->sc_cfg_hdr.page_length * 4) != 0) {
347 				panic("%s: can't get IOC page 2", DEVNAME(sc));
348 			}
349 
350 			sc->sc_vol_list = (struct mpi_cfg_raid_vol *)
351 			    (sc->sc_vol_page + 1);
352 
353 			sc->sc_ioctl = mpi_ioctl;
354 		}
355 	}
356 #endif /* NBIO > 0 */
357 
358 	/* we should be good to go now, attach scsibus */
359 	sc->sc_link.adapter = &mpi_switch;
360 	sc->sc_link.adapter_softc = sc;
361 	sc->sc_link.adapter_target = sc->sc_target;
362 	sc->sc_link.adapter_buswidth = sc->sc_buswidth;
363 	sc->sc_link.openings = MAX(sc->sc_maxcmds / sc->sc_buswidth, 16);
364 	sc->sc_link.pool = &sc->sc_iopool;
365 
366 	saa.saa_sc_link = &sc->sc_link;
367 
368 	/* config_found() returns the scsibus attached to us */
369 	sc->sc_scsibus = (struct scsibus_softc *) config_found(&sc->sc_dev,
370 	    &saa, scsiprint);
371 
372 	/* do domain validation */
373 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI)
374 		mpi_run_ppr(sc);
375 
376 	/* enable interrupts */
377 	mpi_write(sc, MPI_INTR_MASK, MPI_INTR_MASK_DOORBELL);
378 
379 #if NBIO > 0
380 #ifndef SMALL_KERNEL
381 	mpi_create_sensors(sc);
382 #endif /* SMALL_KERNEL */
383 #endif /* NBIO > 0 */
384 
385 	return (0);
386 
387 free_replies:
388 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies), 0,
389 	    sc->sc_repq * MPI_REPLY_SIZE, BUS_DMASYNC_POSTREAD);
390 	mpi_dmamem_free(sc, sc->sc_replies);
391 free_ccbs:
392 	while ((ccb = mpi_get_ccb(sc)) != NULL)
393 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
394 	mpi_dmamem_free(sc, sc->sc_requests);
395 	free(sc->sc_ccbs, M_DEVBUF, 0);
396 
397 	return(1);
398 }
399 
400 int
401 mpi_cfg_spi_port(struct mpi_softc *sc)
402 {
403 	struct mpi_cfg_hdr		hdr;
404 	struct mpi_cfg_spi_port_pg1	port;
405 
406 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 1, 0x0,
407 	    &hdr) != 0)
408 		return (1);
409 
410 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port, sizeof(port)) != 0)
411 		return (1);
412 
413 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_spi_port_pg1\n", DEVNAME(sc));
414 	DNPRINTF(MPI_D_MISC, "%s:  port_scsi_id: %d port_resp_ids 0x%04x\n",
415 	    DEVNAME(sc), port.port_scsi_id, letoh16(port.port_resp_ids));
416 	DNPRINTF(MPI_D_MISC, "%s:  on_bus_timer_value: 0x%08x\n", DEVNAME(sc),
417 	    letoh32(port.port_scsi_id));
418 	DNPRINTF(MPI_D_MISC, "%s:  target_config: 0x%02x id_config: 0x%04x\n",
419 	    DEVNAME(sc), port.target_config, letoh16(port.id_config));
420 
421 	if (port.port_scsi_id == sc->sc_target &&
422 	    port.port_resp_ids == htole16(1 << sc->sc_target) &&
423 	    port.on_bus_timer_value != htole32(0x0))
424 		return (0);
425 
426 	DNPRINTF(MPI_D_MISC, "%s: setting port scsi id to %d\n", DEVNAME(sc),
427 	    sc->sc_target);
428 	port.port_scsi_id = sc->sc_target;
429 	port.port_resp_ids = htole16(1 << sc->sc_target);
430 	port.on_bus_timer_value = htole32(0x07000000); /* XXX magic */
431 
432 	if (mpi_cfg_page(sc, 0x0, &hdr, 0, &port, sizeof(port)) != 0) {
433 		printf("%s: unable to configure port scsi id\n", DEVNAME(sc));
434 		return (1);
435 	}
436 
437 	return (0);
438 }
439 
440 void
441 mpi_squash_ppr(struct mpi_softc *sc)
442 {
443 	struct mpi_cfg_hdr		hdr;
444 	struct mpi_cfg_spi_dev_pg1	page;
445 	int				i;
446 
447 	DNPRINTF(MPI_D_PPR, "%s: mpi_squash_ppr\n", DEVNAME(sc));
448 
449 	for (i = 0; i < sc->sc_buswidth; i++) {
450 		if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV,
451 		    1, i, &hdr) != 0)
452 			return;
453 
454 		if (mpi_cfg_page(sc, i, &hdr, 1, &page, sizeof(page)) != 0)
455 			return;
456 
457 		DNPRINTF(MPI_D_PPR, "%s:  target: %d req_params1: 0x%02x "
458 		    "req_offset: 0x%02x req_period: 0x%02x "
459 		    "req_params2: 0x%02x conf: 0x%08x\n", DEVNAME(sc), i,
460 		    page.req_params1, page.req_offset, page.req_period,
461 		    page.req_params2, letoh32(page.configuration));
462 
463 		page.req_params1 = 0x0;
464 		page.req_offset = 0x0;
465 		page.req_period = 0x0;
466 		page.req_params2 = 0x0;
467 		page.configuration = htole32(0x0);
468 
469 		if (mpi_cfg_page(sc, i, &hdr, 0, &page, sizeof(page)) != 0)
470 			return;
471 	}
472 }
473 
474 void
475 mpi_run_ppr(struct mpi_softc *sc)
476 {
477 	struct mpi_cfg_hdr		hdr;
478 	struct mpi_cfg_spi_port_pg0	port_pg;
479 	struct mpi_cfg_ioc_pg3		*physdisk_pg;
480 	struct mpi_cfg_raid_physdisk	*physdisk_list, *physdisk;
481 	size_t				pagelen;
482 	struct scsi_link		*link;
483 	int				i, tries;
484 
485 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 0, 0x0,
486 	    &hdr) != 0) {
487 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch header\n",
488 		    DEVNAME(sc));
489 		return;
490 	}
491 
492 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port_pg, sizeof(port_pg)) != 0) {
493 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch page\n",
494 		    DEVNAME(sc));
495 		return;
496 	}
497 
498 	for (i = 0; i < sc->sc_buswidth; i++) {
499 		link = scsi_get_link(sc->sc_scsibus, i, 0);
500 		if (link == NULL)
501 			continue;
502 
503 		/* do not ppr volumes */
504 		if (link->flags & SDEV_VIRTUAL)
505 			continue;
506 
507 		tries = 0;
508 		while (mpi_ppr(sc, link, NULL, port_pg.min_period,
509 		    port_pg.max_offset, tries) == EAGAIN)
510 			tries++;
511 	}
512 
513 	if ((sc->sc_flags & MPI_F_RAID) == 0)
514 		return;
515 
516 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 3, 0x0,
517 	    &hdr) != 0) {
518 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
519 		    "fetch ioc pg 3 header\n", DEVNAME(sc));
520 		return;
521 	}
522 
523 	pagelen = hdr.page_length * 4; /* dwords to bytes */
524 	physdisk_pg = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
525 	if (physdisk_pg == NULL) {
526 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
527 		    "allocate ioc pg 3\n", DEVNAME(sc));
528 		return;
529 	}
530 	physdisk_list = (struct mpi_cfg_raid_physdisk *)(physdisk_pg + 1);
531 
532 	if (mpi_cfg_page(sc, 0, &hdr, 1, physdisk_pg, pagelen) != 0) {
533 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s: mpi_run_ppr unable to "
534 		    "fetch ioc page 3\n", DEVNAME(sc));
535 		goto out;
536 	}
537 
538 	DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  no_phys_disks: %d\n", DEVNAME(sc),
539 	    physdisk_pg->no_phys_disks);
540 
541 	for (i = 0; i < physdisk_pg->no_phys_disks; i++) {
542 		physdisk = &physdisk_list[i];
543 
544 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  id: %d bus: %d ioc: %d "
545 		    "num: %d\n", DEVNAME(sc), physdisk->phys_disk_id,
546 		    physdisk->phys_disk_bus, physdisk->phys_disk_ioc,
547 		    physdisk->phys_disk_num);
548 
549 		if (physdisk->phys_disk_ioc != sc->sc_ioc_number)
550 			continue;
551 
552 		tries = 0;
553 		while (mpi_ppr(sc, NULL, physdisk, port_pg.min_period,
554 		    port_pg.max_offset, tries) == EAGAIN)
555 			tries++;
556 	}
557 
558 out:
559 	free(physdisk_pg, M_TEMP, pagelen);
560 }
561 
562 int
563 mpi_ppr(struct mpi_softc *sc, struct scsi_link *link,
564     struct mpi_cfg_raid_physdisk *physdisk, int period, int offset, int try)
565 {
566 	struct mpi_cfg_hdr		hdr0, hdr1;
567 	struct mpi_cfg_spi_dev_pg0	pg0;
568 	struct mpi_cfg_spi_dev_pg1	pg1;
569 	u_int32_t			address;
570 	int				id;
571 	int				raid = 0;
572 
573 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr period: %d offset: %d try: %d "
574 	    "link quirks: 0x%x\n", DEVNAME(sc), period, offset, try,
575 	    link->quirks);
576 
577 	if (try >= 3)
578 		return (EIO);
579 
580 	if (physdisk == NULL) {
581 		if ((link->inqdata.device & SID_TYPE) == T_PROCESSOR)
582 			return (EIO);
583 
584 		address = link->target;
585 		id = link->target;
586 	} else {
587 		raid = 1;
588 		address = (physdisk->phys_disk_bus << 8) |
589 		    (physdisk->phys_disk_id);
590 		id = physdisk->phys_disk_num;
591 	}
592 
593 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 0,
594 	    address, &hdr0) != 0) {
595 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 0\n",
596 		    DEVNAME(sc));
597 		return (EIO);
598 	}
599 
600 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 1,
601 	    address, &hdr1) != 0) {
602 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 1\n",
603 		    DEVNAME(sc));
604 		return (EIO);
605 	}
606 
607 #ifdef MPI_DEBUG
608 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
609 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 0\n",
610 		    DEVNAME(sc));
611 		return (EIO);
612 	}
613 
614 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
615 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
616 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
617 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
618 #endif
619 
620 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
621 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 1\n",
622 		    DEVNAME(sc));
623 		return (EIO);
624 	}
625 
626 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
627 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
628 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
629 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
630 
631 	pg1.req_params1 = 0;
632 	pg1.req_offset = offset;
633 	pg1.req_period = period;
634 	pg1.req_params2 &= ~MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH;
635 
636 	if (raid || !(link->quirks & SDEV_NOSYNC)) {
637 		pg1.req_params2 |= MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH_WIDE;
638 
639 		switch (try) {
640 		case 0: /* U320 */
641 			break;
642 		case 1: /* U160 */
643 			pg1.req_period = 0x09;
644 			break;
645 		case 2: /* U80 */
646 			pg1.req_period = 0x0a;
647 			break;
648 		}
649 
650 		if (pg1.req_period < 0x09) {
651 			/* Ultra320: enable QAS & PACKETIZED */
652 			pg1.req_params1 |= MPI_CFG_SPI_DEV_1_REQPARAMS_QAS |
653 			    MPI_CFG_SPI_DEV_1_REQPARAMS_PACKETIZED;
654 		}
655 		if (pg1.req_period < 0xa) {
656 			/* >= Ultra160: enable dual xfers */
657 			pg1.req_params1 |=
658 			    MPI_CFG_SPI_DEV_1_REQPARAMS_DUALXFERS;
659 		}
660 	}
661 
662 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
663 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
664 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
665 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
666 
667 	if (mpi_cfg_page(sc, address, &hdr1, 0, &pg1, sizeof(pg1)) != 0) {
668 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to write page 1\n",
669 		    DEVNAME(sc));
670 		return (EIO);
671 	}
672 
673 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
674 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 1\n",
675 		    DEVNAME(sc));
676 		return (EIO);
677 	}
678 
679 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
680 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
681 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
682 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
683 
684 	if (mpi_inq(sc, id, raid) != 0) {
685 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to do inquiry against "
686 		    "target %d\n", DEVNAME(sc), link->target);
687 		return (EIO);
688 	}
689 
690 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
691 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 0 after "
692 		    "inquiry\n", DEVNAME(sc));
693 		return (EIO);
694 	}
695 
696 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
697 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
698 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
699 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
700 
701 	if (!(lemtoh32(&pg0.information) & 0x07) && (try == 0)) {
702 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U320 ppr rejected\n",
703 		    DEVNAME(sc));
704 		return (EAGAIN);
705 	}
706 
707 	if ((((lemtoh32(&pg0.information) >> 8) & 0xff) > 0x09) && (try == 1)) {
708 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U160 ppr rejected\n",
709 		    DEVNAME(sc));
710 		return (EAGAIN);
711 	}
712 
713 	if (lemtoh32(&pg0.information) & 0x0e) {
714 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr ppr rejected: %0x\n",
715 		    DEVNAME(sc), lemtoh32(&pg0.information));
716 		return (EAGAIN);
717 	}
718 
719 	switch(pg0.neg_period) {
720 	case 0x08:
721 		period = 160;
722 		break;
723 	case 0x09:
724 		period = 80;
725 		break;
726 	case 0x0a:
727 		period = 40;
728 		break;
729 	case 0x0b:
730 		period = 20;
731 		break;
732 	case 0x0c:
733 		period = 10;
734 		break;
735 	default:
736 		period = 0;
737 		break;
738 	}
739 
740 	printf("%s: %s %d %s at %dMHz width %dbit offset %d "
741 	    "QAS %d DT %d IU %d\n", DEVNAME(sc), raid ? "phys disk" : "target",
742 	    id, period ? "Sync" : "Async", period,
743 	    (pg0.neg_params2 & MPI_CFG_SPI_DEV_0_NEGPARAMS_WIDTH_WIDE) ? 16 : 8,
744 	    pg0.neg_offset,
745 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_QAS) ? 1 : 0,
746 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_DUALXFERS) ? 1 : 0,
747 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_PACKETIZED) ? 1 : 0);
748 
749 	return (0);
750 }
751 
752 int
753 mpi_inq(struct mpi_softc *sc, u_int16_t target, int physdisk)
754 {
755 	struct mpi_ccb			*ccb;
756 	struct scsi_inquiry		inq;
757 	struct inq_bundle {
758 		struct mpi_msg_scsi_io		io;
759 		struct mpi_sge			sge;
760 		struct scsi_inquiry_data	inqbuf;
761 		struct scsi_sense_data		sense;
762 	} __packed			*bundle;
763 	struct mpi_msg_scsi_io		*io;
764 	struct mpi_sge			*sge;
765 
766 	DNPRINTF(MPI_D_PPR, "%s: mpi_inq\n", DEVNAME(sc));
767 
768 	memset(&inq, 0, sizeof(inq));
769 	inq.opcode = INQUIRY;
770 	_lto2b(sizeof(struct scsi_inquiry_data), inq.length);
771 
772 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
773 	if (ccb == NULL)
774 		return (1);
775 
776 	ccb->ccb_done = mpi_empty_done;
777 
778 	bundle = ccb->ccb_cmd;
779 	io = &bundle->io;
780 	sge = &bundle->sge;
781 
782 	io->function = physdisk ? MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH :
783 	    MPI_FUNCTION_SCSI_IO_REQUEST;
784 	/*
785 	 * bus is always 0
786 	 * io->bus = htole16(sc->sc_bus);
787 	 */
788 	io->target_id = target;
789 
790 	io->cdb_length = sizeof(inq);
791 	io->sense_buf_len = sizeof(struct scsi_sense_data);
792 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
793 
794 	/*
795 	 * always lun 0
796 	 * io->lun[0] = htobe16(link->lun);
797 	 */
798 
799 	io->direction = MPI_SCSIIO_DIR_READ;
800 	io->tagging = MPI_SCSIIO_ATTR_NO_DISCONNECT;
801 
802 	memcpy(io->cdb, &inq, sizeof(inq));
803 
804 	htolem32(&io->data_length, sizeof(struct scsi_inquiry_data));
805 
806 	htolem32(&io->sense_buf_low_addr, ccb->ccb_cmd_dva +
807 	    offsetof(struct inq_bundle, sense));
808 
809 	htolem32(&sge->sg_hdr, MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64 |
810 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
811 	    (u_int32_t)sizeof(inq));
812 
813 	mpi_dvatosge(sge, ccb->ccb_cmd_dva +
814 	    offsetof(struct inq_bundle, inqbuf));
815 
816 	if (mpi_poll(sc, ccb, 5000) != 0)
817 		return (1);
818 
819 	if (ccb->ccb_rcb != NULL)
820 		mpi_push_reply(sc, ccb->ccb_rcb);
821 
822 	scsi_io_put(&sc->sc_iopool, ccb);
823 
824 	return (0);
825 }
826 
827 int
828 mpi_cfg_sas(struct mpi_softc *sc)
829 {
830 	struct mpi_ecfg_hdr		ehdr;
831 	struct mpi_cfg_sas_iou_pg1	*pg;
832 	size_t				pagelen;
833 	int				rv = 0;
834 
835 	if (mpi_ecfg_header(sc, MPI_CONFIG_REQ_EXTPAGE_TYPE_SAS_IO_UNIT, 1, 0,
836 	    &ehdr) != 0)
837 		return (0);
838 
839 	pagelen = lemtoh16(&ehdr.ext_page_length) * 4;
840 	pg = malloc(pagelen, M_TEMP, M_NOWAIT | M_ZERO);
841 	if (pg == NULL)
842 		return (ENOMEM);
843 
844 	if (mpi_ecfg_page(sc, 0, &ehdr, 1, pg, pagelen) != 0)
845 		goto out;
846 
847 	if (pg->max_sata_q_depth != 32) {
848 		pg->max_sata_q_depth = 32;
849 
850 		if (mpi_ecfg_page(sc, 0, &ehdr, 0, pg, pagelen) != 0)
851 			goto out;
852 	}
853 
854 out:
855 	free(pg, M_TEMP, pagelen);
856 	return (rv);
857 }
858 
859 int
860 mpi_cfg_fc(struct mpi_softc *sc)
861 {
862 	struct mpi_cfg_hdr		hdr;
863 	struct mpi_cfg_fc_port_pg0	pg0;
864 	struct mpi_cfg_fc_port_pg1	pg1;
865 
866 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_FC_PORT, 0, 0,
867 	    &hdr) != 0) {
868 		printf("%s: unable to fetch FC port header 0\n", DEVNAME(sc));
869 		return (1);
870 	}
871 
872 	if (mpi_cfg_page(sc, 0, &hdr, 1, &pg0, sizeof(pg0)) != 0) {
873 		printf("%s: unable to fetch FC port page 0\n", DEVNAME(sc));
874 		return (1);
875 	}
876 
877 	sc->sc_link.port_wwn = letoh64(pg0.wwpn);
878 	sc->sc_link.node_wwn = letoh64(pg0.wwnn);
879 
880 	/* configure port config more to our liking */
881 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_FC_PORT, 1, 0,
882 	    &hdr) != 0) {
883 		printf("%s: unable to fetch FC port header 1\n", DEVNAME(sc));
884 		return (1);
885 	}
886 
887 	if (mpi_cfg_page(sc, 0, &hdr, 1, &pg1, sizeof(pg1)) != 0) {
888 		printf("%s: unable to fetch FC port page 1\n", DEVNAME(sc));
889 		return (1);
890 	}
891 
892 	SET(pg1.flags, htole32(MPI_CFG_FC_PORT_0_FLAGS_IMMEDIATE_ERROR |
893 	    MPI_CFG_FC_PORT_0_FLAGS_VERBOSE_RESCAN));
894 
895 	if (mpi_cfg_page(sc, 0, &hdr, 0, &pg1, sizeof(pg1)) != 0) {
896 		printf("%s: unable to set FC port page 1\n", DEVNAME(sc));
897 		return (1);
898 	}
899 
900 	return (0);
901 }
902 
903 void
904 mpi_detach(struct mpi_softc *sc)
905 {
906 
907 }
908 
909 int
910 mpi_intr(void *arg)
911 {
912 	struct mpi_softc		*sc = arg;
913 	u_int32_t			reg;
914 	int				rv = 0;
915 
916 	if ((mpi_read_intr(sc) & MPI_INTR_STATUS_REPLY) == 0)
917 		return (rv);
918 
919 	while ((reg = mpi_pop_reply(sc)) != 0xffffffff) {
920 		mpi_reply(sc, reg);
921 		rv = 1;
922 	}
923 
924 	return (rv);
925 }
926 
927 void
928 mpi_reply(struct mpi_softc *sc, u_int32_t reg)
929 {
930 	struct mpi_ccb			*ccb;
931 	struct mpi_rcb			*rcb = NULL;
932 	struct mpi_msg_reply		*reply = NULL;
933 	u_int32_t			reply_dva;
934 	int				id;
935 	int				i;
936 
937 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply reg: 0x%08x\n", DEVNAME(sc), reg);
938 
939 	if (reg & MPI_REPLY_QUEUE_ADDRESS) {
940 		reply_dva = (reg & MPI_REPLY_QUEUE_ADDRESS_MASK) << 1;
941 		i = (reply_dva - (u_int32_t)MPI_DMA_DVA(sc->sc_replies)) /
942 		    MPI_REPLY_SIZE;
943 		rcb = &sc->sc_rcbs[i];
944 
945 		bus_dmamap_sync(sc->sc_dmat,
946 		    MPI_DMA_MAP(sc->sc_replies), rcb->rcb_offset,
947 		    MPI_REPLY_SIZE, BUS_DMASYNC_POSTREAD);
948 
949 		reply = rcb->rcb_reply;
950 
951 		id = lemtoh32(&reply->msg_context);
952 	} else {
953 		switch (reg & MPI_REPLY_QUEUE_TYPE_MASK) {
954 		case MPI_REPLY_QUEUE_TYPE_INIT:
955 			id = reg & MPI_REPLY_QUEUE_CONTEXT;
956 			break;
957 
958 		default:
959 			panic("%s: unsupported context reply",
960 			    DEVNAME(sc));
961 		}
962 	}
963 
964 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply id: %d reply: %p\n",
965 	    DEVNAME(sc), id, reply);
966 
967 	ccb = &sc->sc_ccbs[id];
968 
969 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
970 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
971 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
972 	ccb->ccb_state = MPI_CCB_READY;
973 	ccb->ccb_rcb = rcb;
974 
975 	ccb->ccb_done(ccb);
976 }
977 
978 struct mpi_dmamem *
979 mpi_dmamem_alloc(struct mpi_softc *sc, size_t size)
980 {
981 	struct mpi_dmamem		*mdm;
982 	int				nsegs;
983 
984 	mdm = malloc(sizeof(struct mpi_dmamem), M_DEVBUF, M_NOWAIT | M_ZERO);
985 	if (mdm == NULL)
986 		return (NULL);
987 
988 	mdm->mdm_size = size;
989 
990 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
991 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mdm->mdm_map) != 0)
992 		goto mdmfree;
993 
994 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &mdm->mdm_seg,
995 	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
996 		goto destroy;
997 
998 	if (bus_dmamem_map(sc->sc_dmat, &mdm->mdm_seg, nsegs, size,
999 	    &mdm->mdm_kva, BUS_DMA_NOWAIT) != 0)
1000 		goto free;
1001 
1002 	if (bus_dmamap_load(sc->sc_dmat, mdm->mdm_map, mdm->mdm_kva, size,
1003 	    NULL, BUS_DMA_NOWAIT) != 0)
1004 		goto unmap;
1005 
1006 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_alloc size: %d mdm: %#x "
1007 	    "map: %#x nsegs: %d segs: %#x kva: %x\n",
1008 	    DEVNAME(sc), size, mdm->mdm_map, nsegs, mdm->mdm_seg, mdm->mdm_kva);
1009 
1010 	return (mdm);
1011 
1012 unmap:
1013 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, size);
1014 free:
1015 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
1016 destroy:
1017 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
1018 mdmfree:
1019 	free(mdm, M_DEVBUF, sizeof *mdm);
1020 
1021 	return (NULL);
1022 }
1023 
1024 void
1025 mpi_dmamem_free(struct mpi_softc *sc, struct mpi_dmamem *mdm)
1026 {
1027 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_free %#x\n", DEVNAME(sc), mdm);
1028 
1029 	bus_dmamap_unload(sc->sc_dmat, mdm->mdm_map);
1030 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, mdm->mdm_size);
1031 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
1032 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
1033 	free(mdm, M_DEVBUF, sizeof *mdm);
1034 }
1035 
1036 int
1037 mpi_alloc_ccbs(struct mpi_softc *sc)
1038 {
1039 	struct mpi_ccb			*ccb;
1040 	u_int8_t			*cmd;
1041 	int				i;
1042 
1043 	SLIST_INIT(&sc->sc_ccb_free);
1044 	mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
1045 
1046 	sc->sc_ccbs = mallocarray(sc->sc_maxcmds, sizeof(struct mpi_ccb),
1047 	    M_DEVBUF, M_WAITOK | M_CANFAIL | M_ZERO);
1048 	if (sc->sc_ccbs == NULL) {
1049 		printf("%s: unable to allocate ccbs\n", DEVNAME(sc));
1050 		return (1);
1051 	}
1052 
1053 	sc->sc_requests = mpi_dmamem_alloc(sc,
1054 	    MPI_REQUEST_SIZE * sc->sc_maxcmds);
1055 	if (sc->sc_requests == NULL) {
1056 		printf("%s: unable to allocate ccb dmamem\n", DEVNAME(sc));
1057 		goto free_ccbs;
1058 	}
1059 	cmd = MPI_DMA_KVA(sc->sc_requests);
1060 	memset(cmd, 0, MPI_REQUEST_SIZE * sc->sc_maxcmds);
1061 
1062 	for (i = 0; i < sc->sc_maxcmds; i++) {
1063 		ccb = &sc->sc_ccbs[i];
1064 
1065 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
1066 		    sc->sc_max_sgl_len, MAXPHYS, 0,
1067 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
1068 		    &ccb->ccb_dmamap) != 0) {
1069 			printf("%s: unable to create dma map\n", DEVNAME(sc));
1070 			goto free_maps;
1071 		}
1072 
1073 		ccb->ccb_sc = sc;
1074 		ccb->ccb_id = i;
1075 		ccb->ccb_offset = MPI_REQUEST_SIZE * i;
1076 		ccb->ccb_state = MPI_CCB_READY;
1077 
1078 		ccb->ccb_cmd = &cmd[ccb->ccb_offset];
1079 		ccb->ccb_cmd_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_requests) +
1080 		    ccb->ccb_offset;
1081 
1082 		DNPRINTF(MPI_D_CCB, "%s: mpi_alloc_ccbs(%d) ccb: %#x map: %#x "
1083 		    "sc: %#x id: %#x offs: %#x cmd: %#x dva: %#x\n",
1084 		    DEVNAME(sc), i, ccb, ccb->ccb_dmamap, ccb->ccb_sc,
1085 		    ccb->ccb_id, ccb->ccb_offset, ccb->ccb_cmd,
1086 		    ccb->ccb_cmd_dva);
1087 
1088 		mpi_put_ccb(sc, ccb);
1089 	}
1090 
1091 	scsi_iopool_init(&sc->sc_iopool, sc, mpi_get_ccb, mpi_put_ccb);
1092 
1093 	return (0);
1094 
1095 free_maps:
1096 	while ((ccb = mpi_get_ccb(sc)) != NULL)
1097 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1098 
1099 	mpi_dmamem_free(sc, sc->sc_requests);
1100 free_ccbs:
1101 	free(sc->sc_ccbs, M_DEVBUF, 0);
1102 
1103 	return (1);
1104 }
1105 
1106 void *
1107 mpi_get_ccb(void *xsc)
1108 {
1109 	struct mpi_softc		*sc = xsc;
1110 	struct mpi_ccb			*ccb;
1111 
1112 	mtx_enter(&sc->sc_ccb_mtx);
1113 	ccb = SLIST_FIRST(&sc->sc_ccb_free);
1114 	if (ccb != NULL) {
1115 		SLIST_REMOVE_HEAD(&sc->sc_ccb_free, ccb_link);
1116 		ccb->ccb_state = MPI_CCB_READY;
1117 	}
1118 	mtx_leave(&sc->sc_ccb_mtx);
1119 
1120 	DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb %p\n", DEVNAME(sc), ccb);
1121 
1122 	return (ccb);
1123 }
1124 
1125 void
1126 mpi_put_ccb(void *xsc, void *io)
1127 {
1128 	struct mpi_softc		*sc = xsc;
1129 	struct mpi_ccb			*ccb = io;
1130 
1131 	DNPRINTF(MPI_D_CCB, "%s: mpi_put_ccb %p\n", DEVNAME(sc), ccb);
1132 
1133 #ifdef DIAGNOSTIC
1134 	if (ccb->ccb_state == MPI_CCB_FREE)
1135 		panic("mpi_put_ccb: double free");
1136 #endif
1137 
1138 	ccb->ccb_state = MPI_CCB_FREE;
1139 	ccb->ccb_cookie = NULL;
1140 	ccb->ccb_done = NULL;
1141 	memset(ccb->ccb_cmd, 0, MPI_REQUEST_SIZE);
1142 	mtx_enter(&sc->sc_ccb_mtx);
1143 	SLIST_INSERT_HEAD(&sc->sc_ccb_free, ccb, ccb_link);
1144 	mtx_leave(&sc->sc_ccb_mtx);
1145 }
1146 
1147 int
1148 mpi_alloc_replies(struct mpi_softc *sc)
1149 {
1150 	DNPRINTF(MPI_D_MISC, "%s: mpi_alloc_replies\n", DEVNAME(sc));
1151 
1152 	sc->sc_rcbs = mallocarray(sc->sc_repq, sizeof(struct mpi_rcb), M_DEVBUF,
1153 	    M_WAITOK|M_CANFAIL);
1154 	if (sc->sc_rcbs == NULL)
1155 		return (1);
1156 
1157 	sc->sc_replies = mpi_dmamem_alloc(sc, sc->sc_repq * MPI_REPLY_SIZE);
1158 	if (sc->sc_replies == NULL) {
1159 		free(sc->sc_rcbs, M_DEVBUF, 0);
1160 		return (1);
1161 	}
1162 
1163 	return (0);
1164 }
1165 
1166 void
1167 mpi_push_reply(struct mpi_softc *sc, struct mpi_rcb *rcb)
1168 {
1169 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
1170 	    rcb->rcb_offset, MPI_REPLY_SIZE, BUS_DMASYNC_PREREAD);
1171 	mpi_push_reply_db(sc, rcb->rcb_reply_dva);
1172 }
1173 
1174 void
1175 mpi_push_replies(struct mpi_softc *sc)
1176 {
1177 	struct mpi_rcb			*rcb;
1178 	char				*kva = MPI_DMA_KVA(sc->sc_replies);
1179 	int				i;
1180 
1181 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies), 0,
1182 	    sc->sc_repq * MPI_REPLY_SIZE, BUS_DMASYNC_PREREAD);
1183 
1184 	for (i = 0; i < sc->sc_repq; i++) {
1185 		rcb = &sc->sc_rcbs[i];
1186 
1187 		rcb->rcb_reply = kva + MPI_REPLY_SIZE * i;
1188 		rcb->rcb_offset = MPI_REPLY_SIZE * i;
1189 		rcb->rcb_reply_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_replies) +
1190 		    MPI_REPLY_SIZE * i;
1191 		mpi_push_reply_db(sc, rcb->rcb_reply_dva);
1192 	}
1193 }
1194 
1195 void
1196 mpi_start(struct mpi_softc *sc, struct mpi_ccb *ccb)
1197 {
1198 	struct mpi_msg_request *msg;
1199 
1200 	DNPRINTF(MPI_D_RW, "%s: mpi_start %#x\n", DEVNAME(sc),
1201 	    ccb->ccb_cmd_dva);
1202 
1203 	msg = ccb->ccb_cmd;
1204 	htolem32(&msg->msg_context, ccb->ccb_id);
1205 
1206 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
1207 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
1208 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1209 
1210 	ccb->ccb_state = MPI_CCB_QUEUED;
1211 	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
1212 	    MPI_REQ_QUEUE, ccb->ccb_cmd_dva);
1213 }
1214 
1215 int
1216 mpi_poll(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1217 {
1218 	void				(*done)(struct mpi_ccb *);
1219 	void				*cookie;
1220 	int				rv = 1;
1221 	u_int32_t			reg;
1222 
1223 	DNPRINTF(MPI_D_INTR, "%s: mpi_poll timeout %d\n", DEVNAME(sc),
1224 	    timeout);
1225 
1226 	done = ccb->ccb_done;
1227 	cookie = ccb->ccb_cookie;
1228 
1229 	ccb->ccb_done = mpi_poll_done;
1230 	ccb->ccb_cookie = &rv;
1231 
1232 	mpi_start(sc, ccb);
1233 	while (rv == 1) {
1234 		reg = mpi_pop_reply(sc);
1235 		if (reg == 0xffffffff) {
1236 			if (timeout-- == 0) {
1237 				printf("%s: timeout\n", DEVNAME(sc));
1238 				goto timeout;
1239 			}
1240 
1241 			delay(1000);
1242 			continue;
1243 		}
1244 
1245 		mpi_reply(sc, reg);
1246 	}
1247 
1248 	ccb->ccb_cookie = cookie;
1249 	done(ccb);
1250 
1251 timeout:
1252 	return (rv);
1253 }
1254 
1255 void
1256 mpi_poll_done(struct mpi_ccb *ccb)
1257 {
1258 	int				*rv = ccb->ccb_cookie;
1259 
1260 	*rv = 0;
1261 }
1262 
1263 void
1264 mpi_wait(struct mpi_softc *sc, struct mpi_ccb *ccb)
1265 {
1266 	struct mutex			cookie = MUTEX_INITIALIZER(IPL_BIO);
1267 	void				(*done)(struct mpi_ccb *);
1268 
1269 	done = ccb->ccb_done;
1270 	ccb->ccb_done = mpi_wait_done;
1271 	ccb->ccb_cookie = &cookie;
1272 
1273 	/* XXX this will wait forever for the ccb to complete */
1274 
1275 	mpi_start(sc, ccb);
1276 
1277 	mtx_enter(&cookie);
1278 	while (ccb->ccb_cookie != NULL)
1279 		msleep_nsec(ccb, &cookie, PRIBIO, "mpiwait", INFSLP);
1280 	mtx_leave(&cookie);
1281 
1282 	done(ccb);
1283 }
1284 
1285 void
1286 mpi_wait_done(struct mpi_ccb *ccb)
1287 {
1288 	struct mutex			*cookie = ccb->ccb_cookie;
1289 
1290 	mtx_enter(cookie);
1291 	ccb->ccb_cookie = NULL;
1292 	wakeup_one(ccb);
1293 	mtx_leave(cookie);
1294 }
1295 
1296 void
1297 mpi_scsi_cmd(struct scsi_xfer *xs)
1298 {
1299 	struct scsi_link		*link = xs->sc_link;
1300 	struct mpi_softc		*sc = link->adapter_softc;
1301 	struct mpi_ccb			*ccb;
1302 	struct mpi_ccb_bundle		*mcb;
1303 	struct mpi_msg_scsi_io		*io;
1304 
1305 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd\n", DEVNAME(sc));
1306 
1307 	KERNEL_UNLOCK();
1308 
1309 	if (xs->cmdlen > MPI_CDB_LEN) {
1310 		DNPRINTF(MPI_D_CMD, "%s: CBD too big %d\n",
1311 		    DEVNAME(sc), xs->cmdlen);
1312 		memset(&xs->sense, 0, sizeof(xs->sense));
1313 		xs->sense.error_code = SSD_ERRCODE_VALID | SSD_ERRCODE_CURRENT;
1314 		xs->sense.flags = SKEY_ILLEGAL_REQUEST;
1315 		xs->sense.add_sense_code = 0x20;
1316 		xs->error = XS_SENSE;
1317 		goto done;
1318 	}
1319 
1320 	ccb = xs->io;
1321 
1322 	DNPRINTF(MPI_D_CMD, "%s: ccb_id: %d xs->flags: 0x%x\n",
1323 	    DEVNAME(sc), ccb->ccb_id, xs->flags);
1324 
1325 	ccb->ccb_cookie = xs;
1326 	ccb->ccb_done = mpi_scsi_cmd_done;
1327 
1328 	mcb = ccb->ccb_cmd;
1329 	io = &mcb->mcb_io;
1330 
1331 	io->function = MPI_FUNCTION_SCSI_IO_REQUEST;
1332 	/*
1333 	 * bus is always 0
1334 	 * io->bus = htole16(sc->sc_bus);
1335 	 */
1336 	io->target_id = link->target;
1337 
1338 	io->cdb_length = xs->cmdlen;
1339 	io->sense_buf_len = sizeof(xs->sense);
1340 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
1341 
1342 	htobem16(&io->lun[0], link->lun);
1343 
1344 	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
1345 	case SCSI_DATA_IN:
1346 		io->direction = MPI_SCSIIO_DIR_READ;
1347 		break;
1348 	case SCSI_DATA_OUT:
1349 		io->direction = MPI_SCSIIO_DIR_WRITE;
1350 		break;
1351 	default:
1352 		io->direction = MPI_SCSIIO_DIR_NONE;
1353 		break;
1354 	}
1355 
1356 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SCSI &&
1357 	    (link->quirks & SDEV_NOTAGS))
1358 		io->tagging = MPI_SCSIIO_ATTR_UNTAGGED;
1359 	else
1360 		io->tagging = MPI_SCSIIO_ATTR_SIMPLE_Q;
1361 
1362 	memcpy(io->cdb, xs->cmd, xs->cmdlen);
1363 
1364 	htolem32(&io->data_length, xs->datalen);
1365 
1366 	htolem32(&io->sense_buf_low_addr, ccb->ccb_cmd_dva +
1367 	    offsetof(struct mpi_ccb_bundle, mcb_sense));
1368 
1369 	if (mpi_load_xs(ccb) != 0)
1370 		goto stuffup;
1371 
1372 	timeout_set(&xs->stimeout, mpi_timeout_xs, ccb);
1373 
1374 	if (xs->flags & SCSI_POLL) {
1375 		if (mpi_poll(sc, ccb, xs->timeout) != 0)
1376 			goto stuffup;
1377 	} else
1378 		mpi_start(sc, ccb);
1379 
1380 	KERNEL_LOCK();
1381 	return;
1382 
1383 stuffup:
1384 	xs->error = XS_DRIVER_STUFFUP;
1385 done:
1386 	KERNEL_LOCK();
1387 	scsi_done(xs);
1388 }
1389 
1390 void
1391 mpi_scsi_cmd_done(struct mpi_ccb *ccb)
1392 {
1393 	struct mpi_softc		*sc = ccb->ccb_sc;
1394 	struct scsi_xfer		*xs = ccb->ccb_cookie;
1395 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1396 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1397 	struct mpi_msg_scsi_io_error	*sie;
1398 
1399 	if (xs->datalen != 0) {
1400 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1401 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1402 		    BUS_DMASYNC_POSTWRITE);
1403 
1404 		bus_dmamap_unload(sc->sc_dmat, dmap);
1405 	}
1406 
1407 	/* timeout_del */
1408 	xs->error = XS_NOERROR;
1409 	xs->resid = 0;
1410 
1411 	if (ccb->ccb_rcb == NULL) {
1412 		/* no scsi error, we're ok so drop out early */
1413 		xs->status = SCSI_OK;
1414 		KERNEL_LOCK();
1415 		scsi_done(xs);
1416 		KERNEL_UNLOCK();
1417 		return;
1418 	}
1419 
1420 	sie = ccb->ccb_rcb->rcb_reply;
1421 
1422 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd_done xs cmd: 0x%02x len: %d "
1423 	    "flags 0x%x\n", DEVNAME(sc), xs->cmd->opcode, xs->datalen,
1424 	    xs->flags);
1425 	DNPRINTF(MPI_D_CMD, "%s:  target_id: %d bus: %d msg_length: %d "
1426 	    "function: 0x%02x\n", DEVNAME(sc), sie->target_id, sie->bus,
1427 	    sie->msg_length, sie->function);
1428 	DNPRINTF(MPI_D_CMD, "%s:  cdb_length: %d sense_buf_length: %d "
1429 	    "msg_flags: 0x%02x\n", DEVNAME(sc), sie->cdb_length,
1430 	    sie->sense_buf_len, sie->msg_flags);
1431 	DNPRINTF(MPI_D_CMD, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1432 	    letoh32(sie->msg_context));
1433 	DNPRINTF(MPI_D_CMD, "%s:  scsi_status: 0x%02x scsi_state: 0x%02x "
1434 	    "ioc_status: 0x%04x\n", DEVNAME(sc), sie->scsi_status,
1435 	    sie->scsi_state, letoh16(sie->ioc_status));
1436 	DNPRINTF(MPI_D_CMD, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1437 	    letoh32(sie->ioc_loginfo));
1438 	DNPRINTF(MPI_D_CMD, "%s:  transfer_count: %d\n", DEVNAME(sc),
1439 	    letoh32(sie->transfer_count));
1440 	DNPRINTF(MPI_D_CMD, "%s:  sense_count: %d\n", DEVNAME(sc),
1441 	    letoh32(sie->sense_count));
1442 	DNPRINTF(MPI_D_CMD, "%s:  response_info: 0x%08x\n", DEVNAME(sc),
1443 	    letoh32(sie->response_info));
1444 	DNPRINTF(MPI_D_CMD, "%s:  tag: 0x%04x\n", DEVNAME(sc),
1445 	    letoh16(sie->tag));
1446 
1447 	if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_NO_SCSI_STATUS)
1448 		xs->status = SCSI_TERMINATED;
1449 	else
1450 		xs->status = sie->scsi_status;
1451 	xs->resid = 0;
1452 
1453 	switch (lemtoh16(&sie->ioc_status)) {
1454 	case MPI_IOCSTATUS_SCSI_DATA_UNDERRUN:
1455 		xs->resid = xs->datalen - lemtoh32(&sie->transfer_count);
1456 		/* FALLTHROUGH */
1457 	case MPI_IOCSTATUS_SUCCESS:
1458 	case MPI_IOCSTATUS_SCSI_RECOVERED_ERROR:
1459 		switch (xs->status) {
1460 		case SCSI_OK:
1461 			xs->error = XS_NOERROR;
1462 			break;
1463 
1464 		case SCSI_CHECK:
1465 			xs->error = XS_SENSE;
1466 			break;
1467 
1468 		case SCSI_BUSY:
1469 		case SCSI_QUEUE_FULL:
1470 			xs->error = XS_BUSY;
1471 			break;
1472 
1473 		default:
1474 			xs->error = XS_DRIVER_STUFFUP;
1475 			break;
1476 		}
1477 		break;
1478 
1479 	case MPI_IOCSTATUS_BUSY:
1480 	case MPI_IOCSTATUS_INSUFFICIENT_RESOURCES:
1481 		xs->error = XS_BUSY;
1482 		break;
1483 
1484 	case MPI_IOCSTATUS_SCSI_INVALID_BUS:
1485 	case MPI_IOCSTATUS_SCSI_INVALID_TARGETID:
1486 	case MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1487 		xs->error = XS_SELTIMEOUT;
1488 		break;
1489 
1490 	case MPI_IOCSTATUS_SCSI_IOC_TERMINATED:
1491 	case MPI_IOCSTATUS_SCSI_EXT_TERMINATED:
1492 		xs->error = XS_RESET;
1493 		break;
1494 
1495 	default:
1496 		xs->error = XS_DRIVER_STUFFUP;
1497 		break;
1498 	}
1499 
1500 	if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_AUTOSENSE_VALID)
1501 		memcpy(&xs->sense, &mcb->mcb_sense, sizeof(xs->sense));
1502 
1503 	DNPRINTF(MPI_D_CMD, "%s:  xs err: 0x%02x status: %d\n", DEVNAME(sc),
1504 	    xs->error, xs->status);
1505 
1506 	mpi_push_reply(sc, ccb->ccb_rcb);
1507 	KERNEL_LOCK();
1508 	scsi_done(xs);
1509 	KERNEL_UNLOCK();
1510 }
1511 
1512 void
1513 mpi_timeout_xs(void *arg)
1514 {
1515 	/* XXX */
1516 }
1517 
1518 int
1519 mpi_load_xs(struct mpi_ccb *ccb)
1520 {
1521 	struct mpi_softc		*sc = ccb->ccb_sc;
1522 	struct scsi_xfer		*xs = ccb->ccb_cookie;
1523 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1524 	struct mpi_msg_scsi_io		*io = &mcb->mcb_io;
1525 	struct mpi_sge			*sge = NULL;
1526 	struct mpi_sge			*nsge = &mcb->mcb_sgl[0];
1527 	struct mpi_sge			*ce = NULL, *nce;
1528 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1529 	u_int32_t			addr, flags;
1530 	int				i, error;
1531 
1532 	if (xs->datalen == 0) {
1533 		htolem32(&nsge->sg_hdr, MPI_SGE_FL_TYPE_SIMPLE |
1534 		    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
1535 		return (0);
1536 	}
1537 
1538 	error = bus_dmamap_load(sc->sc_dmat, dmap,
1539 	    xs->data, xs->datalen, NULL, BUS_DMA_STREAMING |
1540 	    ((xs->flags & SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK));
1541 	if (error) {
1542 		printf("%s: error %d loading dmamap\n", DEVNAME(sc), error);
1543 		return (1);
1544 	}
1545 
1546 	flags = MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64;
1547 	if (xs->flags & SCSI_DATA_OUT)
1548 		flags |= MPI_SGE_FL_DIR_OUT;
1549 
1550 	if (dmap->dm_nsegs > sc->sc_first_sgl_len) {
1551 		ce = &mcb->mcb_sgl[sc->sc_first_sgl_len - 1];
1552 		io->chain_offset = (u_int32_t *)ce - (u_int32_t *)io;
1553 	}
1554 
1555 	for (i = 0; i < dmap->dm_nsegs; i++) {
1556 
1557 		if (nsge == ce) {
1558 			nsge++;
1559 			sge->sg_hdr |= htole32(MPI_SGE_FL_LAST);
1560 
1561 			if ((dmap->dm_nsegs - i) > sc->sc_chain_len) {
1562 				nce = &nsge[sc->sc_chain_len - 1];
1563 				addr = (u_int32_t *)nce - (u_int32_t *)nsge;
1564 				addr = addr << 16 |
1565 				    sizeof(struct mpi_sge) * sc->sc_chain_len;
1566 			} else {
1567 				nce = NULL;
1568 				addr = sizeof(struct mpi_sge) *
1569 				    (dmap->dm_nsegs - i);
1570 			}
1571 
1572 			ce->sg_hdr = htole32(MPI_SGE_FL_TYPE_CHAIN |
1573 			    MPI_SGE_FL_SIZE_64 | addr);
1574 
1575 			mpi_dvatosge(ce, ccb->ccb_cmd_dva +
1576 			    ((u_int8_t *)nsge - (u_int8_t *)mcb));
1577 
1578 			ce = nce;
1579 		}
1580 
1581 		DNPRINTF(MPI_D_DMA, "%s:  %d: %d 0x%016llx\n", DEVNAME(sc),
1582 		    i, dmap->dm_segs[i].ds_len,
1583 		    (u_int64_t)dmap->dm_segs[i].ds_addr);
1584 
1585 		sge = nsge++;
1586 
1587 		sge->sg_hdr = htole32(flags | dmap->dm_segs[i].ds_len);
1588 		mpi_dvatosge(sge, dmap->dm_segs[i].ds_addr);
1589 	}
1590 
1591 	/* terminate list */
1592 	sge->sg_hdr |= htole32(MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
1593 	    MPI_SGE_FL_EOL);
1594 
1595 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1596 	    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
1597 	    BUS_DMASYNC_PREWRITE);
1598 
1599 	return (0);
1600 }
1601 
1602 int
1603 mpi_scsi_probe_virtual(struct scsi_link *link)
1604 {
1605 	struct mpi_softc		*sc = link->adapter_softc;
1606 	struct mpi_cfg_hdr		hdr;
1607 	struct mpi_cfg_raid_vol_pg0	*rp0;
1608 	int				len;
1609 	int				rv;
1610 
1611 	if (!ISSET(sc->sc_flags, MPI_F_RAID))
1612 		return (0);
1613 
1614 	if (link->lun > 0)
1615 		return (0);
1616 
1617 	rv = mpi_req_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_VOL,
1618 	    0, link->target, MPI_PG_POLL, &hdr);
1619 	if (rv != 0)
1620 		return (0);
1621 
1622 	len = hdr.page_length * 4;
1623 	rp0 = malloc(len, M_TEMP, M_NOWAIT);
1624 	if (rp0 == NULL)
1625 		return (ENOMEM);
1626 
1627 	rv = mpi_req_cfg_page(sc, link->target, MPI_PG_POLL, &hdr, 1, rp0, len);
1628 	if (rv == 0)
1629 		SET(link->flags, SDEV_VIRTUAL);
1630 
1631 	free(rp0, M_TEMP, len);
1632 	return (0);
1633 }
1634 
1635 int
1636 mpi_scsi_probe(struct scsi_link *link)
1637 {
1638 	struct mpi_softc		*sc = link->adapter_softc;
1639 	struct mpi_ecfg_hdr		ehdr;
1640 	struct mpi_cfg_sas_dev_pg0	pg0;
1641 	u_int32_t			address;
1642 	int				rv;
1643 
1644 	rv = mpi_scsi_probe_virtual(link);
1645 	if (rv != 0)
1646 		return (rv);
1647 
1648 	if (ISSET(link->flags, SDEV_VIRTUAL))
1649 		return (0);
1650 
1651 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SAS)
1652 		return (0);
1653 
1654 	address = MPI_CFG_SAS_DEV_ADDR_BUS | link->target;
1655 
1656 	if (mpi_ecfg_header(sc, MPI_CONFIG_REQ_EXTPAGE_TYPE_SAS_DEVICE, 0,
1657 	    address, &ehdr) != 0)
1658 		return (EIO);
1659 
1660 	if (mpi_ecfg_page(sc, address, &ehdr, 1, &pg0, sizeof(pg0)) != 0)
1661 		return (0);
1662 
1663 	DNPRINTF(MPI_D_MISC, "%s: mpi_scsi_probe sas dev pg 0 for target %d:\n",
1664 	    DEVNAME(sc), link->target);
1665 	DNPRINTF(MPI_D_MISC, "%s:  slot: 0x%04x enc_handle: 0x%04x\n",
1666 	    DEVNAME(sc), letoh16(pg0.slot), letoh16(pg0.enc_handle));
1667 	DNPRINTF(MPI_D_MISC, "%s:  sas_addr: 0x%016llx\n", DEVNAME(sc),
1668 	    letoh64(pg0.sas_addr));
1669 	DNPRINTF(MPI_D_MISC, "%s:  parent_dev_handle: 0x%04x phy_num: 0x%02x "
1670 	    "access_status: 0x%02x\n", DEVNAME(sc),
1671 	    letoh16(pg0.parent_dev_handle), pg0.phy_num, pg0.access_status);
1672 	DNPRINTF(MPI_D_MISC, "%s:  dev_handle: 0x%04x "
1673 	    "bus: 0x%02x target: 0x%02x\n", DEVNAME(sc),
1674 	    letoh16(pg0.dev_handle), pg0.bus, pg0.target);
1675 	DNPRINTF(MPI_D_MISC, "%s:  device_info: 0x%08x\n", DEVNAME(sc),
1676 	    letoh32(pg0.device_info));
1677 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%04x physical_port: 0x%02x\n",
1678 	    DEVNAME(sc), letoh16(pg0.flags), pg0.physical_port);
1679 
1680 	if (ISSET(lemtoh32(&pg0.device_info),
1681 	    MPI_CFG_SAS_DEV_0_DEVINFO_ATAPI_DEVICE)) {
1682 		DNPRINTF(MPI_D_MISC, "%s: target %d is an ATAPI device\n",
1683 		    DEVNAME(sc), link->target);
1684 		link->flags |= SDEV_ATAPI;
1685 		link->quirks |= SDEV_ONLYBIG;
1686 	}
1687 
1688 	return (0);
1689 }
1690 
1691 u_int32_t
1692 mpi_read(struct mpi_softc *sc, bus_size_t r)
1693 {
1694 	u_int32_t			rv;
1695 
1696 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1697 	    BUS_SPACE_BARRIER_READ);
1698 	rv = bus_space_read_4(sc->sc_iot, sc->sc_ioh, r);
1699 
1700 	DNPRINTF(MPI_D_RW, "%s: mpi_read %#x %#x\n", DEVNAME(sc), r, rv);
1701 
1702 	return (rv);
1703 }
1704 
1705 void
1706 mpi_write(struct mpi_softc *sc, bus_size_t r, u_int32_t v)
1707 {
1708 	DNPRINTF(MPI_D_RW, "%s: mpi_write %#x %#x\n", DEVNAME(sc), r, v);
1709 
1710 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
1711 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1712 	    BUS_SPACE_BARRIER_WRITE);
1713 }
1714 
1715 int
1716 mpi_wait_eq(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1717     u_int32_t target)
1718 {
1719 	int				i;
1720 
1721 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_eq %#x %#x %#x\n", DEVNAME(sc), r,
1722 	    mask, target);
1723 
1724 	for (i = 0; i < 10000; i++) {
1725 		if ((mpi_read(sc, r) & mask) == target)
1726 			return (0);
1727 		delay(1000);
1728 	}
1729 
1730 	return (1);
1731 }
1732 
1733 int
1734 mpi_wait_ne(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1735     u_int32_t target)
1736 {
1737 	int				i;
1738 
1739 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_ne %#x %#x %#x\n", DEVNAME(sc), r,
1740 	    mask, target);
1741 
1742 	for (i = 0; i < 10000; i++) {
1743 		if ((mpi_read(sc, r) & mask) != target)
1744 			return (0);
1745 		delay(1000);
1746 	}
1747 
1748 	return (1);
1749 }
1750 
1751 int
1752 mpi_init(struct mpi_softc *sc)
1753 {
1754 	u_int32_t			db;
1755 	int				i;
1756 
1757 	/* spin until the IOC leaves the RESET state */
1758 	if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1759 	    MPI_DOORBELL_STATE_RESET) != 0) {
1760 		DNPRINTF(MPI_D_MISC, "%s: mpi_init timeout waiting to leave "
1761 		    "reset state\n", DEVNAME(sc));
1762 		return (1);
1763 	}
1764 
1765 	/* check current ownership */
1766 	db = mpi_read_db(sc);
1767 	if ((db & MPI_DOORBELL_WHOINIT) == MPI_DOORBELL_WHOINIT_PCIPEER) {
1768 		DNPRINTF(MPI_D_MISC, "%s: mpi_init initialised by pci peer\n",
1769 		    DEVNAME(sc));
1770 		return (0);
1771 	}
1772 
1773 	for (i = 0; i < 5; i++) {
1774 		switch (db & MPI_DOORBELL_STATE) {
1775 		case MPI_DOORBELL_STATE_READY:
1776 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is ready\n",
1777 			    DEVNAME(sc));
1778 			return (0);
1779 
1780 		case MPI_DOORBELL_STATE_OPER:
1781 		case MPI_DOORBELL_STATE_FAULT:
1782 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is being "
1783 			    "reset\n" , DEVNAME(sc));
1784 			if (mpi_reset_soft(sc) != 0)
1785 				mpi_reset_hard(sc);
1786 			break;
1787 
1788 		case MPI_DOORBELL_STATE_RESET:
1789 			DNPRINTF(MPI_D_MISC, "%s: mpi_init waiting to come "
1790 			    "out of reset\n", DEVNAME(sc));
1791 			if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1792 			    MPI_DOORBELL_STATE_RESET) != 0)
1793 				return (1);
1794 			break;
1795 		}
1796 		db = mpi_read_db(sc);
1797 	}
1798 
1799 	return (1);
1800 }
1801 
1802 int
1803 mpi_reset_soft(struct mpi_softc *sc)
1804 {
1805 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_soft\n", DEVNAME(sc));
1806 
1807 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1808 		return (1);
1809 
1810 	mpi_write_db(sc,
1811 	    MPI_DOORBELL_FUNCTION(MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET));
1812 	if (mpi_wait_eq(sc, MPI_INTR_STATUS,
1813 	    MPI_INTR_STATUS_IOCDOORBELL, 0) != 0)
1814 		return (1);
1815 
1816 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1817 	    MPI_DOORBELL_STATE_READY) != 0)
1818 		return (1);
1819 
1820 	return (0);
1821 }
1822 
1823 int
1824 mpi_reset_hard(struct mpi_softc *sc)
1825 {
1826 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_hard\n", DEVNAME(sc));
1827 
1828 	/* enable diagnostic register */
1829 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1830 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_1);
1831 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_2);
1832 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_3);
1833 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_4);
1834 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_5);
1835 
1836 	/* reset ioc */
1837 	mpi_write(sc, MPI_HOSTDIAG, MPI_HOSTDIAG_RESET_ADAPTER);
1838 
1839 	delay(10000);
1840 
1841 	/* disable diagnostic register */
1842 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1843 
1844 	/* restore pci bits? */
1845 
1846 	/* firmware bits? */
1847 	return (0);
1848 }
1849 
1850 int
1851 mpi_handshake_send(struct mpi_softc *sc, void *buf, size_t dwords)
1852 {
1853 	u_int32_t				*query = buf;
1854 	int					i;
1855 
1856 	/* make sure the doorbell is not in use. */
1857 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1858 		return (1);
1859 
1860 	/* clear pending doorbell interrupts */
1861 	if (mpi_read_intr(sc) & MPI_INTR_STATUS_DOORBELL)
1862 		mpi_write_intr(sc, 0);
1863 
1864 	/*
1865 	 * first write the doorbell with the handshake function and the
1866 	 * dword count.
1867 	 */
1868 	mpi_write_db(sc, MPI_DOORBELL_FUNCTION(MPI_FUNCTION_HANDSHAKE) |
1869 	    MPI_DOORBELL_DWORDS(dwords));
1870 
1871 	/*
1872 	 * the doorbell used bit will be set because a doorbell function has
1873 	 * started. Wait for the interrupt and then ack it.
1874 	 */
1875 	if (mpi_wait_db_int(sc) != 0)
1876 		return (1);
1877 	mpi_write_intr(sc, 0);
1878 
1879 	/* poll for the acknowledgement. */
1880 	if (mpi_wait_db_ack(sc) != 0)
1881 		return (1);
1882 
1883 	/* write the query through the doorbell. */
1884 	for (i = 0; i < dwords; i++) {
1885 		mpi_write_db(sc, htole32(query[i]));
1886 		if (mpi_wait_db_ack(sc) != 0)
1887 			return (1);
1888 	}
1889 
1890 	return (0);
1891 }
1892 
1893 int
1894 mpi_handshake_recv_dword(struct mpi_softc *sc, u_int32_t *dword)
1895 {
1896 	u_int16_t				*words = (u_int16_t *)dword;
1897 	int					i;
1898 
1899 	for (i = 0; i < 2; i++) {
1900 		if (mpi_wait_db_int(sc) != 0)
1901 			return (1);
1902 		words[i] = letoh16(mpi_read_db(sc) & MPI_DOORBELL_DATA_MASK);
1903 		mpi_write_intr(sc, 0);
1904 	}
1905 
1906 	return (0);
1907 }
1908 
1909 int
1910 mpi_handshake_recv(struct mpi_softc *sc, void *buf, size_t dwords)
1911 {
1912 	struct mpi_msg_reply			*reply = buf;
1913 	u_int32_t				*dbuf = buf, dummy;
1914 	int					i;
1915 
1916 	/* get the first dword so we can read the length out of the header. */
1917 	if (mpi_handshake_recv_dword(sc, &dbuf[0]) != 0)
1918 		return (1);
1919 
1920 	DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dwords: %d reply: %d\n",
1921 	    DEVNAME(sc), dwords, reply->msg_length);
1922 
1923 	/*
1924 	 * the total length, in dwords, is in the message length field of the
1925 	 * reply header.
1926 	 */
1927 	for (i = 1; i < MIN(dwords, reply->msg_length); i++) {
1928 		if (mpi_handshake_recv_dword(sc, &dbuf[i]) != 0)
1929 			return (1);
1930 	}
1931 
1932 	/* if there's extra stuff to come off the ioc, discard it */
1933 	while (i++ < reply->msg_length) {
1934 		if (mpi_handshake_recv_dword(sc, &dummy) != 0)
1935 			return (1);
1936 		DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dummy read: "
1937 		    "0x%08x\n", DEVNAME(sc), dummy);
1938 	}
1939 
1940 	/* wait for the doorbell used bit to be reset and clear the intr */
1941 	if (mpi_wait_db_int(sc) != 0)
1942 		return (1);
1943 	mpi_write_intr(sc, 0);
1944 
1945 	return (0);
1946 }
1947 
1948 void
1949 mpi_empty_done(struct mpi_ccb *ccb)
1950 {
1951 	/* nothing to do */
1952 }
1953 
1954 int
1955 mpi_iocfacts(struct mpi_softc *sc)
1956 {
1957 	struct mpi_msg_iocfacts_request		ifq;
1958 	struct mpi_msg_iocfacts_reply		ifp;
1959 
1960 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts\n", DEVNAME(sc));
1961 
1962 	memset(&ifq, 0, sizeof(ifq));
1963 	memset(&ifp, 0, sizeof(ifp));
1964 
1965 	ifq.function = MPI_FUNCTION_IOC_FACTS;
1966 	ifq.chain_offset = 0;
1967 	ifq.msg_flags = 0;
1968 	ifq.msg_context = htole32(0xdeadbeef);
1969 
1970 	if (mpi_handshake_send(sc, &ifq, dwordsof(ifq)) != 0) {
1971 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts send failed\n",
1972 		    DEVNAME(sc));
1973 		return (1);
1974 	}
1975 
1976 	if (mpi_handshake_recv(sc, &ifp, dwordsof(ifp)) != 0) {
1977 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts recv failed\n",
1978 		    DEVNAME(sc));
1979 		return (1);
1980 	}
1981 
1982 	DNPRINTF(MPI_D_MISC, "%s:  func: 0x%02x len: %d msgver: %d.%d\n",
1983 	    DEVNAME(sc), ifp.function, ifp.msg_length,
1984 	    ifp.msg_version_maj, ifp.msg_version_min);
1985 	DNPRINTF(MPI_D_MISC, "%s:  msgflags: 0x%02x iocnumber: 0x%02x "
1986 	    "hdrver: %d.%d\n", DEVNAME(sc), ifp.msg_flags,
1987 	    ifp.ioc_number, ifp.header_version_maj,
1988 	    ifp.header_version_min);
1989 	DNPRINTF(MPI_D_MISC, "%s:  message context: 0x%08x\n", DEVNAME(sc),
1990 	    letoh32(ifp.msg_context));
1991 	DNPRINTF(MPI_D_MISC, "%s:  iocstatus: 0x%04x ioexcept: 0x%04x\n",
1992 	    DEVNAME(sc), letoh16(ifp.ioc_status),
1993 	    letoh16(ifp.ioc_exceptions));
1994 	DNPRINTF(MPI_D_MISC, "%s:  iocloginfo: 0x%08x\n", DEVNAME(sc),
1995 	    letoh32(ifp.ioc_loginfo));
1996 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%02x blocksize: %d whoinit: 0x%02x "
1997 	    "maxchdepth: %d\n", DEVNAME(sc), ifp.flags,
1998 	    ifp.block_size, ifp.whoinit, ifp.max_chain_depth);
1999 	DNPRINTF(MPI_D_MISC, "%s:  reqfrsize: %d replyqdepth: %d\n",
2000 	    DEVNAME(sc), letoh16(ifp.request_frame_size),
2001 	    letoh16(ifp.reply_queue_depth));
2002 	DNPRINTF(MPI_D_MISC, "%s:  productid: 0x%04x\n", DEVNAME(sc),
2003 	    letoh16(ifp.product_id));
2004 	DNPRINTF(MPI_D_MISC, "%s:  hostmfahiaddr: 0x%08x\n", DEVNAME(sc),
2005 	    letoh32(ifp.current_host_mfa_hi_addr));
2006 	DNPRINTF(MPI_D_MISC, "%s:  event_state: 0x%02x number_of_ports: %d "
2007 	    "global_credits: %d\n",
2008 	    DEVNAME(sc), ifp.event_state, ifp.number_of_ports,
2009 	    letoh16(ifp.global_credits));
2010 	DNPRINTF(MPI_D_MISC, "%s:  sensebufhiaddr: 0x%08x\n", DEVNAME(sc),
2011 	    letoh32(ifp.current_sense_buffer_hi_addr));
2012 	DNPRINTF(MPI_D_MISC, "%s:  maxbus: %d maxdev: %d replyfrsize: %d\n",
2013 	    DEVNAME(sc), ifp.max_buses, ifp.max_devices,
2014 	    letoh16(ifp.current_reply_frame_size));
2015 	DNPRINTF(MPI_D_MISC, "%s:  fw_image_size: %d\n", DEVNAME(sc),
2016 	    letoh32(ifp.fw_image_size));
2017 	DNPRINTF(MPI_D_MISC, "%s:  ioc_capabilities: 0x%08x\n", DEVNAME(sc),
2018 	    letoh32(ifp.ioc_capabilities));
2019 	DNPRINTF(MPI_D_MISC, "%s:  fw_version: %d.%d fw_version_unit: 0x%02x "
2020 	    "fw_version_dev: 0x%02x\n", DEVNAME(sc),
2021 	    ifp.fw_version_maj, ifp.fw_version_min,
2022 	    ifp.fw_version_unit, ifp.fw_version_dev);
2023 	DNPRINTF(MPI_D_MISC, "%s:  hi_priority_queue_depth: 0x%04x\n",
2024 	    DEVNAME(sc), letoh16(ifp.hi_priority_queue_depth));
2025 	DNPRINTF(MPI_D_MISC, "%s:  host_page_buffer_sge: hdr: 0x%08x "
2026 	    "addr 0x%08lx%08lx\n", DEVNAME(sc),
2027 	    letoh32(ifp.host_page_buffer_sge.sg_hdr),
2028 	    letoh32(ifp.host_page_buffer_sge.sg_addr_hi),
2029 	    letoh32(ifp.host_page_buffer_sge.sg_addr_lo));
2030 
2031 	sc->sc_fw_maj = ifp.fw_version_maj;
2032 	sc->sc_fw_min = ifp.fw_version_min;
2033 	sc->sc_fw_unit = ifp.fw_version_unit;
2034 	sc->sc_fw_dev = ifp.fw_version_dev;
2035 
2036 	sc->sc_maxcmds = lemtoh16(&ifp.global_credits);
2037 	sc->sc_maxchdepth = ifp.max_chain_depth;
2038 	sc->sc_ioc_number = ifp.ioc_number;
2039 	if (sc->sc_flags & MPI_F_SPI)
2040 		sc->sc_buswidth = 16;
2041 	else
2042 		sc->sc_buswidth =
2043 		    (ifp.max_devices == 0) ? 256 : ifp.max_devices;
2044 	if (ifp.flags & MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT)
2045 		sc->sc_fw_len = lemtoh32(&ifp.fw_image_size);
2046 
2047 	sc->sc_repq = MIN(MPI_REPLYQ_DEPTH, lemtoh16(&ifp.reply_queue_depth));
2048 
2049 	/*
2050 	 * you can fit sg elements on the end of the io cmd if they fit in the
2051 	 * request frame size.
2052 	 */
2053 	sc->sc_first_sgl_len = ((lemtoh16(&ifp.request_frame_size) * 4) -
2054 	    sizeof(struct mpi_msg_scsi_io)) / sizeof(struct mpi_sge);
2055 	DNPRINTF(MPI_D_MISC, "%s:   first sgl len: %d\n", DEVNAME(sc),
2056 	    sc->sc_first_sgl_len);
2057 
2058 	sc->sc_chain_len = (lemtoh16(&ifp.request_frame_size) * 4) /
2059 	    sizeof(struct mpi_sge);
2060 	DNPRINTF(MPI_D_MISC, "%s:   chain len: %d\n", DEVNAME(sc),
2061 	    sc->sc_chain_len);
2062 
2063 	/* the sgl tailing the io cmd loses an entry to the chain element. */
2064 	sc->sc_max_sgl_len = MPI_MAX_SGL - 1;
2065 	/* the sgl chains lose an entry for each chain element */
2066 	sc->sc_max_sgl_len -= (MPI_MAX_SGL - sc->sc_first_sgl_len) /
2067 	    sc->sc_chain_len;
2068 	DNPRINTF(MPI_D_MISC, "%s:   max sgl len: %d\n", DEVNAME(sc),
2069 	    sc->sc_max_sgl_len);
2070 
2071 	/* XXX we're ignoring the max chain depth */
2072 
2073 	return (0);
2074 }
2075 
2076 int
2077 mpi_iocinit(struct mpi_softc *sc)
2078 {
2079 	struct mpi_msg_iocinit_request		iiq;
2080 	struct mpi_msg_iocinit_reply		iip;
2081 	u_int32_t				hi_addr;
2082 
2083 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit\n", DEVNAME(sc));
2084 
2085 	memset(&iiq, 0, sizeof(iiq));
2086 	memset(&iip, 0, sizeof(iip));
2087 
2088 	iiq.function = MPI_FUNCTION_IOC_INIT;
2089 	iiq.whoinit = MPI_WHOINIT_HOST_DRIVER;
2090 
2091 	iiq.max_devices = (sc->sc_buswidth == 256) ? 0 : sc->sc_buswidth;
2092 	iiq.max_buses = 1;
2093 
2094 	iiq.msg_context = htole32(0xd00fd00f);
2095 
2096 	iiq.reply_frame_size = htole16(MPI_REPLY_SIZE);
2097 
2098 	hi_addr = (u_int32_t)(MPI_DMA_DVA(sc->sc_requests) >> 32);
2099 	htolem32(&iiq.host_mfa_hi_addr, hi_addr);
2100 	htolem32(&iiq.sense_buffer_hi_addr, hi_addr);
2101 
2102 	iiq.msg_version_maj = 0x01;
2103 	iiq.msg_version_min = 0x02;
2104 
2105 	iiq.hdr_version_unit = 0x0d;
2106 	iiq.hdr_version_dev = 0x00;
2107 
2108 	if (mpi_handshake_send(sc, &iiq, dwordsof(iiq)) != 0) {
2109 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit send failed\n",
2110 		    DEVNAME(sc));
2111 		return (1);
2112 	}
2113 
2114 	if (mpi_handshake_recv(sc, &iip, dwordsof(iip)) != 0) {
2115 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit recv failed\n",
2116 		    DEVNAME(sc));
2117 		return (1);
2118 	}
2119 
2120 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d "
2121 	    "whoinit: 0x%02x\n", DEVNAME(sc), iip.function,
2122 	    iip.msg_length, iip.whoinit);
2123 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x max_buses: %d "
2124 	    "max_devices: %d flags: 0x%02x\n", DEVNAME(sc), iip.msg_flags,
2125 	    iip.max_buses, iip.max_devices, iip.flags);
2126 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2127 	    letoh32(iip.msg_context));
2128 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2129 	    letoh16(iip.ioc_status));
2130 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2131 	    letoh32(iip.ioc_loginfo));
2132 
2133 	return (0);
2134 }
2135 
2136 int
2137 mpi_portfacts(struct mpi_softc *sc)
2138 {
2139 	struct mpi_ccb				*ccb;
2140 	struct mpi_msg_portfacts_request	*pfq;
2141 	volatile struct mpi_msg_portfacts_reply	*pfp;
2142 	int					rv = 1;
2143 
2144 	DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts\n", DEVNAME(sc));
2145 
2146 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
2147 	if (ccb == NULL) {
2148 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts ccb_get\n",
2149 		    DEVNAME(sc));
2150 		return (rv);
2151 	}
2152 
2153 	ccb->ccb_done = mpi_empty_done;
2154 	pfq = ccb->ccb_cmd;
2155 
2156 	pfq->function = MPI_FUNCTION_PORT_FACTS;
2157 	pfq->chain_offset = 0;
2158 	pfq->msg_flags = 0;
2159 	pfq->port_number = 0;
2160 
2161 	if (mpi_poll(sc, ccb, 50000) != 0) {
2162 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts poll\n", DEVNAME(sc));
2163 		goto err;
2164 	}
2165 
2166 	if (ccb->ccb_rcb == NULL) {
2167 		DNPRINTF(MPI_D_MISC, "%s: empty portfacts reply\n",
2168 		    DEVNAME(sc));
2169 		goto err;
2170 	}
2171 	pfp = ccb->ccb_rcb->rcb_reply;
2172 
2173 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d\n",
2174 	    DEVNAME(sc), pfp->function, pfp->msg_length);
2175 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x port_number: %d\n",
2176 	    DEVNAME(sc), pfp->msg_flags, pfp->port_number);
2177 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2178 	    letoh32(pfp->msg_context));
2179 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2180 	    letoh16(pfp->ioc_status));
2181 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2182 	    letoh32(pfp->ioc_loginfo));
2183 	DNPRINTF(MPI_D_MISC, "%s:  max_devices: %d port_type: 0x%02x\n",
2184 	    DEVNAME(sc), letoh16(pfp->max_devices), pfp->port_type);
2185 	DNPRINTF(MPI_D_MISC, "%s:  protocol_flags: 0x%04x port_scsi_id: %d\n",
2186 	    DEVNAME(sc), letoh16(pfp->protocol_flags),
2187 	    letoh16(pfp->port_scsi_id));
2188 	DNPRINTF(MPI_D_MISC, "%s:  max_persistent_ids: %d "
2189 	    "max_posted_cmd_buffers: %d\n", DEVNAME(sc),
2190 	    letoh16(pfp->max_persistent_ids),
2191 	    letoh16(pfp->max_posted_cmd_buffers));
2192 	DNPRINTF(MPI_D_MISC, "%s:  max_lan_buckets: %d\n", DEVNAME(sc),
2193 	    letoh16(pfp->max_lan_buckets));
2194 
2195 	sc->sc_porttype = pfp->port_type;
2196 	if (sc->sc_target == -1)
2197 		sc->sc_target = lemtoh16(&pfp->port_scsi_id);
2198 
2199 	mpi_push_reply(sc, ccb->ccb_rcb);
2200 	rv = 0;
2201 err:
2202 	scsi_io_put(&sc->sc_iopool, ccb);
2203 
2204 	return (rv);
2205 }
2206 
2207 int
2208 mpi_cfg_coalescing(struct mpi_softc *sc)
2209 {
2210 	struct mpi_cfg_hdr		hdr;
2211 	struct mpi_cfg_ioc_pg1		pg;
2212 	u_int32_t			flags;
2213 
2214 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 1, 0, &hdr) != 0) {
2215 		DNPRINTF(MPI_D_MISC, "%s: unable to fetch IOC page 1 header\n",
2216 		    DEVNAME(sc));
2217 		return (1);
2218 	}
2219 
2220 	if (mpi_cfg_page(sc, 0, &hdr, 1, &pg, sizeof(pg)) != 0) {
2221 		DNPRINTF(MPI_D_MISC, "%s: unable to fetch IOC page 1\n",
2222 		    DEVNAME(sc));
2223 		return (1);
2224 	}
2225 
2226 	DNPRINTF(MPI_D_MISC, "%s: IOC page 1\n", DEVNAME(sc));
2227 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%08x\n", DEVNAME(sc),
2228 	    letoh32(pg.flags));
2229 	DNPRINTF(MPI_D_MISC, "%s:  coalescing_timeout: %d\n", DEVNAME(sc),
2230 	    letoh32(pg.coalescing_timeout));
2231 	DNPRINTF(MPI_D_MISC, "%s:  coalescing_depth: %d pci_slot_num: %d\n",
2232 	    DEVNAME(sc), pg.coalescing_depth, pg.pci_slot_num);
2233 
2234 	flags = lemtoh32(&pg.flags);
2235 	if (!ISSET(flags, MPI_CFG_IOC_1_REPLY_COALESCING))
2236 		return (0);
2237 
2238 	CLR(pg.flags, htole32(MPI_CFG_IOC_1_REPLY_COALESCING));
2239 	if (mpi_cfg_page(sc, 0, &hdr, 0, &pg, sizeof(pg)) != 0) {
2240 		DNPRINTF(MPI_D_MISC, "%s: unable to clear coalescing\n",
2241 		    DEVNAME(sc));
2242 		return (1);
2243 	}
2244 
2245 	return (0);
2246 }
2247 
2248 int
2249 mpi_eventnotify(struct mpi_softc *sc)
2250 {
2251 	struct mpi_ccb				*ccb;
2252 	struct mpi_msg_event_request		*enq;
2253 
2254 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
2255 	if (ccb == NULL) {
2256 		DNPRINTF(MPI_D_MISC, "%s: mpi_eventnotify ccb_get\n",
2257 		    DEVNAME(sc));
2258 		return (1);
2259 	}
2260 
2261 	sc->sc_evt_ccb = ccb;
2262 	SIMPLEQ_INIT(&sc->sc_evt_ack_queue);
2263 	mtx_init(&sc->sc_evt_ack_mtx, IPL_BIO);
2264 	scsi_ioh_set(&sc->sc_evt_ack_handler, &sc->sc_iopool,
2265 	    mpi_eventack, sc);
2266 
2267 	ccb->ccb_done = mpi_eventnotify_done;
2268 	enq = ccb->ccb_cmd;
2269 
2270 	enq->function = MPI_FUNCTION_EVENT_NOTIFICATION;
2271 	enq->chain_offset = 0;
2272 	enq->event_switch = MPI_EVENT_SWITCH_ON;
2273 
2274 	mpi_start(sc, ccb);
2275 	return (0);
2276 }
2277 
2278 void
2279 mpi_eventnotify_done(struct mpi_ccb *ccb)
2280 {
2281 	struct mpi_softc			*sc = ccb->ccb_sc;
2282 	struct mpi_rcb				*rcb = ccb->ccb_rcb;
2283 	struct mpi_msg_event_reply		*enp = rcb->rcb_reply;
2284 
2285 	DNPRINTF(MPI_D_EVT, "%s: mpi_eventnotify_done\n", DEVNAME(sc));
2286 
2287 	DNPRINTF(MPI_D_EVT, "%s:  function: 0x%02x msg_length: %d "
2288 	    "data_length: %d\n", DEVNAME(sc), enp->function, enp->msg_length,
2289 	    letoh16(enp->data_length));
2290 	DNPRINTF(MPI_D_EVT, "%s:  ack_required: %d msg_flags 0x%02x\n",
2291 	    DEVNAME(sc), enp->ack_required, enp->msg_flags);
2292 	DNPRINTF(MPI_D_EVT, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2293 	    letoh32(enp->msg_context));
2294 	DNPRINTF(MPI_D_EVT, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2295 	    letoh16(enp->ioc_status));
2296 	DNPRINTF(MPI_D_EVT, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2297 	    letoh32(enp->ioc_loginfo));
2298 	DNPRINTF(MPI_D_EVT, "%s:  event: 0x%08x\n", DEVNAME(sc),
2299 	    letoh32(enp->event));
2300 	DNPRINTF(MPI_D_EVT, "%s:  event_context: 0x%08x\n", DEVNAME(sc),
2301 	    letoh32(enp->event_context));
2302 
2303 	switch (lemtoh32(&enp->event)) {
2304 	/* ignore these */
2305 	case MPI_EVENT_EVENT_CHANGE:
2306 	case MPI_EVENT_SAS_PHY_LINK_STATUS:
2307 		break;
2308 
2309 	case MPI_EVENT_SAS_DEVICE_STATUS_CHANGE:
2310 		if (sc->sc_scsibus == NULL)
2311 			break;
2312 
2313 		if (mpi_evt_sas(sc, rcb) != 0) {
2314 			/* reply is freed later on */
2315 			return;
2316 		}
2317 		break;
2318 
2319 	case MPI_EVENT_RESCAN:
2320 		if (sc->sc_scsibus != NULL &&
2321 		    sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_FC)
2322 			task_add(systq, &sc->sc_evt_rescan);
2323 		break;
2324 
2325 	default:
2326 		DNPRINTF(MPI_D_EVT, "%s:  unhandled event 0x%02x\n",
2327 		    DEVNAME(sc), lemtoh32(&enp->event));
2328 		break;
2329 	}
2330 
2331 	mpi_eventnotify_free(sc, rcb);
2332 }
2333 
2334 void
2335 mpi_eventnotify_free(struct mpi_softc *sc, struct mpi_rcb *rcb)
2336 {
2337 	struct mpi_msg_event_reply		*enp = rcb->rcb_reply;
2338 
2339 	if (enp->ack_required) {
2340 		mtx_enter(&sc->sc_evt_ack_mtx);
2341 		SIMPLEQ_INSERT_TAIL(&sc->sc_evt_ack_queue, rcb, rcb_link);
2342 		mtx_leave(&sc->sc_evt_ack_mtx);
2343 		scsi_ioh_add(&sc->sc_evt_ack_handler);
2344 	} else
2345 		mpi_push_reply(sc, rcb);
2346 }
2347 
2348 int
2349 mpi_evt_sas(struct mpi_softc *sc, struct mpi_rcb *rcb)
2350 {
2351 	struct mpi_evt_sas_change		*ch;
2352 	u_int8_t				*data;
2353 
2354 	data = rcb->rcb_reply;
2355 	data += sizeof(struct mpi_msg_event_reply);
2356 	ch = (struct mpi_evt_sas_change *)data;
2357 
2358 	if (ch->bus != 0)
2359 		return (0);
2360 
2361 	switch (ch->reason) {
2362 	case MPI_EVT_SASCH_REASON_ADDED:
2363 	case MPI_EVT_SASCH_REASON_NO_PERSIST_ADDED:
2364 		KERNEL_LOCK();
2365 		if (scsi_req_probe(sc->sc_scsibus, ch->target, -1) != 0) {
2366 			printf("%s: unable to request attach of %d\n",
2367 			    DEVNAME(sc), ch->target);
2368 		}
2369 		KERNEL_UNLOCK();
2370 		break;
2371 
2372 	case MPI_EVT_SASCH_REASON_NOT_RESPONDING:
2373 		KERNEL_LOCK();
2374 		scsi_activate(sc->sc_scsibus, ch->target, -1, DVACT_DEACTIVATE);
2375 		KERNEL_UNLOCK();
2376 
2377 		mtx_enter(&sc->sc_evt_scan_mtx);
2378 		SIMPLEQ_INSERT_TAIL(&sc->sc_evt_scan_queue, rcb, rcb_link);
2379 		mtx_leave(&sc->sc_evt_scan_mtx);
2380 		scsi_ioh_add(&sc->sc_evt_scan_handler);
2381 
2382 		/* we'll handle event ack later on */
2383 		return (1);
2384 
2385 	case MPI_EVT_SASCH_REASON_SMART_DATA:
2386 	case MPI_EVT_SASCH_REASON_UNSUPPORTED:
2387 	case MPI_EVT_SASCH_REASON_INTERNAL_RESET:
2388 		break;
2389 	default:
2390 		printf("%s: unknown reason for SAS device status change: "
2391 		    "0x%02x\n", DEVNAME(sc), ch->reason);
2392 		break;
2393 	}
2394 
2395 	return (0);
2396 }
2397 
2398 void
2399 mpi_evt_sas_detach(void *cookie, void *io)
2400 {
2401 	struct mpi_softc			*sc = cookie;
2402 	struct mpi_ccb				*ccb = io;
2403 	struct mpi_rcb				*rcb, *next;
2404 	struct mpi_msg_event_reply		*enp;
2405 	struct mpi_evt_sas_change		*ch;
2406 	struct mpi_msg_scsi_task_request	*str;
2407 
2408 	DNPRINTF(MPI_D_EVT, "%s: event sas detach handler\n", DEVNAME(sc));
2409 
2410 	mtx_enter(&sc->sc_evt_scan_mtx);
2411 	rcb = SIMPLEQ_FIRST(&sc->sc_evt_scan_queue);
2412 	if (rcb != NULL) {
2413 		next = SIMPLEQ_NEXT(rcb, rcb_link);
2414 		SIMPLEQ_REMOVE_HEAD(&sc->sc_evt_scan_queue, rcb_link);
2415 	}
2416 	mtx_leave(&sc->sc_evt_scan_mtx);
2417 
2418 	if (rcb == NULL) {
2419 		scsi_io_put(&sc->sc_iopool, ccb);
2420 		return;
2421 	}
2422 
2423 	enp = rcb->rcb_reply;
2424 	ch = (struct mpi_evt_sas_change *)(enp + 1);
2425 
2426 	ccb->ccb_done = mpi_evt_sas_detach_done;
2427 	str = ccb->ccb_cmd;
2428 
2429 	str->target_id = ch->target;
2430 	str->bus = 0;
2431 	str->function = MPI_FUNCTION_SCSI_TASK_MGMT;
2432 
2433 	str->task_type = MPI_MSG_SCSI_TASK_TYPE_TARGET_RESET;
2434 
2435 	mpi_eventnotify_free(sc, rcb);
2436 
2437 	mpi_start(sc, ccb);
2438 
2439 	if (next != NULL)
2440 		scsi_ioh_add(&sc->sc_evt_scan_handler);
2441 }
2442 
2443 void
2444 mpi_evt_sas_detach_done(struct mpi_ccb *ccb)
2445 {
2446 	struct mpi_softc			*sc = ccb->ccb_sc;
2447 	struct mpi_msg_scsi_task_reply		*r = ccb->ccb_rcb->rcb_reply;
2448 
2449 	KERNEL_LOCK();
2450 	if (scsi_req_detach(sc->sc_scsibus, r->target_id, -1,
2451 	    DETACH_FORCE) != 0) {
2452 		printf("%s: unable to request detach of %d\n",
2453 		    DEVNAME(sc), r->target_id);
2454 	}
2455 	KERNEL_UNLOCK();
2456 
2457 	mpi_push_reply(sc, ccb->ccb_rcb);
2458 	scsi_io_put(&sc->sc_iopool, ccb);
2459 }
2460 
2461 void
2462 mpi_fc_rescan(void *xsc)
2463 {
2464 	struct mpi_softc			*sc = xsc;
2465 	struct mpi_cfg_hdr			hdr;
2466 	struct mpi_cfg_fc_device_pg0		pg;
2467 	struct scsi_link			*link;
2468 	u_int8_t				devmap[256 / NBBY];
2469 	u_int32_t				id = 0xffffff;
2470 	int					i;
2471 
2472 	memset(devmap, 0, sizeof(devmap));
2473 
2474 	do {
2475 		if (mpi_req_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_FC_DEV, 0,
2476 		    id, 0, &hdr) != 0) {
2477 			printf("%s: header get for rescan of 0x%08x failed\n",
2478 			    DEVNAME(sc), id);
2479 			return;
2480 		}
2481 
2482 		memset(&pg, 0, sizeof(pg));
2483 		if (mpi_req_cfg_page(sc, id, 0, &hdr, 1, &pg, sizeof(pg)) != 0)
2484 			break;
2485 
2486 		if (ISSET(pg.flags, MPI_CFG_FC_DEV_0_FLAGS_BUSADDR_VALID) &&
2487 		    pg.current_bus == 0)
2488 			setbit(devmap, pg.current_target_id);
2489 
2490 		id = lemtoh32(&pg.port_id);
2491 	} while (id <= 0xff0000);
2492 
2493 	for (i = 0; i < sc->sc_buswidth; i++) {
2494 		link = scsi_get_link(sc->sc_scsibus, i, 0);
2495 
2496 		if (isset(devmap, i)) {
2497 			if (link == NULL)
2498 				scsi_probe_target(sc->sc_scsibus, i);
2499 		} else {
2500 			if (link != NULL) {
2501 				scsi_activate(sc->sc_scsibus, i, -1,
2502 				    DVACT_DEACTIVATE);
2503 				scsi_detach_target(sc->sc_scsibus, i,
2504 				    DETACH_FORCE);
2505 			}
2506 		}
2507 	}
2508 }
2509 
2510 void
2511 mpi_eventack(void *cookie, void *io)
2512 {
2513 	struct mpi_softc			*sc = cookie;
2514 	struct mpi_ccb				*ccb = io;
2515 	struct mpi_rcb				*rcb, *next;
2516 	struct mpi_msg_event_reply		*enp;
2517 	struct mpi_msg_eventack_request		*eaq;
2518 
2519 	DNPRINTF(MPI_D_EVT, "%s: event ack\n", DEVNAME(sc));
2520 
2521 	mtx_enter(&sc->sc_evt_ack_mtx);
2522 	rcb = SIMPLEQ_FIRST(&sc->sc_evt_ack_queue);
2523 	if (rcb != NULL) {
2524 		next = SIMPLEQ_NEXT(rcb, rcb_link);
2525 		SIMPLEQ_REMOVE_HEAD(&sc->sc_evt_ack_queue, rcb_link);
2526 	}
2527 	mtx_leave(&sc->sc_evt_ack_mtx);
2528 
2529 	if (rcb == NULL) {
2530 		scsi_io_put(&sc->sc_iopool, ccb);
2531 		return;
2532 	}
2533 
2534 	enp = rcb->rcb_reply;
2535 
2536 	ccb->ccb_done = mpi_eventack_done;
2537 	eaq = ccb->ccb_cmd;
2538 
2539 	eaq->function = MPI_FUNCTION_EVENT_ACK;
2540 
2541 	eaq->event = enp->event;
2542 	eaq->event_context = enp->event_context;
2543 
2544 	mpi_push_reply(sc, rcb);
2545 	mpi_start(sc, ccb);
2546 
2547 	if (next != NULL)
2548 		scsi_ioh_add(&sc->sc_evt_ack_handler);
2549 }
2550 
2551 void
2552 mpi_eventack_done(struct mpi_ccb *ccb)
2553 {
2554 	struct mpi_softc			*sc = ccb->ccb_sc;
2555 
2556 	DNPRINTF(MPI_D_EVT, "%s: event ack done\n", DEVNAME(sc));
2557 
2558 	mpi_push_reply(sc, ccb->ccb_rcb);
2559 	scsi_io_put(&sc->sc_iopool, ccb);
2560 }
2561 
2562 int
2563 mpi_portenable(struct mpi_softc *sc)
2564 {
2565 	struct mpi_ccb				*ccb;
2566 	struct mpi_msg_portenable_request	*peq;
2567 	int					rv = 0;
2568 
2569 	DNPRINTF(MPI_D_MISC, "%s: mpi_portenable\n", DEVNAME(sc));
2570 
2571 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
2572 	if (ccb == NULL) {
2573 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable ccb_get\n",
2574 		    DEVNAME(sc));
2575 		return (1);
2576 	}
2577 
2578 	ccb->ccb_done = mpi_empty_done;
2579 	peq = ccb->ccb_cmd;
2580 
2581 	peq->function = MPI_FUNCTION_PORT_ENABLE;
2582 	peq->port_number = 0;
2583 
2584 	if (mpi_poll(sc, ccb, 50000) != 0) {
2585 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable poll\n", DEVNAME(sc));
2586 		return (1);
2587 	}
2588 
2589 	if (ccb->ccb_rcb == NULL) {
2590 		DNPRINTF(MPI_D_MISC, "%s: empty portenable reply\n",
2591 		    DEVNAME(sc));
2592 		rv = 1;
2593 	} else
2594 		mpi_push_reply(sc, ccb->ccb_rcb);
2595 
2596 	scsi_io_put(&sc->sc_iopool, ccb);
2597 
2598 	return (rv);
2599 }
2600 
2601 int
2602 mpi_fwupload(struct mpi_softc *sc)
2603 {
2604 	struct mpi_ccb				*ccb;
2605 	struct {
2606 		struct mpi_msg_fwupload_request		req;
2607 		struct mpi_sge				sge;
2608 	} __packed				*bundle;
2609 	struct mpi_msg_fwupload_reply		*upp;
2610 	int					rv = 0;
2611 
2612 	if (sc->sc_fw_len == 0)
2613 		return (0);
2614 
2615 	DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload\n", DEVNAME(sc));
2616 
2617 	sc->sc_fw = mpi_dmamem_alloc(sc, sc->sc_fw_len);
2618 	if (sc->sc_fw == NULL) {
2619 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload unable to allocate %d\n",
2620 		    DEVNAME(sc), sc->sc_fw_len);
2621 		return (1);
2622 	}
2623 
2624 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
2625 	if (ccb == NULL) {
2626 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload ccb_get\n",
2627 		    DEVNAME(sc));
2628 		goto err;
2629 	}
2630 
2631 	ccb->ccb_done = mpi_empty_done;
2632 	bundle = ccb->ccb_cmd;
2633 
2634 	bundle->req.function = MPI_FUNCTION_FW_UPLOAD;
2635 
2636 	bundle->req.image_type = MPI_FWUPLOAD_IMAGETYPE_IOC_FW;
2637 
2638 	bundle->req.tce.details_length = 12;
2639 	htolem32(&bundle->req.tce.image_size, sc->sc_fw_len);
2640 
2641 	htolem32(&bundle->sge.sg_hdr, MPI_SGE_FL_TYPE_SIMPLE |
2642 	    MPI_SGE_FL_SIZE_64 | MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
2643 	    MPI_SGE_FL_EOL | (u_int32_t)sc->sc_fw_len);
2644 	mpi_dvatosge(&bundle->sge, MPI_DMA_DVA(sc->sc_fw));
2645 
2646 	if (mpi_poll(sc, ccb, 50000) != 0) {
2647 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2648 		goto err;
2649 	}
2650 
2651 	if (ccb->ccb_rcb == NULL)
2652 		panic("%s: unable to do fw upload", DEVNAME(sc));
2653 	upp = ccb->ccb_rcb->rcb_reply;
2654 
2655 	if (lemtoh16(&upp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2656 		rv = 1;
2657 
2658 	mpi_push_reply(sc, ccb->ccb_rcb);
2659 	scsi_io_put(&sc->sc_iopool, ccb);
2660 
2661 	return (rv);
2662 
2663 err:
2664 	mpi_dmamem_free(sc, sc->sc_fw);
2665 	return (1);
2666 }
2667 
2668 int
2669 mpi_manufacturing(struct mpi_softc *sc)
2670 {
2671 	char board_name[33];
2672 	struct mpi_cfg_hdr hdr;
2673 	struct mpi_cfg_manufacturing_pg0 *pg;
2674 	size_t pagelen;
2675 	int rv = 1;
2676 
2677 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_MANUFACTURING,
2678 	    0, 0, &hdr) != 0)
2679 		return (1);
2680 
2681 	pagelen = hdr.page_length * 4; /* dwords to bytes */
2682 	if (pagelen < sizeof(*pg))
2683 		return (1);
2684 
2685 	pg = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
2686 	if (pg == NULL)
2687 		return (1);
2688 
2689 	if (mpi_cfg_page(sc, 0, &hdr, 1, pg, pagelen) != 0)
2690 		goto out;
2691 
2692 	scsi_strvis(board_name, pg->board_name, sizeof(pg->board_name));
2693 
2694 	printf("%s: %s, firmware %d.%d.%d.%d\n", DEVNAME(sc), board_name,
2695 	    sc->sc_fw_maj, sc->sc_fw_min, sc->sc_fw_unit, sc->sc_fw_dev);
2696 
2697 	rv = 0;
2698 
2699 out:
2700 	free(pg, M_TEMP, pagelen);
2701 	return (rv);
2702 }
2703 
2704 void
2705 mpi_get_raid(struct mpi_softc *sc)
2706 {
2707 	struct mpi_cfg_hdr		hdr;
2708 	struct mpi_cfg_ioc_pg2		*vol_page;
2709 	size_t				pagelen;
2710 	u_int32_t			capabilities;
2711 
2712 	DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid\n", DEVNAME(sc));
2713 
2714 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 2, 0, &hdr) != 0) {
2715 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch header"
2716 		    "for IOC page 2\n", DEVNAME(sc));
2717 		return;
2718 	}
2719 
2720 	pagelen = hdr.page_length * 4; /* dwords to bytes */
2721 	vol_page = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
2722 	if (vol_page == NULL) {
2723 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to allocate "
2724 		    "space for ioc config page 2\n", DEVNAME(sc));
2725 		return;
2726 	}
2727 
2728 	if (mpi_cfg_page(sc, 0, &hdr, 1, vol_page, pagelen) != 0) {
2729 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch IOC "
2730 		    "page 2\n", DEVNAME(sc));
2731 		goto out;
2732 	}
2733 
2734 	capabilities = lemtoh32(&vol_page->capabilities);
2735 
2736 	DNPRINTF(MPI_D_RAID, "%s:  capabilities: 0x08%x\n", DEVNAME(sc),
2737 	    letoh32(vol_page->capabilities));
2738 	DNPRINTF(MPI_D_RAID, "%s:  active_vols: %d max_vols: %d "
2739 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
2740 	    vol_page->active_vols, vol_page->max_vols,
2741 	    vol_page->active_physdisks, vol_page->max_physdisks);
2742 
2743 	/* don't walk list if there are no RAID capability */
2744 	if (capabilities == 0xdeadbeef) {
2745 		printf("%s: deadbeef in raid configuration\n", DEVNAME(sc));
2746 		goto out;
2747 	}
2748 
2749 	if (ISSET(capabilities, MPI_CFG_IOC_2_CAPABILITIES_RAID))
2750 		sc->sc_flags |= MPI_F_RAID;
2751 
2752 out:
2753 	free(vol_page, M_TEMP, pagelen);
2754 }
2755 
2756 int
2757 mpi_req_cfg_header(struct mpi_softc *sc, u_int8_t type, u_int8_t number,
2758     u_int32_t address, int flags, void *p)
2759 {
2760 	struct mpi_ccb				*ccb;
2761 	struct mpi_msg_config_request		*cq;
2762 	struct mpi_msg_config_reply		*cp;
2763 	struct mpi_cfg_hdr			*hdr = p;
2764 	struct mpi_ecfg_hdr			*ehdr = p;
2765 	int					etype = 0;
2766 	int					rv = 0;
2767 
2768 	DNPRINTF(MPI_D_MISC, "%s: mpi_req_cfg_header type: %#x number: %x "
2769 	    "address: 0x%08x flags: 0x%b\n", DEVNAME(sc), type, number,
2770 	    address, flags, MPI_PG_FMT);
2771 
2772 	ccb = scsi_io_get(&sc->sc_iopool,
2773 	    ISSET(flags, MPI_PG_POLL) ? SCSI_NOSLEEP : 0);
2774 	if (ccb == NULL) {
2775 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header ccb_get\n",
2776 		    DEVNAME(sc));
2777 		return (1);
2778 	}
2779 
2780 	if (ISSET(flags, MPI_PG_EXTENDED)) {
2781 		etype = type;
2782 		type = MPI_CONFIG_REQ_PAGE_TYPE_EXTENDED;
2783 	}
2784 
2785 	cq = ccb->ccb_cmd;
2786 
2787 	cq->function = MPI_FUNCTION_CONFIG;
2788 
2789 	cq->action = MPI_CONFIG_REQ_ACTION_PAGE_HEADER;
2790 
2791 	cq->config_header.page_number = number;
2792 	cq->config_header.page_type = type;
2793 	cq->ext_page_type = etype;
2794 	htolem32(&cq->page_address, address);
2795 	htolem32(&cq->page_buffer.sg_hdr, MPI_SGE_FL_TYPE_SIMPLE |
2796 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
2797 
2798 	ccb->ccb_done = mpi_empty_done;
2799 	if (ISSET(flags, MPI_PG_POLL)) {
2800 		if (mpi_poll(sc, ccb, 50000) != 0) {
2801 			DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n",
2802 			    DEVNAME(sc));
2803 			return (1);
2804 		}
2805 	} else
2806 		mpi_wait(sc, ccb);
2807 
2808 	if (ccb->ccb_rcb == NULL)
2809 		panic("%s: unable to fetch config header", DEVNAME(sc));
2810 	cp = ccb->ccb_rcb->rcb_reply;
2811 
2812 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2813 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2814 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2815 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2816 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2817 	    cp->msg_flags);
2818 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2819 	    letoh32(cp->msg_context));
2820 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2821 	    letoh16(cp->ioc_status));
2822 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2823 	    letoh32(cp->ioc_loginfo));
2824 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2825 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2826 	    cp->config_header.page_version,
2827 	    cp->config_header.page_length,
2828 	    cp->config_header.page_number,
2829 	    cp->config_header.page_type);
2830 
2831 	if (lemtoh16(&cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2832 		rv = 1;
2833 	else if (ISSET(flags, MPI_PG_EXTENDED)) {
2834 		memset(ehdr, 0, sizeof(*ehdr));
2835 		ehdr->page_version = cp->config_header.page_version;
2836 		ehdr->page_number = cp->config_header.page_number;
2837 		ehdr->page_type = cp->config_header.page_type;
2838 		ehdr->ext_page_length = cp->ext_page_length;
2839 		ehdr->ext_page_type = cp->ext_page_type;
2840 	} else
2841 		*hdr = cp->config_header;
2842 
2843 	mpi_push_reply(sc, ccb->ccb_rcb);
2844 	scsi_io_put(&sc->sc_iopool, ccb);
2845 
2846 	return (rv);
2847 }
2848 
2849 int
2850 mpi_req_cfg_page(struct mpi_softc *sc, u_int32_t address, int flags,
2851     void *p, int read, void *page, size_t len)
2852 {
2853 	struct mpi_ccb				*ccb;
2854 	struct mpi_msg_config_request		*cq;
2855 	struct mpi_msg_config_reply		*cp;
2856 	struct mpi_cfg_hdr			*hdr = p;
2857 	struct mpi_ecfg_hdr			*ehdr = p;
2858 	char					*kva;
2859 	int					page_length;
2860 	int					rv = 0;
2861 
2862 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page address: %d read: %d type: %x\n",
2863 	    DEVNAME(sc), address, read, hdr->page_type);
2864 
2865 	page_length = ISSET(flags, MPI_PG_EXTENDED) ?
2866 	    lemtoh16(&ehdr->ext_page_length) : hdr->page_length;
2867 
2868 	if (len > MPI_REQUEST_SIZE - sizeof(struct mpi_msg_config_request) ||
2869 	    len < page_length * 4)
2870 		return (1);
2871 
2872 	ccb = scsi_io_get(&sc->sc_iopool,
2873 	    ISSET(flags, MPI_PG_POLL) ? SCSI_NOSLEEP : 0);
2874 	if (ccb == NULL) {
2875 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page ccb_get\n", DEVNAME(sc));
2876 		return (1);
2877 	}
2878 
2879 	cq = ccb->ccb_cmd;
2880 
2881 	cq->function = MPI_FUNCTION_CONFIG;
2882 
2883 	cq->action = (read ? MPI_CONFIG_REQ_ACTION_PAGE_READ_CURRENT :
2884 	    MPI_CONFIG_REQ_ACTION_PAGE_WRITE_CURRENT);
2885 
2886 	if (ISSET(flags, MPI_PG_EXTENDED)) {
2887 		cq->config_header.page_version = ehdr->page_version;
2888 		cq->config_header.page_number = ehdr->page_number;
2889 		cq->config_header.page_type = ehdr->page_type;
2890 		cq->ext_page_len = ehdr->ext_page_length;
2891 		cq->ext_page_type = ehdr->ext_page_type;
2892 	} else
2893 		cq->config_header = *hdr;
2894 	cq->config_header.page_type &= MPI_CONFIG_REQ_PAGE_TYPE_MASK;
2895 	htolem32(&cq->page_address, address);
2896 	htolem32(&cq->page_buffer.sg_hdr, MPI_SGE_FL_TYPE_SIMPLE |
2897 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
2898 	    (page_length * 4) |
2899 	    (read ? MPI_SGE_FL_DIR_IN : MPI_SGE_FL_DIR_OUT));
2900 
2901 	/* bounce the page via the request space to avoid more bus_dma games */
2902 	mpi_dvatosge(&cq->page_buffer, ccb->ccb_cmd_dva +
2903 	    sizeof(struct mpi_msg_config_request));
2904 
2905 	kva = ccb->ccb_cmd;
2906 	kva += sizeof(struct mpi_msg_config_request);
2907 	if (!read)
2908 		memcpy(kva, page, len);
2909 
2910 	ccb->ccb_done = mpi_empty_done;
2911 	if (ISSET(flags, MPI_PG_POLL)) {
2912 		if (mpi_poll(sc, ccb, 50000) != 0) {
2913 			DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n",
2914 			    DEVNAME(sc));
2915 			return (1);
2916 		}
2917 	} else
2918 		mpi_wait(sc, ccb);
2919 
2920 	if (ccb->ccb_rcb == NULL) {
2921 		scsi_io_put(&sc->sc_iopool, ccb);
2922 		return (1);
2923 	}
2924 	cp = ccb->ccb_rcb->rcb_reply;
2925 
2926 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2927 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2928 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2929 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2930 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2931 	    cp->msg_flags);
2932 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2933 	    letoh32(cp->msg_context));
2934 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2935 	    letoh16(cp->ioc_status));
2936 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2937 	    letoh32(cp->ioc_loginfo));
2938 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2939 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2940 	    cp->config_header.page_version,
2941 	    cp->config_header.page_length,
2942 	    cp->config_header.page_number,
2943 	    cp->config_header.page_type);
2944 
2945 	if (lemtoh16(&cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2946 		rv = 1;
2947 	else if (read)
2948 		memcpy(page, kva, len);
2949 
2950 	mpi_push_reply(sc, ccb->ccb_rcb);
2951 	scsi_io_put(&sc->sc_iopool, ccb);
2952 
2953 	return (rv);
2954 }
2955 
2956 int
2957 mpi_scsi_ioctl(struct scsi_link *link, u_long cmd, caddr_t addr, int flag)
2958 {
2959 	struct mpi_softc	*sc = (struct mpi_softc *)link->adapter_softc;
2960 
2961 	DNPRINTF(MPI_D_IOCTL, "%s: mpi_scsi_ioctl\n", DEVNAME(sc));
2962 
2963 	switch (cmd) {
2964 	case DIOCGCACHE:
2965 	case DIOCSCACHE:
2966 		if (ISSET(link->flags, SDEV_VIRTUAL)) {
2967 			return (mpi_ioctl_cache(link, cmd,
2968 			    (struct dk_cache *)addr));
2969 		}
2970 		break;
2971 
2972 	default:
2973 		if (sc->sc_ioctl)
2974 			return (sc->sc_ioctl(link->adapter_softc, cmd, addr));
2975 
2976 		break;
2977 	}
2978 
2979 	return (ENOTTY);
2980 }
2981 
2982 int
2983 mpi_ioctl_cache(struct scsi_link *link, u_long cmd, struct dk_cache *dc)
2984 {
2985 	struct mpi_softc	*sc = (struct mpi_softc *)link->adapter_softc;
2986 	struct mpi_ccb		*ccb;
2987 	int			len, rv;
2988 	struct mpi_cfg_hdr	hdr;
2989 	struct mpi_cfg_raid_vol_pg0 *rpg0;
2990 	int			enabled;
2991 	struct mpi_msg_raid_action_request *req;
2992 	struct mpi_msg_raid_action_reply *rep;
2993 	struct mpi_raid_settings settings;
2994 
2995 	rv = mpi_req_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_VOL, 0,
2996 	    link->target, MPI_PG_POLL, &hdr);
2997 	if (rv != 0)
2998 		return (EIO);
2999 
3000 	len = sizeof(*rpg0) + sc->sc_vol_page->max_physdisks *
3001 	    sizeof(struct mpi_cfg_raid_vol_pg0_physdisk);
3002 	rpg0 = malloc(len, M_TEMP, M_NOWAIT);
3003 	if (rpg0 == NULL)
3004 		return (ENOMEM);
3005 
3006 	if (mpi_req_cfg_page(sc, link->target, MPI_PG_POLL, &hdr, 1,
3007 	    rpg0, len) != 0) {
3008 		DNPRINTF(MPI_D_RAID, "%s: can't get RAID vol cfg page 0\n",
3009 		    DEVNAME(sc));
3010 		rv = EIO;
3011 		goto done;
3012 	}
3013 
3014 	enabled = ISSET(lemtoh16(&rpg0->settings.volume_settings),
3015 	    MPI_CFG_RAID_VOL_0_SETTINGS_WRITE_CACHE_EN) ? 1 : 0;
3016 
3017 	if (cmd == DIOCGCACHE) {
3018 		dc->wrcache = enabled;
3019 		dc->rdcache = 0;
3020 		goto done;
3021 	} /* else DIOCSCACHE */
3022 
3023 	if (dc->rdcache) {
3024 		rv = EOPNOTSUPP;
3025 		goto done;
3026 	}
3027 
3028 	if (((dc->wrcache) ? 1 : 0) == enabled)
3029 		goto done;
3030 
3031 	settings = rpg0->settings;
3032 	if (dc->wrcache) {
3033 		SET(settings.volume_settings,
3034 		    htole16(MPI_CFG_RAID_VOL_0_SETTINGS_WRITE_CACHE_EN));
3035 	} else {
3036 		CLR(settings.volume_settings,
3037 		    htole16(MPI_CFG_RAID_VOL_0_SETTINGS_WRITE_CACHE_EN));
3038 	}
3039 
3040 	ccb = scsi_io_get(&sc->sc_iopool, SCSI_NOSLEEP);
3041 	if (ccb == NULL) {
3042 		rv = ENOMEM;
3043 		goto done;
3044 	}
3045 
3046 	req = ccb->ccb_cmd;
3047 	req->function = MPI_FUNCTION_RAID_ACTION;
3048 	req->action = MPI_MSG_RAID_ACTION_CH_VOL_SETTINGS;
3049 	req->vol_id = rpg0->volume_id;
3050 	req->vol_bus = rpg0->volume_bus;
3051 
3052 	memcpy(&req->data_word, &settings, sizeof(req->data_word));
3053 	ccb->ccb_done = mpi_empty_done;
3054 	if (mpi_poll(sc, ccb, 50000) != 0) {
3055 		rv = EIO;
3056 		goto done;
3057 	}
3058 
3059 	rep = (struct mpi_msg_raid_action_reply *)ccb->ccb_rcb;
3060 	if (rep == NULL)
3061 		panic("%s: raid volume settings change failed", DEVNAME(sc));
3062 
3063 	switch (lemtoh16(&rep->action_status)) {
3064 	case MPI_RAID_ACTION_STATUS_OK:
3065 		rv = 0;
3066 		break;
3067 	default:
3068 		rv = EIO;
3069 		break;
3070 	}
3071 
3072 	mpi_push_reply(sc, ccb->ccb_rcb);
3073 	scsi_io_put(&sc->sc_iopool, ccb);
3074 
3075 done:
3076 	free(rpg0, M_TEMP, len);
3077 	return (rv);
3078 }
3079 
3080 #if NBIO > 0
3081 int
3082 mpi_bio_get_pg0_raid(struct mpi_softc *sc, int id)
3083 {
3084 	int			len, rv = EINVAL;
3085 	u_int32_t		address;
3086 	struct mpi_cfg_hdr	hdr;
3087 	struct mpi_cfg_raid_vol_pg0 *rpg0;
3088 
3089 	/* get IOC page 2 */
3090 	if (mpi_req_cfg_page(sc, 0, 0, &sc->sc_cfg_hdr, 1, sc->sc_vol_page,
3091 	    sc->sc_cfg_hdr.page_length * 4) != 0) {
3092 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_bio_get_pg0_raid unable to "
3093 		    "fetch IOC page 2\n", DEVNAME(sc));
3094 		goto done;
3095 	}
3096 
3097 	/* XXX return something else than EINVAL to indicate within hs range */
3098 	if (id > sc->sc_vol_page->active_vols) {
3099 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_bio_get_pg0_raid invalid vol "
3100 		    "id: %d\n", DEVNAME(sc), id);
3101 		goto done;
3102 	}
3103 
3104 	/* replace current buffer with new one */
3105 	len = sizeof *rpg0 + sc->sc_vol_page->max_physdisks *
3106 	    sizeof(struct mpi_cfg_raid_vol_pg0_physdisk);
3107 	rpg0 = malloc(len, M_DEVBUF, M_WAITOK | M_CANFAIL);
3108 	if (rpg0 == NULL) {
3109 		printf("%s: can't get memory for RAID page 0, "
3110 		    "bio disabled\n", DEVNAME(sc));
3111 		goto done;
3112 	}
3113 	if (sc->sc_rpg0)
3114 		free(sc->sc_rpg0, M_DEVBUF, 0);
3115 	sc->sc_rpg0 = rpg0;
3116 
3117 	/* get raid vol page 0 */
3118 	address = sc->sc_vol_list[id].vol_id |
3119 	    (sc->sc_vol_list[id].vol_bus << 8);
3120 	if (mpi_req_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_VOL, 0,
3121 	    address, 0, &hdr) != 0)
3122 		goto done;
3123 	if (mpi_req_cfg_page(sc, address, 0, &hdr, 1, rpg0, len)) {
3124 		DNPRINTF(MPI_D_RAID, "%s: can't get RAID vol cfg page 0\n",
3125 		    DEVNAME(sc));
3126 		goto done;
3127 	}
3128 
3129 	rv = 0;
3130 done:
3131 	return (rv);
3132 }
3133 
3134 int
3135 mpi_ioctl(struct device *dev, u_long cmd, caddr_t addr)
3136 {
3137 	struct mpi_softc	*sc = (struct mpi_softc *)dev;
3138 	int error = 0;
3139 
3140 	DNPRINTF(MPI_D_IOCTL, "%s: mpi_ioctl ", DEVNAME(sc));
3141 
3142 	/* make sure we have bio enabled */
3143 	if (sc->sc_ioctl != mpi_ioctl)
3144 		return (EINVAL);
3145 
3146 	rw_enter_write(&sc->sc_lock);
3147 
3148 	switch (cmd) {
3149 	case BIOCINQ:
3150 		DNPRINTF(MPI_D_IOCTL, "inq\n");
3151 		error = mpi_ioctl_inq(sc, (struct bioc_inq *)addr);
3152 		break;
3153 
3154 	case BIOCVOL:
3155 		DNPRINTF(MPI_D_IOCTL, "vol\n");
3156 		error = mpi_ioctl_vol(sc, (struct bioc_vol *)addr);
3157 		break;
3158 
3159 	case BIOCDISK:
3160 		DNPRINTF(MPI_D_IOCTL, "disk\n");
3161 		error = mpi_ioctl_disk(sc, (struct bioc_disk *)addr);
3162 		break;
3163 
3164 	case BIOCALARM:
3165 		DNPRINTF(MPI_D_IOCTL, "alarm\n");
3166 		break;
3167 
3168 	case BIOCBLINK:
3169 		DNPRINTF(MPI_D_IOCTL, "blink\n");
3170 		break;
3171 
3172 	case BIOCSETSTATE:
3173 		DNPRINTF(MPI_D_IOCTL, "setstate\n");
3174 		error = mpi_ioctl_setstate(sc, (struct bioc_setstate *)addr);
3175 		break;
3176 
3177 	default:
3178 		DNPRINTF(MPI_D_IOCTL, " invalid ioctl\n");
3179 		error = ENOTTY;
3180 	}
3181 
3182 	rw_exit_write(&sc->sc_lock);
3183 
3184 	return (error);
3185 }
3186 
3187 int
3188 mpi_ioctl_inq(struct mpi_softc *sc, struct bioc_inq *bi)
3189 {
3190 	if (!(sc->sc_flags & MPI_F_RAID)) {
3191 		bi->bi_novol = 0;
3192 		bi->bi_nodisk = 0;
3193 	}
3194 
3195 	if (mpi_cfg_page(sc, 0, &sc->sc_cfg_hdr, 1, sc->sc_vol_page,
3196 	    sc->sc_cfg_hdr.page_length * 4) != 0) {
3197 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_get_raid unable to fetch IOC "
3198 		    "page 2\n", DEVNAME(sc));
3199 		return (EINVAL);
3200 	}
3201 
3202 	DNPRINTF(MPI_D_IOCTL, "%s:  active_vols: %d max_vols: %d "
3203 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
3204 	    sc->sc_vol_page->active_vols, sc->sc_vol_page->max_vols,
3205 	    sc->sc_vol_page->active_physdisks, sc->sc_vol_page->max_physdisks);
3206 
3207 	bi->bi_novol = sc->sc_vol_page->active_vols;
3208 	bi->bi_nodisk = sc->sc_vol_page->active_physdisks;
3209 	strlcpy(bi->bi_dev, DEVNAME(sc), sizeof(bi->bi_dev));
3210 
3211 	return (0);
3212 }
3213 
3214 int
3215 mpi_ioctl_vol(struct mpi_softc *sc, struct bioc_vol *bv)
3216 {
3217 	int			i, vol, id, rv = EINVAL;
3218 	struct device		*dev;
3219 	struct scsi_link	*link;
3220 	struct mpi_cfg_raid_vol_pg0 *rpg0;
3221 	char			*vendp;
3222 
3223 	id = bv->bv_volid;
3224 	if (mpi_bio_get_pg0_raid(sc, id))
3225 		goto done;
3226 
3227 	if (id > sc->sc_vol_page->active_vols)
3228 		return (EINVAL); /* XXX deal with hot spares */
3229 
3230 	rpg0 = sc->sc_rpg0;
3231 	if (rpg0 == NULL)
3232 		goto done;
3233 
3234 	/* determine status */
3235 	switch (rpg0->volume_state) {
3236 	case MPI_CFG_RAID_VOL_0_STATE_OPTIMAL:
3237 		bv->bv_status = BIOC_SVONLINE;
3238 		break;
3239 	case MPI_CFG_RAID_VOL_0_STATE_DEGRADED:
3240 		bv->bv_status = BIOC_SVDEGRADED;
3241 		break;
3242 	case MPI_CFG_RAID_VOL_0_STATE_FAILED:
3243 	case MPI_CFG_RAID_VOL_0_STATE_MISSING:
3244 		bv->bv_status = BIOC_SVOFFLINE;
3245 		break;
3246 	default:
3247 		bv->bv_status = BIOC_SVINVALID;
3248 	}
3249 
3250 	/* override status if scrubbing or something */
3251 	if (rpg0->volume_status & MPI_CFG_RAID_VOL_0_STATUS_RESYNCING)
3252 		bv->bv_status = BIOC_SVREBUILD;
3253 
3254 	bv->bv_size = (uint64_t)lemtoh32(&rpg0->max_lba) * 512;
3255 
3256 	switch (sc->sc_vol_list[id].vol_type) {
3257 	case MPI_CFG_RAID_TYPE_RAID_IS:
3258 		bv->bv_level = 0;
3259 		break;
3260 	case MPI_CFG_RAID_TYPE_RAID_IME:
3261 	case MPI_CFG_RAID_TYPE_RAID_IM:
3262 		bv->bv_level = 1;
3263 		break;
3264 	case MPI_CFG_RAID_TYPE_RAID_5:
3265 		bv->bv_level = 5;
3266 		break;
3267 	case MPI_CFG_RAID_TYPE_RAID_6:
3268 		bv->bv_level = 6;
3269 		break;
3270 	case MPI_CFG_RAID_TYPE_RAID_10:
3271 		bv->bv_level = 10;
3272 		break;
3273 	case MPI_CFG_RAID_TYPE_RAID_50:
3274 		bv->bv_level = 50;
3275 		break;
3276 	default:
3277 		bv->bv_level = -1;
3278 	}
3279 
3280 	bv->bv_nodisk = rpg0->num_phys_disks;
3281 
3282 	for (i = 0, vol = -1; i < sc->sc_buswidth; i++) {
3283 		link = scsi_get_link(sc->sc_scsibus, i, 0);
3284 		if (link == NULL)
3285 			continue;
3286 
3287 		/* skip if not a virtual disk */
3288 		if (!(link->flags & SDEV_VIRTUAL))
3289 			continue;
3290 
3291 		vol++;
3292 		/* are we it? */
3293 		if (vol == bv->bv_volid) {
3294 			dev = link->device_softc;
3295 			vendp = link->inqdata.vendor;
3296 			memcpy(bv->bv_vendor, vendp, sizeof bv->bv_vendor);
3297 			bv->bv_vendor[sizeof(bv->bv_vendor) - 1] = '\0';
3298 			strlcpy(bv->bv_dev, dev->dv_xname, sizeof bv->bv_dev);
3299 			break;
3300 		}
3301 	}
3302 	rv = 0;
3303 done:
3304 	return (rv);
3305 }
3306 
3307 int
3308 mpi_ioctl_disk(struct mpi_softc *sc, struct bioc_disk *bd)
3309 {
3310 	int			pdid, id, rv = EINVAL;
3311 	u_int32_t		address;
3312 	struct mpi_cfg_hdr	hdr;
3313 	struct mpi_cfg_raid_vol_pg0 *rpg0;
3314 	struct mpi_cfg_raid_vol_pg0_physdisk *physdisk;
3315 	struct mpi_cfg_raid_physdisk_pg0 pdpg0;
3316 
3317 	id = bd->bd_volid;
3318 	if (mpi_bio_get_pg0_raid(sc, id))
3319 		goto done;
3320 
3321 	if (id > sc->sc_vol_page->active_vols)
3322 		return (EINVAL); /* XXX deal with hot spares */
3323 
3324 	rpg0 = sc->sc_rpg0;
3325 	if (rpg0 == NULL)
3326 		goto done;
3327 
3328 	pdid = bd->bd_diskid;
3329 	if (pdid > rpg0->num_phys_disks)
3330 		goto done;
3331 	physdisk = (struct mpi_cfg_raid_vol_pg0_physdisk *)(rpg0 + 1);
3332 	physdisk += pdid;
3333 
3334 	/* get raid phys disk page 0 */
3335 	address = physdisk->phys_disk_num;
3336 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_PD, 0, address,
3337 	    &hdr) != 0)
3338 		goto done;
3339 	if (mpi_cfg_page(sc, address, &hdr, 1, &pdpg0, sizeof pdpg0)) {
3340 		bd->bd_status = BIOC_SDFAILED;
3341 		return (0);
3342 	}
3343 	bd->bd_channel = pdpg0.phys_disk_bus;
3344 	bd->bd_target = pdpg0.phys_disk_id;
3345 	bd->bd_lun = 0;
3346 	bd->bd_size = (uint64_t)lemtoh32(&pdpg0.max_lba) * 512;
3347 	strlcpy(bd->bd_vendor, (char *)pdpg0.vendor_id, sizeof(bd->bd_vendor));
3348 
3349 	switch (pdpg0.phys_disk_state) {
3350 	case MPI_CFG_RAID_PHYDISK_0_STATE_ONLINE:
3351 		bd->bd_status = BIOC_SDONLINE;
3352 		break;
3353 	case MPI_CFG_RAID_PHYDISK_0_STATE_MISSING:
3354 	case MPI_CFG_RAID_PHYDISK_0_STATE_FAILED:
3355 		bd->bd_status = BIOC_SDFAILED;
3356 		break;
3357 	case MPI_CFG_RAID_PHYDISK_0_STATE_HOSTFAIL:
3358 	case MPI_CFG_RAID_PHYDISK_0_STATE_OTHER:
3359 	case MPI_CFG_RAID_PHYDISK_0_STATE_OFFLINE:
3360 		bd->bd_status = BIOC_SDOFFLINE;
3361 		break;
3362 	case MPI_CFG_RAID_PHYDISK_0_STATE_INIT:
3363 		bd->bd_status = BIOC_SDSCRUB;
3364 		break;
3365 	case MPI_CFG_RAID_PHYDISK_0_STATE_INCOMPAT:
3366 	default:
3367 		bd->bd_status = BIOC_SDINVALID;
3368 		break;
3369 	}
3370 
3371 	/* XXX figure this out */
3372 	/* bd_serial[32]; */
3373 	/* bd_procdev[16]; */
3374 
3375 	rv = 0;
3376 done:
3377 	return (rv);
3378 }
3379 
3380 int
3381 mpi_ioctl_setstate(struct mpi_softc *sc, struct bioc_setstate *bs)
3382 {
3383 	return (ENOTTY);
3384 }
3385 
3386 #ifndef SMALL_KERNEL
3387 int
3388 mpi_create_sensors(struct mpi_softc *sc)
3389 {
3390 	struct device		*dev;
3391 	struct scsi_link	*link;
3392 	int			i, vol, nsensors;
3393 
3394 	/* count volumes */
3395 	for (i = 0, vol = 0; i < sc->sc_buswidth; i++) {
3396 		link = scsi_get_link(sc->sc_scsibus, i, 0);
3397 		if (link == NULL)
3398 			continue;
3399 		/* skip if not a virtual disk */
3400 		if (!(link->flags & SDEV_VIRTUAL))
3401 			continue;
3402 
3403 		vol++;
3404 	}
3405 	if (vol == 0)
3406 		return (0);
3407 
3408 	sc->sc_sensors = mallocarray(vol, sizeof(struct ksensor),
3409 	    M_DEVBUF, M_NOWAIT | M_ZERO);
3410 	if (sc->sc_sensors == NULL)
3411 		return (1);
3412 	nsensors = vol;
3413 
3414 	strlcpy(sc->sc_sensordev.xname, DEVNAME(sc),
3415 	    sizeof(sc->sc_sensordev.xname));
3416 
3417 	for (i = 0, vol= 0; i < sc->sc_buswidth; i++) {
3418 		link = scsi_get_link(sc->sc_scsibus, i, 0);
3419 		if (link == NULL)
3420 			continue;
3421 		/* skip if not a virtual disk */
3422 		if (!(link->flags & SDEV_VIRTUAL))
3423 			continue;
3424 
3425 		dev = link->device_softc;
3426 		strlcpy(sc->sc_sensors[vol].desc, dev->dv_xname,
3427 		    sizeof(sc->sc_sensors[vol].desc));
3428 		sc->sc_sensors[vol].type = SENSOR_DRIVE;
3429 		sc->sc_sensors[vol].status = SENSOR_S_UNKNOWN;
3430 		sensor_attach(&sc->sc_sensordev, &sc->sc_sensors[vol]);
3431 
3432 		vol++;
3433 	}
3434 
3435 	if (sensor_task_register(sc, mpi_refresh_sensors, 10) == NULL)
3436 		goto bad;
3437 
3438 	sensordev_install(&sc->sc_sensordev);
3439 
3440 	return (0);
3441 
3442 bad:
3443 	free(sc->sc_sensors, M_DEVBUF, nsensors * sizeof(struct ksensor));
3444 	return (1);
3445 }
3446 
3447 void
3448 mpi_refresh_sensors(void *arg)
3449 {
3450 	int			i, vol;
3451 	struct scsi_link	*link;
3452 	struct mpi_softc	*sc = arg;
3453 	struct mpi_cfg_raid_vol_pg0 *rpg0;
3454 
3455 	rw_enter_write(&sc->sc_lock);
3456 
3457 	for (i = 0, vol = 0; i < sc->sc_buswidth; i++) {
3458 		link = scsi_get_link(sc->sc_scsibus, i, 0);
3459 		if (link == NULL)
3460 			continue;
3461 		/* skip if not a virtual disk */
3462 		if (!(link->flags & SDEV_VIRTUAL))
3463 			continue;
3464 
3465 		if (mpi_bio_get_pg0_raid(sc, vol))
3466 			continue;
3467 
3468 		rpg0 = sc->sc_rpg0;
3469 		if (rpg0 == NULL)
3470 			goto done;
3471 
3472 		/* determine status */
3473 		switch (rpg0->volume_state) {
3474 		case MPI_CFG_RAID_VOL_0_STATE_OPTIMAL:
3475 			sc->sc_sensors[vol].value = SENSOR_DRIVE_ONLINE;
3476 			sc->sc_sensors[vol].status = SENSOR_S_OK;
3477 			break;
3478 		case MPI_CFG_RAID_VOL_0_STATE_DEGRADED:
3479 			sc->sc_sensors[vol].value = SENSOR_DRIVE_PFAIL;
3480 			sc->sc_sensors[vol].status = SENSOR_S_WARN;
3481 			break;
3482 		case MPI_CFG_RAID_VOL_0_STATE_FAILED:
3483 		case MPI_CFG_RAID_VOL_0_STATE_MISSING:
3484 			sc->sc_sensors[vol].value = SENSOR_DRIVE_FAIL;
3485 			sc->sc_sensors[vol].status = SENSOR_S_CRIT;
3486 			break;
3487 		default:
3488 			sc->sc_sensors[vol].value = 0; /* unknown */
3489 			sc->sc_sensors[vol].status = SENSOR_S_UNKNOWN;
3490 		}
3491 
3492 		/* override status if scrubbing or something */
3493 		if (rpg0->volume_status & MPI_CFG_RAID_VOL_0_STATUS_RESYNCING) {
3494 			sc->sc_sensors[vol].value = SENSOR_DRIVE_REBUILD;
3495 			sc->sc_sensors[vol].status = SENSOR_S_WARN;
3496 		}
3497 
3498 		vol++;
3499 	}
3500 done:
3501 	rw_exit_write(&sc->sc_lock);
3502 }
3503 #endif /* SMALL_KERNEL */
3504 #endif /* NBIO > 0 */
3505