xref: /openbsd-src/sys/dev/ic/mpi.c (revision cd1eb269cafb12c415be1749cd4a4b5422710415)
1 /*	$OpenBSD: mpi.c,v 1.146 2010/04/28 11:46:23 marco Exp $ */
2 
3 /*
4  * Copyright (c) 2005, 2006, 2009 David Gwynne <dlg@openbsd.org>
5  * Copyright (c) 2005, 2008, 2009 Marco Peereboom <marco@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include "bio.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/buf.h>
25 #include <sys/device.h>
26 #include <sys/ioctl.h>
27 #include <sys/proc.h>
28 #include <sys/malloc.h>
29 #include <sys/kernel.h>
30 #include <sys/mutex.h>
31 #include <sys/rwlock.h>
32 #include <sys/sensors.h>
33 
34 #include <machine/bus.h>
35 
36 #include <scsi/scsi_all.h>
37 #include <scsi/scsiconf.h>
38 
39 #include <dev/biovar.h>
40 #include <dev/ic/mpireg.h>
41 #include <dev/ic/mpivar.h>
42 
43 #ifdef MPI_DEBUG
44 uint32_t	mpi_debug = 0
45 /*		    | MPI_D_CMD */
46 /*		    | MPI_D_INTR */
47 /*		    | MPI_D_MISC */
48 /*		    | MPI_D_DMA */
49 /*		    | MPI_D_IOCTL */
50 /*		    | MPI_D_RW */
51 /*		    | MPI_D_MEM */
52 /*		    | MPI_D_CCB */
53 /*		    | MPI_D_PPR */
54 /*		    | MPI_D_RAID */
55 /*		    | MPI_D_EVT */
56 		;
57 #endif
58 
59 struct cfdriver mpi_cd = {
60 	NULL,
61 	"mpi",
62 	DV_DULL
63 };
64 
65 void			mpi_scsi_cmd(struct scsi_xfer *);
66 void			mpi_scsi_cmd_done(struct mpi_ccb *);
67 void			mpi_minphys(struct buf *bp, struct scsi_link *sl);
68 int			mpi_scsi_probe(struct scsi_link *);
69 int			mpi_scsi_ioctl(struct scsi_link *, u_long, caddr_t,
70 			    int, struct proc *);
71 
72 struct scsi_adapter mpi_switch = {
73 	mpi_scsi_cmd,
74 	mpi_minphys,
75 	mpi_scsi_probe,
76 	NULL,
77 	mpi_scsi_ioctl
78 };
79 
80 struct scsi_device mpi_dev = {
81 	NULL,
82 	NULL,
83 	NULL,
84 	NULL
85 };
86 
87 struct mpi_dmamem	*mpi_dmamem_alloc(struct mpi_softc *, size_t);
88 void			mpi_dmamem_free(struct mpi_softc *,
89 			    struct mpi_dmamem *);
90 int			mpi_alloc_ccbs(struct mpi_softc *);
91 struct mpi_ccb		*mpi_get_ccb(struct mpi_softc *);
92 void			mpi_put_ccb(struct mpi_softc *, struct mpi_ccb *);
93 int			mpi_alloc_replies(struct mpi_softc *);
94 void			mpi_push_replies(struct mpi_softc *);
95 void			mpi_push_reply(struct mpi_softc *, struct mpi_rcb *);
96 
97 void			mpi_start(struct mpi_softc *, struct mpi_ccb *);
98 int			mpi_poll(struct mpi_softc *, struct mpi_ccb *, int);
99 void			mpi_poll_done(struct mpi_ccb *);
100 void			mpi_reply(struct mpi_softc *, u_int32_t);
101 
102 void			mpi_wait(struct mpi_softc *sc, struct mpi_ccb *);
103 void			mpi_wait_done(struct mpi_ccb *);
104 
105 int			mpi_cfg_spi_port(struct mpi_softc *);
106 void			mpi_squash_ppr(struct mpi_softc *);
107 void			mpi_run_ppr(struct mpi_softc *);
108 int			mpi_ppr(struct mpi_softc *, struct scsi_link *,
109 			    struct mpi_cfg_raid_physdisk *, int, int, int);
110 int			mpi_inq(struct mpi_softc *, u_int16_t, int);
111 
112 void			mpi_fc_info(struct mpi_softc *);
113 
114 void			mpi_timeout_xs(void *);
115 int			mpi_load_xs(struct mpi_ccb *);
116 
117 u_int32_t		mpi_read(struct mpi_softc *, bus_size_t);
118 void			mpi_write(struct mpi_softc *, bus_size_t, u_int32_t);
119 int			mpi_wait_eq(struct mpi_softc *, bus_size_t, u_int32_t,
120 			    u_int32_t);
121 int			mpi_wait_ne(struct mpi_softc *, bus_size_t, u_int32_t,
122 			    u_int32_t);
123 
124 int			mpi_init(struct mpi_softc *);
125 int			mpi_reset_soft(struct mpi_softc *);
126 int			mpi_reset_hard(struct mpi_softc *);
127 
128 int			mpi_handshake_send(struct mpi_softc *, void *, size_t);
129 int			mpi_handshake_recv_dword(struct mpi_softc *,
130 			    u_int32_t *);
131 int			mpi_handshake_recv(struct mpi_softc *, void *, size_t);
132 
133 void			mpi_empty_done(struct mpi_ccb *);
134 
135 int			mpi_iocinit(struct mpi_softc *);
136 int			mpi_iocfacts(struct mpi_softc *);
137 int			mpi_portfacts(struct mpi_softc *);
138 int			mpi_portenable(struct mpi_softc *);
139 int			mpi_cfg_coalescing(struct mpi_softc *);
140 void			mpi_get_raid(struct mpi_softc *);
141 int			mpi_fwupload(struct mpi_softc *);
142 
143 int			mpi_eventnotify(struct mpi_softc *);
144 void			mpi_eventnotify_done(struct mpi_ccb *);
145 void			mpi_eventack(struct mpi_softc *,
146 			    struct mpi_msg_event_reply *);
147 void			mpi_eventack_done(struct mpi_ccb *);
148 void			mpi_evt_sas(struct mpi_softc *, struct mpi_rcb *);
149 
150 int			mpi_req_cfg_header(struct mpi_softc *, u_int8_t,
151 			    u_int8_t, u_int32_t, int, void *);
152 int			mpi_req_cfg_page(struct mpi_softc *, u_int32_t, int,
153 			    void *, int, void *, size_t);
154 
155 #if NBIO > 0
156 int		mpi_bio_get_pg0_raid(struct mpi_softc *, int);
157 int		mpi_ioctl(struct device *, u_long, caddr_t);
158 int		mpi_ioctl_inq(struct mpi_softc *, struct bioc_inq *);
159 int		mpi_ioctl_vol(struct mpi_softc *, struct bioc_vol *);
160 int		mpi_ioctl_disk(struct mpi_softc *, struct bioc_disk *);
161 int		mpi_ioctl_setstate(struct mpi_softc *, struct bioc_setstate *);
162 #ifndef SMALL_KERNEL
163 int		mpi_create_sensors(struct mpi_softc *);
164 void		mpi_refresh_sensors(void *);
165 #endif /* SMALL_KERNEL */
166 #endif /* NBIO > 0 */
167 
168 #define DEVNAME(s)		((s)->sc_dev.dv_xname)
169 
170 #define	dwordsof(s)		(sizeof(s) / sizeof(u_int32_t))
171 
172 #define mpi_read_db(s)		mpi_read((s), MPI_DOORBELL)
173 #define mpi_write_db(s, v)	mpi_write((s), MPI_DOORBELL, (v))
174 #define mpi_read_intr(s)	mpi_read((s), MPI_INTR_STATUS)
175 #define mpi_write_intr(s, v)	mpi_write((s), MPI_INTR_STATUS, (v))
176 #define mpi_pop_reply(s)	mpi_read((s), MPI_REPLY_QUEUE)
177 #define mpi_push_reply_db(s, v)	mpi_write((s), MPI_REPLY_QUEUE, (v))
178 
179 #define mpi_wait_db_int(s)	mpi_wait_ne((s), MPI_INTR_STATUS, \
180 				    MPI_INTR_STATUS_DOORBELL, 0)
181 #define mpi_wait_db_ack(s)	mpi_wait_eq((s), MPI_INTR_STATUS, \
182 				    MPI_INTR_STATUS_IOCDOORBELL, 0)
183 
184 #define MPI_PG_EXTENDED		(1<<0)
185 #define MPI_PG_POLL		(1<<1)
186 #define MPI_PG_FMT		"\020" "\002POLL" "\001EXTENDED"
187 
188 #define mpi_cfg_header(_s, _t, _n, _a, _h) \
189 	mpi_req_cfg_header((_s), (_t), (_n), (_a), \
190 	    MPI_PG_POLL, (_h))
191 #define mpi_ecfg_header(_s, _t, _n, _a, _h) \
192 	mpi_req_cfg_header((_s), (_t), (_n), (_a), \
193 	    MPI_PG_POLL|MPI_PG_EXTENDED, (_h))
194 
195 #define mpi_cfg_page(_s, _a, _h, _r, _p, _l) \
196 	mpi_req_cfg_page((_s), (_a), MPI_PG_POLL, \
197 	    (_h), (_r), (_p), (_l))
198 #define mpi_ecfg_page(_s, _a, _h, _r, _p, _l) \
199 	mpi_req_cfg_page((_s), (_a), MPI_PG_POLL|MPI_PG_EXTENDED, \
200 	    (_h), (_r), (_p), (_l))
201 
202 int
203 mpi_attach(struct mpi_softc *sc)
204 {
205 	struct scsibus_attach_args	saa;
206 	struct mpi_ccb			*ccb;
207 
208 	printf("\n");
209 
210 	/* disable interrupts */
211 	mpi_write(sc, MPI_INTR_MASK,
212 	    MPI_INTR_MASK_REPLY | MPI_INTR_MASK_DOORBELL);
213 
214 	if (mpi_init(sc) != 0) {
215 		printf("%s: unable to initialise\n", DEVNAME(sc));
216 		return (1);
217 	}
218 
219 	if (mpi_iocfacts(sc) != 0) {
220 		printf("%s: unable to get iocfacts\n", DEVNAME(sc));
221 		return (1);
222 	}
223 
224 	if (mpi_alloc_ccbs(sc) != 0) {
225 		/* error already printed */
226 		return (1);
227 	}
228 
229 	if (mpi_alloc_replies(sc) != 0) {
230 		printf("%s: unable to allocate reply space\n", DEVNAME(sc));
231 		goto free_ccbs;
232 	}
233 
234 	if (mpi_iocinit(sc) != 0) {
235 		printf("%s: unable to send iocinit\n", DEVNAME(sc));
236 		goto free_ccbs;
237 	}
238 
239 	/* spin until we're operational */
240 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
241 	    MPI_DOORBELL_STATE_OPER) != 0) {
242 		printf("%s: state: 0x%08x\n", DEVNAME(sc),
243 		    mpi_read_db(sc) & MPI_DOORBELL_STATE);
244 		printf("%s: operational state timeout\n", DEVNAME(sc));
245 		goto free_ccbs;
246 	}
247 
248 	mpi_push_replies(sc);
249 
250 	if (mpi_portfacts(sc) != 0) {
251 		printf("%s: unable to get portfacts\n", DEVNAME(sc));
252 		goto free_replies;
253 	}
254 
255 	if (mpi_cfg_coalescing(sc) != 0) {
256 		printf("%s: unable to configure coalescing\n", DEVNAME(sc));
257 		goto free_replies;
258 	}
259 
260 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SAS) {
261 		if (mpi_eventnotify(sc) != 0) {
262 			printf("%s: unable to enable events\n", DEVNAME(sc));
263 			goto free_replies;
264 		}
265 	}
266 
267 	if (mpi_portenable(sc) != 0) {
268 		printf("%s: unable to enable port\n", DEVNAME(sc));
269 		goto free_replies;
270 	}
271 
272 	if (mpi_fwupload(sc) != 0) {
273 		printf("%s: unable to upload firmware\n", DEVNAME(sc));
274 		goto free_replies;
275 	}
276 
277 	switch (sc->sc_porttype) {
278 	case MPI_PORTFACTS_PORTTYPE_SCSI:
279 		if (mpi_cfg_spi_port(sc) != 0)
280 			goto free_replies;
281 		mpi_squash_ppr(sc);
282 		break;
283 	case MPI_PORTFACTS_PORTTYPE_FC:
284 		mpi_fc_info(sc);
285 		break;
286 	}
287 
288 	rw_init(&sc->sc_lock, "mpi_lock");
289 
290 	/* we should be good to go now, attach scsibus */
291 	sc->sc_link.device = &mpi_dev;
292 	sc->sc_link.adapter = &mpi_switch;
293 	sc->sc_link.adapter_softc = sc;
294 	sc->sc_link.adapter_target = sc->sc_target;
295 	sc->sc_link.adapter_buswidth = sc->sc_buswidth;
296 	sc->sc_link.openings = sc->sc_maxcmds;
297 	sc->sc_link.pool = &sc->sc_iopool;
298 
299 	bzero(&saa, sizeof(saa));
300 	saa.saa_sc_link = &sc->sc_link;
301 
302 	/* config_found() returns the scsibus attached to us */
303 	sc->sc_scsibus = (struct scsibus_softc *) config_found(&sc->sc_dev,
304 	    &saa, scsiprint);
305 
306 	/* get raid pages */
307 	mpi_get_raid(sc);
308 
309 	/* do domain validation */
310 	if (sc->sc_porttype == MPI_PORTFACTS_PORTTYPE_SCSI)
311 		mpi_run_ppr(sc);
312 
313 	/* enable interrupts */
314 	mpi_write(sc, MPI_INTR_MASK, MPI_INTR_MASK_DOORBELL);
315 
316 #if NBIO > 0
317 	if (sc->sc_flags & MPI_F_RAID) {
318 		if (bio_register(&sc->sc_dev, mpi_ioctl) != 0)
319 			panic("%s: controller registration failed",
320 			    DEVNAME(sc));
321 		else {
322 			if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC,
323 			    2, 0, &sc->sc_cfg_hdr) != 0) {
324 				printf("%s: can't get IOC page 2 hdr, bio "
325 				    "disabled\n", DEVNAME(sc));
326 				goto done;
327 			}
328 			sc->sc_vol_page = malloc(sc->sc_cfg_hdr.page_length * 4,
329 			    M_TEMP, M_WAITOK | M_CANFAIL);
330 			if (sc->sc_vol_page == NULL) {
331 				printf("%s: can't get memory for IOC page 2, "
332 				    "bio disabled\n", DEVNAME(sc));
333 				goto done;
334 			}
335 			sc->sc_vol_list = (struct mpi_cfg_raid_vol *)
336 			    (sc->sc_vol_page + 1);
337 
338 			sc->sc_ioctl = mpi_ioctl;
339 		}
340 	}
341 #ifndef SMALL_KERNEL
342 	mpi_create_sensors(sc);
343 #endif /* SMALL_KERNEL */
344 done:
345 #endif /* NBIO > 0 */
346 
347 	return (0);
348 
349 free_replies:
350 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies), 0,
351 	    sc->sc_repq * MPI_REPLY_SIZE, BUS_DMASYNC_POSTREAD);
352 	mpi_dmamem_free(sc, sc->sc_replies);
353 free_ccbs:
354 	while ((ccb = mpi_get_ccb(sc)) != NULL)
355 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
356 	mpi_dmamem_free(sc, sc->sc_requests);
357 	free(sc->sc_ccbs, M_DEVBUF);
358 
359 	return(1);
360 }
361 
362 int
363 mpi_cfg_spi_port(struct mpi_softc *sc)
364 {
365 	struct mpi_cfg_hdr		hdr;
366 	struct mpi_cfg_spi_port_pg1	port;
367 
368 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 1, 0x0,
369 	    &hdr) != 0)
370 		return (1);
371 
372 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port, sizeof(port)) != 0)
373 		return (1);
374 
375 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_spi_port_pg1\n", DEVNAME(sc));
376 	DNPRINTF(MPI_D_MISC, "%s:  port_scsi_id: %d port_resp_ids 0x%04x\n",
377 	    DEVNAME(sc), port.port_scsi_id, letoh16(port.port_resp_ids));
378 	DNPRINTF(MPI_D_MISC, "%s:  on_bus_timer_value: 0x%08x\n", DEVNAME(sc),
379 	    letoh32(port.port_scsi_id));
380 	DNPRINTF(MPI_D_MISC, "%s:  target_config: 0x%02x id_config: 0x%04x\n",
381 	    DEVNAME(sc), port.target_config, letoh16(port.id_config));
382 
383 	if (port.port_scsi_id == sc->sc_target &&
384 	    port.port_resp_ids == htole16(1 << sc->sc_target) &&
385 	    port.on_bus_timer_value != htole32(0x0))
386 		return (0);
387 
388 	DNPRINTF(MPI_D_MISC, "%s: setting port scsi id to %d\n", DEVNAME(sc),
389 	    sc->sc_target);
390 	port.port_scsi_id = sc->sc_target;
391 	port.port_resp_ids = htole16(1 << sc->sc_target);
392 	port.on_bus_timer_value = htole32(0x07000000); /* XXX magic */
393 
394 	if (mpi_cfg_page(sc, 0x0, &hdr, 0, &port, sizeof(port)) != 0) {
395 		printf("%s: unable to configure port scsi id\n", DEVNAME(sc));
396 		return (1);
397 	}
398 
399 	return (0);
400 }
401 
402 void
403 mpi_squash_ppr(struct mpi_softc *sc)
404 {
405 	struct mpi_cfg_hdr		hdr;
406 	struct mpi_cfg_spi_dev_pg1	page;
407 	int				i;
408 
409 	DNPRINTF(MPI_D_PPR, "%s: mpi_squash_ppr\n", DEVNAME(sc));
410 
411 	for (i = 0; i < sc->sc_buswidth; i++) {
412 		if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV,
413 		    1, i, &hdr) != 0)
414 			return;
415 
416 		if (mpi_cfg_page(sc, i, &hdr, 1, &page, sizeof(page)) != 0)
417 			return;
418 
419 		DNPRINTF(MPI_D_PPR, "%s:  target: %d req_params1: 0x%02x "
420 		    "req_offset: 0x%02x req_period: 0x%02x "
421 		    "req_params2: 0x%02x conf: 0x%08x\n", DEVNAME(sc), i,
422 		    page.req_params1, page.req_offset, page.req_period,
423 		    page.req_params2, letoh32(page.configuration));
424 
425 		page.req_params1 = 0x0;
426 		page.req_offset = 0x0;
427 		page.req_period = 0x0;
428 		page.req_params2 = 0x0;
429 		page.configuration = htole32(0x0);
430 
431 		if (mpi_cfg_page(sc, i, &hdr, 0, &page, sizeof(page)) != 0)
432 			return;
433 	}
434 }
435 
436 void
437 mpi_run_ppr(struct mpi_softc *sc)
438 {
439 	struct mpi_cfg_hdr		hdr;
440 	struct mpi_cfg_spi_port_pg0	port_pg;
441 	struct mpi_cfg_ioc_pg3		*physdisk_pg;
442 	struct mpi_cfg_raid_physdisk	*physdisk_list, *physdisk;
443 	size_t				pagelen;
444 	struct scsi_link		*link;
445 	int				i, tries;
446 
447 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_PORT, 0, 0x0,
448 	    &hdr) != 0) {
449 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch header\n",
450 		    DEVNAME(sc));
451 		return;
452 	}
453 
454 	if (mpi_cfg_page(sc, 0x0, &hdr, 1, &port_pg, sizeof(port_pg)) != 0) {
455 		DNPRINTF(MPI_D_PPR, "%s: mpi_run_ppr unable to fetch page\n",
456 		    DEVNAME(sc));
457 		return;
458 	}
459 
460 	for (i = 0; i < sc->sc_buswidth; i++) {
461 		link = sc->sc_scsibus->sc_link[i][0];
462 		if (link == NULL)
463 			continue;
464 
465 		/* do not ppr volumes */
466 		if (link->flags & SDEV_VIRTUAL)
467 			continue;
468 
469 		tries = 0;
470 		while (mpi_ppr(sc, link, NULL, port_pg.min_period,
471 		    port_pg.max_offset, tries) == EAGAIN)
472 			tries++;
473 	}
474 
475 	if ((sc->sc_flags & MPI_F_RAID) == 0)
476 		return;
477 
478 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 3, 0x0,
479 	    &hdr) != 0) {
480 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
481 		    "fetch ioc pg 3 header\n", DEVNAME(sc));
482 		return;
483 	}
484 
485 	pagelen = hdr.page_length * 4; /* dwords to bytes */
486 	physdisk_pg = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
487 	if (physdisk_pg == NULL) {
488 		DNPRINTF(MPI_D_RAID|MPI_D_PPR, "%s: mpi_run_ppr unable to "
489 		    "allocate ioc pg 3\n", DEVNAME(sc));
490 		return;
491 	}
492 	physdisk_list = (struct mpi_cfg_raid_physdisk *)(physdisk_pg + 1);
493 
494 	if (mpi_cfg_page(sc, 0, &hdr, 1, physdisk_pg, pagelen) != 0) {
495 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s: mpi_run_ppr unable to "
496 		    "fetch ioc page 3\n", DEVNAME(sc));
497 		goto out;
498 	}
499 
500 	DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  no_phys_disks: %d\n", DEVNAME(sc),
501 	    physdisk_pg->no_phys_disks);
502 
503 	for (i = 0; i < physdisk_pg->no_phys_disks; i++) {
504 		physdisk = &physdisk_list[i];
505 
506 		DNPRINTF(MPI_D_PPR|MPI_D_PPR, "%s:  id: %d bus: %d ioc: %d "
507 		    "num: %d\n", DEVNAME(sc), physdisk->phys_disk_id,
508 		    physdisk->phys_disk_bus, physdisk->phys_disk_ioc,
509 		    physdisk->phys_disk_num);
510 
511 		if (physdisk->phys_disk_ioc != sc->sc_ioc_number)
512 			continue;
513 
514 		tries = 0;
515 		while (mpi_ppr(sc, NULL, physdisk, port_pg.min_period,
516 		    port_pg.max_offset, tries) == EAGAIN)
517 			tries++;
518 	}
519 
520 out:
521 	free(physdisk_pg, M_TEMP);
522 }
523 
524 int
525 mpi_ppr(struct mpi_softc *sc, struct scsi_link *link,
526     struct mpi_cfg_raid_physdisk *physdisk, int period, int offset, int try)
527 {
528 	struct mpi_cfg_hdr		hdr0, hdr1;
529 	struct mpi_cfg_spi_dev_pg0	pg0;
530 	struct mpi_cfg_spi_dev_pg1	pg1;
531 	u_int32_t			address;
532 	int				id;
533 	int				raid = 0;
534 
535 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr period: %d offset: %d try: %d "
536 	    "link quirks: 0x%x\n", DEVNAME(sc), period, offset, try,
537 	    link->quirks);
538 
539 	if (try >= 3)
540 		return (EIO);
541 
542 	if (physdisk == NULL) {
543 		if ((link->inqdata.device & SID_TYPE) == T_PROCESSOR)
544 			return (EIO);
545 
546 		address = link->target;
547 		id = link->target;
548 	} else {
549 		raid = 1;
550 		address = (physdisk->phys_disk_bus << 8) |
551 		    (physdisk->phys_disk_id);
552 		id = physdisk->phys_disk_num;
553 	}
554 
555 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 0,
556 	    address, &hdr0) != 0) {
557 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 0\n",
558 		    DEVNAME(sc));
559 		return (EIO);
560 	}
561 
562 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_SCSI_SPI_DEV, 1,
563 	    address, &hdr1) != 0) {
564 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch header 1\n",
565 		    DEVNAME(sc));
566 		return (EIO);
567 	}
568 
569 #ifdef MPI_DEBUG
570 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
571 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 0\n",
572 		    DEVNAME(sc));
573 		return (EIO);
574 	}
575 
576 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
577 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
578 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
579 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
580 #endif
581 
582 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
583 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to fetch page 1\n",
584 		    DEVNAME(sc));
585 		return (EIO);
586 	}
587 
588 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
589 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
590 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
591 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
592 
593 	pg1.req_params1 = 0;
594 	pg1.req_offset = offset;
595 	pg1.req_period = period;
596 	pg1.req_params2 &= ~MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH;
597 
598 	if (raid || !(link->quirks & SDEV_NOSYNC)) {
599 		pg1.req_params2 |= MPI_CFG_SPI_DEV_1_REQPARAMS_WIDTH_WIDE;
600 
601 		switch (try) {
602 		case 0: /* U320 */
603 			break;
604 		case 1: /* U160 */
605 			pg1.req_period = 0x09;
606 			break;
607 		case 2: /* U80 */
608 			pg1.req_period = 0x0a;
609 			break;
610 		}
611 
612 		if (pg1.req_period < 0x09) {
613 			/* Ultra320: enable QAS & PACKETIZED */
614 			pg1.req_params1 |= MPI_CFG_SPI_DEV_1_REQPARAMS_QAS |
615 			    MPI_CFG_SPI_DEV_1_REQPARAMS_PACKETIZED;
616 		}
617 		if (pg1.req_period < 0xa) {
618 			/* >= Ultra160: enable dual xfers */
619 			pg1.req_params1 |=
620 			    MPI_CFG_SPI_DEV_1_REQPARAMS_DUALXFERS;
621 		}
622 	}
623 
624 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
625 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
626 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
627 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
628 
629 	if (mpi_cfg_page(sc, address, &hdr1, 0, &pg1, sizeof(pg1)) != 0) {
630 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to write page 1\n",
631 		    DEVNAME(sc));
632 		return (EIO);
633 	}
634 
635 	if (mpi_cfg_page(sc, address, &hdr1, 1, &pg1, sizeof(pg1)) != 0) {
636 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 1\n",
637 		    DEVNAME(sc));
638 		return (EIO);
639 	}
640 
641 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 1 req_params1: 0x%02x "
642 	    "req_offset: 0x%02x req_period: 0x%02x req_params2: 0x%02x "
643 	    "conf: 0x%08x\n", DEVNAME(sc), pg1.req_params1, pg1.req_offset,
644 	    pg1.req_period, pg1.req_params2, letoh32(pg1.configuration));
645 
646 	if (mpi_inq(sc, id, raid) != 0) {
647 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to do inquiry against "
648 		    "target %d\n", DEVNAME(sc), link->target);
649 		return (EIO);
650 	}
651 
652 	if (mpi_cfg_page(sc, address, &hdr0, 1, &pg0, sizeof(pg0)) != 0) {
653 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr unable to read page 0 after "
654 		    "inquiry\n", DEVNAME(sc));
655 		return (EIO);
656 	}
657 
658 	DNPRINTF(MPI_D_PPR, "%s: mpi_ppr dev pg 0 neg_params1: 0x%02x "
659 	    "neg_offset: %d neg_period: 0x%02x neg_params2: 0x%02x "
660 	    "info: 0x%08x\n", DEVNAME(sc), pg0.neg_params1, pg0.neg_offset,
661 	    pg0.neg_period, pg0.neg_params2, letoh32(pg0.information));
662 
663 	if (!(letoh32(pg0.information) & 0x07) && (try == 0)) {
664 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U320 ppr rejected\n",
665 		    DEVNAME(sc));
666 		return (EAGAIN);
667 	}
668 
669 	if ((((letoh32(pg0.information) >> 8) & 0xff) > 0x09) && (try == 1)) {
670 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr U160 ppr rejected\n",
671 		    DEVNAME(sc));
672 		return (EAGAIN);
673 	}
674 
675 	if (letoh32(pg0.information) & 0x0e) {
676 		DNPRINTF(MPI_D_PPR, "%s: mpi_ppr ppr rejected: %0x\n",
677 		    DEVNAME(sc), letoh32(pg0.information));
678 		return (EAGAIN);
679 	}
680 
681 	switch(pg0.neg_period) {
682 	case 0x08:
683 		period = 160;
684 		break;
685 	case 0x09:
686 		period = 80;
687 		break;
688 	case 0x0a:
689 		period = 40;
690 		break;
691 	case 0x0b:
692 		period = 20;
693 		break;
694 	case 0x0c:
695 		period = 10;
696 		break;
697 	default:
698 		period = 0;
699 		break;
700 	}
701 
702 	printf("%s: %s %d %s at %dMHz width %dbit offset %d "
703 	    "QAS %d DT %d IU %d\n", DEVNAME(sc), raid ? "phys disk" : "target",
704 	    id, period ? "Sync" : "Async", period,
705 	    (pg0.neg_params2 & MPI_CFG_SPI_DEV_0_NEGPARAMS_WIDTH_WIDE) ? 16 : 8,
706 	    pg0.neg_offset,
707 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_QAS) ? 1 : 0,
708 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_DUALXFERS) ? 1 : 0,
709 	    (pg0.neg_params1 & MPI_CFG_SPI_DEV_0_NEGPARAMS_PACKETIZED) ? 1 : 0);
710 
711 	return (0);
712 }
713 
714 int
715 mpi_inq(struct mpi_softc *sc, u_int16_t target, int physdisk)
716 {
717 	struct mpi_ccb			*ccb;
718 	struct scsi_inquiry		inq;
719 	struct {
720 		struct mpi_msg_scsi_io		io;
721 		struct mpi_sge			sge;
722 		struct scsi_inquiry_data	inqbuf;
723 		struct scsi_sense_data		sense;
724 	} __packed			*bundle;
725 	struct mpi_msg_scsi_io		*io;
726 	struct mpi_sge			*sge;
727 	u_int64_t			addr;
728 
729 	DNPRINTF(MPI_D_PPR, "%s: mpi_inq\n", DEVNAME(sc));
730 
731 	bzero(&inq, sizeof(inq));
732 	inq.opcode = INQUIRY;
733 	_lto2b(sizeof(struct scsi_inquiry_data), inq.length);
734 
735 	ccb = mpi_get_ccb(sc);
736 	if (ccb == NULL)
737 		return (1);
738 
739 	ccb->ccb_done = mpi_empty_done;
740 
741 	bundle = ccb->ccb_cmd;
742 	io = &bundle->io;
743 	sge = &bundle->sge;
744 
745 	io->function = physdisk ? MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH :
746 	    MPI_FUNCTION_SCSI_IO_REQUEST;
747 	/*
748 	 * bus is always 0
749 	 * io->bus = htole16(sc->sc_bus);
750 	 */
751 	io->target_id = target;
752 
753 	io->cdb_length = sizeof(inq);
754 	io->sense_buf_len = sizeof(struct scsi_sense_data);
755 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
756 
757 	io->msg_context = htole32(ccb->ccb_id);
758 
759 	/*
760 	 * always lun 0
761 	 * io->lun[0] = htobe16(link->lun);
762 	 */
763 
764 	io->direction = MPI_SCSIIO_DIR_READ;
765 	io->tagging = MPI_SCSIIO_ATTR_NO_DISCONNECT;
766 
767 	bcopy(&inq, io->cdb, sizeof(inq));
768 
769 	io->data_length = htole32(sizeof(struct scsi_inquiry_data));
770 
771 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
772 	    ((u_int8_t *)&bundle->sense - (u_int8_t *)bundle));
773 
774 	sge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64 |
775 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
776 	    (u_int32_t)sizeof(inq));
777 
778 	addr = ccb->ccb_cmd_dva +
779 	    ((u_int8_t *)&bundle->inqbuf - (u_int8_t *)bundle);
780 	sge->sg_hi_addr = htole32((u_int32_t)(addr >> 32));
781 	sge->sg_lo_addr = htole32((u_int32_t)addr);
782 
783 	if (mpi_poll(sc, ccb, 5000) != 0)
784 		return (1);
785 
786 	if (ccb->ccb_rcb != NULL)
787 		mpi_push_reply(sc, ccb->ccb_rcb);
788 
789 	mpi_put_ccb(sc, ccb);
790 
791 	return (0);
792 }
793 
794 void
795 mpi_fc_info(struct mpi_softc *sc)
796 {
797 	struct mpi_cfg_hdr		hdr;
798 	struct mpi_cfg_fc_port_pg0	pg;
799 
800 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_FC_PORT, 0, 0,
801 	    &hdr) != 0) {
802 		DNPRINTF(MPI_D_MISC, "%s: mpi_fc_print unable to fetch "
803 		    "FC port header 0\n", DEVNAME(sc));
804 		return;
805 	}
806 
807 	if (mpi_cfg_page(sc, 0, &hdr, 1, &pg, sizeof(pg)) != 0) {
808 		DNPRINTF(MPI_D_MISC, "%s: mpi_fc_print unable to fetch "
809 		    "FC port page 0\n",
810 		    DEVNAME(sc));
811 		return;
812 	}
813 
814 	sc->sc_link.port_wwn = letoh64(pg.wwpn);
815 	sc->sc_link.node_wwn = letoh64(pg.wwnn);
816 }
817 
818 void
819 mpi_detach(struct mpi_softc *sc)
820 {
821 
822 }
823 
824 int
825 mpi_intr(void *arg)
826 {
827 	struct mpi_softc		*sc = arg;
828 	u_int32_t			reg;
829 	int				rv = 0;
830 
831 	while ((reg = mpi_pop_reply(sc)) != 0xffffffff) {
832 		mpi_reply(sc, reg);
833 		rv = 1;
834 	}
835 
836 	return (rv);
837 }
838 
839 void
840 mpi_reply(struct mpi_softc *sc, u_int32_t reg)
841 {
842 	struct mpi_ccb			*ccb;
843 	struct mpi_rcb			*rcb = NULL;
844 	struct mpi_msg_reply		*reply = NULL;
845 	u_int32_t			reply_dva;
846 	int				id;
847 	int				i;
848 
849 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply reg: 0x%08x\n", DEVNAME(sc), reg);
850 
851 	if (reg & MPI_REPLY_QUEUE_ADDRESS) {
852 		reply_dva = (reg & MPI_REPLY_QUEUE_ADDRESS_MASK) << 1;
853 		i = (reply_dva - (u_int32_t)MPI_DMA_DVA(sc->sc_replies)) /
854 		    MPI_REPLY_SIZE;
855 		rcb = &sc->sc_rcbs[i];
856 
857 		bus_dmamap_sync(sc->sc_dmat,
858 		    MPI_DMA_MAP(sc->sc_replies), rcb->rcb_offset,
859 		    MPI_REPLY_SIZE, BUS_DMASYNC_POSTREAD);
860 
861 		reply = rcb->rcb_reply;
862 
863 		id = letoh32(reply->msg_context);
864 	} else {
865 		switch (reg & MPI_REPLY_QUEUE_TYPE_MASK) {
866 		case MPI_REPLY_QUEUE_TYPE_INIT:
867 			id = reg & MPI_REPLY_QUEUE_CONTEXT;
868 			break;
869 
870 		default:
871 			panic("%s: unsupported context reply\n",
872 			    DEVNAME(sc));
873 		}
874 	}
875 
876 	DNPRINTF(MPI_D_INTR, "%s: mpi_reply id: %d reply: %p\n",
877 	    DEVNAME(sc), id, reply);
878 
879 	ccb = &sc->sc_ccbs[id];
880 
881 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
882 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
883 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
884 	ccb->ccb_state = MPI_CCB_READY;
885 	ccb->ccb_rcb = rcb;
886 
887 	ccb->ccb_done(ccb);
888 }
889 
890 struct mpi_dmamem *
891 mpi_dmamem_alloc(struct mpi_softc *sc, size_t size)
892 {
893 	struct mpi_dmamem		*mdm;
894 	int				nsegs;
895 
896 	mdm = malloc(sizeof(struct mpi_dmamem), M_DEVBUF, M_NOWAIT | M_ZERO);
897 	if (mdm == NULL)
898 		return (NULL);
899 
900 	mdm->mdm_size = size;
901 
902 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
903 	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mdm->mdm_map) != 0)
904 		goto mdmfree;
905 
906 	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &mdm->mdm_seg,
907 	    1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
908 		goto destroy;
909 
910 	if (bus_dmamem_map(sc->sc_dmat, &mdm->mdm_seg, nsegs, size,
911 	    &mdm->mdm_kva, BUS_DMA_NOWAIT) != 0)
912 		goto free;
913 
914 	if (bus_dmamap_load(sc->sc_dmat, mdm->mdm_map, mdm->mdm_kva, size,
915 	    NULL, BUS_DMA_NOWAIT) != 0)
916 		goto unmap;
917 
918 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_alloc size: %d mdm: %#x "
919 	    "map: %#x nsegs: %d segs: %#x kva: %x\n",
920 	    DEVNAME(sc), size, mdm->mdm_map, nsegs, mdm->mdm_seg, mdm->mdm_kva);
921 
922 	return (mdm);
923 
924 unmap:
925 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, size);
926 free:
927 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
928 destroy:
929 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
930 mdmfree:
931 	free(mdm, M_DEVBUF);
932 
933 	return (NULL);
934 }
935 
936 void
937 mpi_dmamem_free(struct mpi_softc *sc, struct mpi_dmamem *mdm)
938 {
939 	DNPRINTF(MPI_D_MEM, "%s: mpi_dmamem_free %#x\n", DEVNAME(sc), mdm);
940 
941 	bus_dmamap_unload(sc->sc_dmat, mdm->mdm_map);
942 	bus_dmamem_unmap(sc->sc_dmat, mdm->mdm_kva, mdm->mdm_size);
943 	bus_dmamem_free(sc->sc_dmat, &mdm->mdm_seg, 1);
944 	bus_dmamap_destroy(sc->sc_dmat, mdm->mdm_map);
945 	free(mdm, M_DEVBUF);
946 }
947 
948 int
949 mpi_alloc_ccbs(struct mpi_softc *sc)
950 {
951 	struct mpi_ccb			*ccb;
952 	u_int8_t			*cmd;
953 	int				i;
954 
955 	SLIST_INIT(&sc->sc_ccb_free);
956 	mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
957 
958 	sc->sc_ccbs = malloc(sizeof(struct mpi_ccb) * sc->sc_maxcmds,
959 	    M_DEVBUF, M_WAITOK | M_CANFAIL | M_ZERO);
960 	if (sc->sc_ccbs == NULL) {
961 		printf("%s: unable to allocate ccbs\n", DEVNAME(sc));
962 		return (1);
963 	}
964 
965 	sc->sc_requests = mpi_dmamem_alloc(sc,
966 	    MPI_REQUEST_SIZE * sc->sc_maxcmds);
967 	if (sc->sc_requests == NULL) {
968 		printf("%s: unable to allocate ccb dmamem\n", DEVNAME(sc));
969 		goto free_ccbs;
970 	}
971 	cmd = MPI_DMA_KVA(sc->sc_requests);
972 	bzero(cmd, MPI_REQUEST_SIZE * sc->sc_maxcmds);
973 
974 	for (i = 0; i < sc->sc_maxcmds; i++) {
975 		ccb = &sc->sc_ccbs[i];
976 
977 		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
978 		    sc->sc_max_sgl_len, MAXPHYS, 0,
979 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
980 		    &ccb->ccb_dmamap) != 0) {
981 			printf("%s: unable to create dma map\n", DEVNAME(sc));
982 			goto free_maps;
983 		}
984 
985 		ccb->ccb_sc = sc;
986 		ccb->ccb_id = i;
987 		ccb->ccb_offset = MPI_REQUEST_SIZE * i;
988 		ccb->ccb_state = MPI_CCB_READY;
989 
990 		ccb->ccb_cmd = &cmd[ccb->ccb_offset];
991 		ccb->ccb_cmd_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_requests) +
992 		    ccb->ccb_offset;
993 
994 		DNPRINTF(MPI_D_CCB, "%s: mpi_alloc_ccbs(%d) ccb: %#x map: %#x "
995 		    "sc: %#x id: %#x offs: %#x cmd: %#x dva: %#x\n",
996 		    DEVNAME(sc), i, ccb, ccb->ccb_dmamap, ccb->ccb_sc,
997 		    ccb->ccb_id, ccb->ccb_offset, ccb->ccb_cmd,
998 		    ccb->ccb_cmd_dva);
999 
1000 		mpi_put_ccb(sc, ccb);
1001 	}
1002 
1003 	scsi_iopool_init(&sc->sc_iopool, sc,
1004 	    (void *(*)(void *))mpi_get_ccb,
1005 	    (void (*)(void *, void *))mpi_put_ccb);
1006 
1007 	return (0);
1008 
1009 free_maps:
1010 	while ((ccb = mpi_get_ccb(sc)) != NULL)
1011 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1012 
1013 	mpi_dmamem_free(sc, sc->sc_requests);
1014 free_ccbs:
1015 	free(sc->sc_ccbs, M_DEVBUF);
1016 
1017 	return (1);
1018 }
1019 
1020 struct mpi_ccb *
1021 mpi_get_ccb(struct mpi_softc *sc)
1022 {
1023 	struct mpi_ccb			*ccb;
1024 
1025 	mtx_enter(&sc->sc_ccb_mtx);
1026 	ccb = SLIST_FIRST(&sc->sc_ccb_free);
1027 	if (ccb != NULL) {
1028 		SLIST_REMOVE_HEAD(&sc->sc_ccb_free, ccb_link);
1029 		ccb->ccb_state = MPI_CCB_READY;
1030 	}
1031 	mtx_leave(&sc->sc_ccb_mtx);
1032 
1033 	DNPRINTF(MPI_D_CCB, "%s: mpi_get_ccb %p\n", DEVNAME(sc), ccb);
1034 
1035 	return (ccb);
1036 }
1037 
1038 void
1039 mpi_put_ccb(struct mpi_softc *sc, struct mpi_ccb *ccb)
1040 {
1041 	DNPRINTF(MPI_D_CCB, "%s: mpi_put_ccb %p\n", DEVNAME(sc), ccb);
1042 
1043 #ifdef DIAGNOSTIC
1044 	if (ccb->ccb_state == MPI_CCB_FREE)
1045 		panic("mpi_put_ccb: double free");
1046 #endif
1047 
1048 	ccb->ccb_state = MPI_CCB_FREE;
1049 	ccb->ccb_cookie = NULL;
1050 	ccb->ccb_done = NULL;
1051 	bzero(ccb->ccb_cmd, MPI_REQUEST_SIZE);
1052 	mtx_enter(&sc->sc_ccb_mtx);
1053 	SLIST_INSERT_HEAD(&sc->sc_ccb_free, ccb, ccb_link);
1054 	mtx_leave(&sc->sc_ccb_mtx);
1055 }
1056 
1057 int
1058 mpi_alloc_replies(struct mpi_softc *sc)
1059 {
1060 	DNPRINTF(MPI_D_MISC, "%s: mpi_alloc_replies\n", DEVNAME(sc));
1061 
1062 	sc->sc_rcbs = malloc(sc->sc_repq * sizeof(struct mpi_rcb), M_DEVBUF,
1063 	    M_WAITOK|M_CANFAIL);
1064 	if (sc->sc_rcbs == NULL)
1065 		return (1);
1066 
1067 	sc->sc_replies = mpi_dmamem_alloc(sc, sc->sc_repq * MPI_REPLY_SIZE);
1068 	if (sc->sc_replies == NULL) {
1069 		free(sc->sc_rcbs, M_DEVBUF);
1070 		return (1);
1071 	}
1072 
1073 	return (0);
1074 }
1075 
1076 void
1077 mpi_push_reply(struct mpi_softc *sc, struct mpi_rcb *rcb)
1078 {
1079 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies),
1080 	    rcb->rcb_offset, MPI_REPLY_SIZE, BUS_DMASYNC_PREREAD);
1081 	mpi_push_reply_db(sc, rcb->rcb_reply_dva);
1082 }
1083 
1084 void
1085 mpi_push_replies(struct mpi_softc *sc)
1086 {
1087 	struct mpi_rcb			*rcb;
1088 	char				*kva = MPI_DMA_KVA(sc->sc_replies);
1089 	int				i;
1090 
1091 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_replies), 0,
1092 	    sc->sc_repq * MPI_REPLY_SIZE, BUS_DMASYNC_PREREAD);
1093 
1094 	for (i = 0; i < sc->sc_repq; i++) {
1095 		rcb = &sc->sc_rcbs[i];
1096 
1097 		rcb->rcb_reply = kva + MPI_REPLY_SIZE * i;
1098 		rcb->rcb_offset = MPI_REPLY_SIZE * i;
1099 		rcb->rcb_reply_dva = (u_int32_t)MPI_DMA_DVA(sc->sc_replies) +
1100 		    MPI_REPLY_SIZE * i;
1101 		mpi_push_reply_db(sc, rcb->rcb_reply_dva);
1102 	}
1103 }
1104 
1105 void
1106 mpi_start(struct mpi_softc *sc, struct mpi_ccb *ccb)
1107 {
1108 	DNPRINTF(MPI_D_RW, "%s: mpi_start %#x\n", DEVNAME(sc),
1109 	    ccb->ccb_cmd_dva);
1110 
1111 	bus_dmamap_sync(sc->sc_dmat, MPI_DMA_MAP(sc->sc_requests),
1112 	    ccb->ccb_offset, MPI_REQUEST_SIZE,
1113 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1114 
1115 	ccb->ccb_state = MPI_CCB_QUEUED;
1116 	mpi_write(sc, MPI_REQ_QUEUE, ccb->ccb_cmd_dva);
1117 }
1118 
1119 int
1120 mpi_poll(struct mpi_softc *sc, struct mpi_ccb *ccb, int timeout)
1121 {
1122 	void				(*done)(struct mpi_ccb *);
1123 	void				*cookie;
1124 	int				rv = 1;
1125 	u_int32_t			reg;
1126 
1127 	DNPRINTF(MPI_D_INTR, "%s: mpi_poll timeout %d\n", DEVNAME(sc),
1128 	    timeout);
1129 
1130 	done = ccb->ccb_done;
1131 	cookie = ccb->ccb_cookie;
1132 
1133 	ccb->ccb_done = mpi_poll_done;
1134 	ccb->ccb_cookie = &rv;
1135 
1136 	mpi_start(sc, ccb);
1137 	while (rv == 1) {
1138 		reg = mpi_pop_reply(sc);
1139 		if (reg == 0xffffffff) {
1140 			if (timeout-- == 0) {
1141 				printf("%s: timeout\n", DEVNAME(sc));
1142 				goto timeout;
1143 			}
1144 
1145 			delay(1000);
1146 			continue;
1147 		}
1148 
1149 		mpi_reply(sc, reg);
1150 	}
1151 
1152 	ccb->ccb_cookie = cookie;
1153 	done(ccb);
1154 
1155 timeout:
1156 	return (rv);
1157 }
1158 
1159 void
1160 mpi_poll_done(struct mpi_ccb *ccb)
1161 {
1162 	int				*rv = ccb->ccb_cookie;
1163 
1164 	*rv = 0;
1165 }
1166 
1167 void
1168 mpi_wait(struct mpi_softc *sc, struct mpi_ccb *ccb)
1169 {
1170 	struct mutex			cookie = MUTEX_INITIALIZER(IPL_BIO);
1171 	void				(*done)(struct mpi_ccb *);
1172 
1173 	done = ccb->ccb_done;
1174 	ccb->ccb_done = mpi_wait_done;
1175 	ccb->ccb_cookie = &cookie;
1176 
1177 	/* XXX this will wait forever for the ccb to complete */
1178 
1179 	mpi_start(sc, ccb);
1180 
1181 	mtx_enter(&cookie);
1182 	while (ccb->ccb_cookie != NULL)
1183 		msleep(ccb, &cookie, PRIBIO, "mpiwait", 0);
1184 	mtx_leave(&cookie);
1185 
1186 	done(ccb);
1187 }
1188 
1189 void
1190 mpi_wait_done(struct mpi_ccb *ccb)
1191 {
1192 	struct mutex			*cookie = ccb->ccb_cookie;
1193 
1194 	mtx_enter(cookie);
1195 	ccb->ccb_cookie = NULL;
1196 	wakeup_one(ccb);
1197 	mtx_leave(cookie);
1198 }
1199 
1200 void
1201 mpi_scsi_cmd(struct scsi_xfer *xs)
1202 {
1203 	struct scsi_link		*link = xs->sc_link;
1204 	struct mpi_softc		*sc = link->adapter_softc;
1205 	struct mpi_ccb			*ccb;
1206 	struct mpi_ccb_bundle		*mcb;
1207 	struct mpi_msg_scsi_io		*io;
1208 
1209 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd\n", DEVNAME(sc));
1210 
1211 	if (xs->cmdlen > MPI_CDB_LEN) {
1212 		DNPRINTF(MPI_D_CMD, "%s: CBD too big %d\n",
1213 		    DEVNAME(sc), xs->cmdlen);
1214 		bzero(&xs->sense, sizeof(xs->sense));
1215 		xs->sense.error_code = SSD_ERRCODE_VALID | 0x70;
1216 		xs->sense.flags = SKEY_ILLEGAL_REQUEST;
1217 		xs->sense.add_sense_code = 0x20;
1218 		xs->error = XS_SENSE;
1219 		scsi_done(xs);
1220 		return;
1221 	}
1222 
1223 	ccb = xs->io;
1224 
1225 	DNPRINTF(MPI_D_CMD, "%s: ccb_id: %d xs->flags: 0x%x\n",
1226 	    DEVNAME(sc), ccb->ccb_id, xs->flags);
1227 
1228 	ccb->ccb_cookie = xs;
1229 	ccb->ccb_done = mpi_scsi_cmd_done;
1230 
1231 	mcb = ccb->ccb_cmd;
1232 	io = &mcb->mcb_io;
1233 
1234 	io->function = MPI_FUNCTION_SCSI_IO_REQUEST;
1235 	/*
1236 	 * bus is always 0
1237 	 * io->bus = htole16(sc->sc_bus);
1238 	 */
1239 	io->target_id = link->target;
1240 
1241 	io->cdb_length = xs->cmdlen;
1242 	io->sense_buf_len = sizeof(xs->sense);
1243 	io->msg_flags = MPI_SCSIIO_SENSE_BUF_ADDR_WIDTH_64;
1244 
1245 	io->msg_context = htole32(ccb->ccb_id);
1246 
1247 	io->lun[0] = htobe16(link->lun);
1248 
1249 	switch (xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) {
1250 	case SCSI_DATA_IN:
1251 		io->direction = MPI_SCSIIO_DIR_READ;
1252 		break;
1253 	case SCSI_DATA_OUT:
1254 		io->direction = MPI_SCSIIO_DIR_WRITE;
1255 		break;
1256 	default:
1257 		io->direction = MPI_SCSIIO_DIR_NONE;
1258 		break;
1259 	}
1260 
1261 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SCSI &&
1262 	    (link->quirks & SDEV_NOTAGS))
1263 		io->tagging = MPI_SCSIIO_ATTR_UNTAGGED;
1264 	else
1265 		io->tagging = MPI_SCSIIO_ATTR_SIMPLE_Q;
1266 
1267 	bcopy(xs->cmd, io->cdb, xs->cmdlen);
1268 
1269 	io->data_length = htole32(xs->datalen);
1270 
1271 	io->sense_buf_low_addr = htole32(ccb->ccb_cmd_dva +
1272 	    ((u_int8_t *)&mcb->mcb_sense - (u_int8_t *)mcb));
1273 
1274 	if (mpi_load_xs(ccb) != 0) {
1275 		xs->error = XS_DRIVER_STUFFUP;
1276 		scsi_done(xs);
1277 		return;
1278 	}
1279 
1280 	timeout_set(&xs->stimeout, mpi_timeout_xs, ccb);
1281 
1282 	if (xs->flags & SCSI_POLL) {
1283 		if (mpi_poll(sc, ccb, xs->timeout) != 0) {
1284 			xs->error = XS_DRIVER_STUFFUP;
1285 			scsi_done(xs);
1286 		}
1287 		return;
1288 	}
1289 
1290 	mpi_start(sc, ccb);
1291 }
1292 
1293 void
1294 mpi_scsi_cmd_done(struct mpi_ccb *ccb)
1295 {
1296 	struct mpi_softc		*sc = ccb->ccb_sc;
1297 	struct scsi_xfer		*xs = ccb->ccb_cookie;
1298 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1299 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1300 	struct mpi_msg_scsi_io_error	*sie;
1301 
1302 	if (xs->datalen != 0) {
1303 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1304 		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1305 		    BUS_DMASYNC_POSTWRITE);
1306 
1307 		bus_dmamap_unload(sc->sc_dmat, dmap);
1308 	}
1309 
1310 	/* timeout_del */
1311 	xs->error = XS_NOERROR;
1312 	xs->resid = 0;
1313 
1314 	if (ccb->ccb_rcb == NULL) {
1315 		/* no scsi error, we're ok so drop out early */
1316 		xs->status = SCSI_OK;
1317 		scsi_done(xs);
1318 		return;
1319 	}
1320 
1321 	sie = ccb->ccb_rcb->rcb_reply;
1322 
1323 	DNPRINTF(MPI_D_CMD, "%s: mpi_scsi_cmd_done xs cmd: 0x%02x len: %d "
1324 	    "flags 0x%x\n", DEVNAME(sc), xs->cmd->opcode, xs->datalen,
1325 	    xs->flags);
1326 	DNPRINTF(MPI_D_CMD, "%s:  target_id: %d bus: %d msg_length: %d "
1327 	    "function: 0x%02x\n", DEVNAME(sc), sie->target_id, sie->bus,
1328 	    sie->msg_length, sie->function);
1329 	DNPRINTF(MPI_D_CMD, "%s:  cdb_length: %d sense_buf_length: %d "
1330 	    "msg_flags: 0x%02x\n", DEVNAME(sc), sie->cdb_length,
1331 	    sie->sense_buf_len, sie->msg_flags);
1332 	DNPRINTF(MPI_D_CMD, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
1333 	    letoh32(sie->msg_context));
1334 	DNPRINTF(MPI_D_CMD, "%s:  scsi_status: 0x%02x scsi_state: 0x%02x "
1335 	    "ioc_status: 0x%04x\n", DEVNAME(sc), sie->scsi_status,
1336 	    sie->scsi_state, letoh16(sie->ioc_status));
1337 	DNPRINTF(MPI_D_CMD, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
1338 	    letoh32(sie->ioc_loginfo));
1339 	DNPRINTF(MPI_D_CMD, "%s:  transfer_count: %d\n", DEVNAME(sc),
1340 	    letoh32(sie->transfer_count));
1341 	DNPRINTF(MPI_D_CMD, "%s:  sense_count: %d\n", DEVNAME(sc),
1342 	    letoh32(sie->sense_count));
1343 	DNPRINTF(MPI_D_CMD, "%s:  response_info: 0x%08x\n", DEVNAME(sc),
1344 	    letoh32(sie->response_info));
1345 	DNPRINTF(MPI_D_CMD, "%s:  tag: 0x%04x\n", DEVNAME(sc),
1346 	    letoh16(sie->tag));
1347 
1348 	xs->status = sie->scsi_status;
1349 	switch (letoh16(sie->ioc_status)) {
1350 	case MPI_IOCSTATUS_SCSI_DATA_UNDERRUN:
1351 		xs->resid = xs->datalen - letoh32(sie->transfer_count);
1352 		if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_NO_SCSI_STATUS) {
1353 			xs->error = XS_DRIVER_STUFFUP;
1354 			break;
1355 		}
1356 		/* FALLTHROUGH */
1357 	case MPI_IOCSTATUS_SUCCESS:
1358 	case MPI_IOCSTATUS_SCSI_RECOVERED_ERROR:
1359 		switch (xs->status) {
1360 		case SCSI_OK:
1361 			xs->resid = 0;
1362 			break;
1363 
1364 		case SCSI_CHECK:
1365 			xs->error = XS_SENSE;
1366 			break;
1367 
1368 		case SCSI_BUSY:
1369 		case SCSI_QUEUE_FULL:
1370 			xs->error = XS_BUSY;
1371 			break;
1372 
1373 		default:
1374 			xs->error = XS_DRIVER_STUFFUP;
1375 			break;
1376 		}
1377 		break;
1378 
1379 	case MPI_IOCSTATUS_BUSY:
1380 	case MPI_IOCSTATUS_INSUFFICIENT_RESOURCES:
1381 		xs->error = XS_BUSY;
1382 		break;
1383 
1384 	case MPI_IOCSTATUS_SCSI_INVALID_BUS:
1385 	case MPI_IOCSTATUS_SCSI_INVALID_TARGETID:
1386 	case MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1387 		xs->error = XS_SELTIMEOUT;
1388 		break;
1389 
1390 	default:
1391 		xs->error = XS_DRIVER_STUFFUP;
1392 		break;
1393 	}
1394 
1395 	if (sie->scsi_state & MPI_SCSIIO_ERR_STATE_AUTOSENSE_VALID)
1396 		bcopy(&mcb->mcb_sense, &xs->sense, sizeof(xs->sense));
1397 
1398 	DNPRINTF(MPI_D_CMD, "%s:  xs err: 0x%02x status: %d\n", DEVNAME(sc),
1399 	    xs->error, xs->status);
1400 
1401 	mpi_push_reply(sc, ccb->ccb_rcb);
1402 	scsi_done(xs);
1403 }
1404 
1405 void
1406 mpi_timeout_xs(void *arg)
1407 {
1408 	/* XXX */
1409 }
1410 
1411 int
1412 mpi_load_xs(struct mpi_ccb *ccb)
1413 {
1414 	struct mpi_softc		*sc = ccb->ccb_sc;
1415 	struct scsi_xfer		*xs = ccb->ccb_cookie;
1416 	struct mpi_ccb_bundle		*mcb = ccb->ccb_cmd;
1417 	struct mpi_msg_scsi_io		*io = &mcb->mcb_io;
1418 	struct mpi_sge			*sge, *nsge = &mcb->mcb_sgl[0];
1419 	struct mpi_sge			*ce = NULL, *nce;
1420 	u_int64_t			ce_dva;
1421 	bus_dmamap_t			dmap = ccb->ccb_dmamap;
1422 	u_int32_t			addr, flags;
1423 	int				i, error;
1424 
1425 	if (xs->datalen == 0) {
1426 		nsge->sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
1427 		    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
1428 		return (0);
1429 	}
1430 
1431 	error = bus_dmamap_load(sc->sc_dmat, dmap,
1432 	    xs->data, xs->datalen, NULL,
1433 	    (xs->flags & SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1434 	if (error) {
1435 		printf("%s: error %d loading dmamap\n", DEVNAME(sc), error);
1436 		return (1);
1437 	}
1438 
1439 	flags = MPI_SGE_FL_TYPE_SIMPLE | MPI_SGE_FL_SIZE_64;
1440 	if (xs->flags & SCSI_DATA_OUT)
1441 		flags |= MPI_SGE_FL_DIR_OUT;
1442 
1443 	if (dmap->dm_nsegs > sc->sc_first_sgl_len) {
1444 		ce = &mcb->mcb_sgl[sc->sc_first_sgl_len - 1];
1445 		io->chain_offset = ((u_int8_t *)ce - (u_int8_t *)io) / 4;
1446 	}
1447 
1448 	for (i = 0; i < dmap->dm_nsegs; i++) {
1449 
1450 		if (nsge == ce) {
1451 			nsge++;
1452 			sge->sg_hdr |= htole32(MPI_SGE_FL_LAST);
1453 
1454 			DNPRINTF(MPI_D_DMA, "%s:   - 0x%08x 0x%08x 0x%08x\n",
1455 			    DEVNAME(sc), sge->sg_hdr,
1456 			    sge->sg_hi_addr, sge->sg_lo_addr);
1457 
1458 			if ((dmap->dm_nsegs - i) > sc->sc_chain_len) {
1459 				nce = &nsge[sc->sc_chain_len - 1];
1460 				addr = ((u_int8_t *)nce - (u_int8_t *)nsge) / 4;
1461 				addr = addr << 16 |
1462 				    sizeof(struct mpi_sge) * sc->sc_chain_len;
1463 			} else {
1464 				nce = NULL;
1465 				addr = sizeof(struct mpi_sge) *
1466 				    (dmap->dm_nsegs - i);
1467 			}
1468 
1469 			ce->sg_hdr = htole32(MPI_SGE_FL_TYPE_CHAIN |
1470 			    MPI_SGE_FL_SIZE_64 | addr);
1471 
1472 			ce_dva = ccb->ccb_cmd_dva +
1473 			    ((u_int8_t *)nsge - (u_int8_t *)mcb);
1474 
1475 			addr = (u_int32_t)(ce_dva >> 32);
1476 			ce->sg_hi_addr = htole32(addr);
1477 			addr = (u_int32_t)ce_dva;
1478 			ce->sg_lo_addr = htole32(addr);
1479 
1480 			DNPRINTF(MPI_D_DMA, "%s:  ce: 0x%08x 0x%08x 0x%08x\n",
1481 			    DEVNAME(sc), ce->sg_hdr, ce->sg_hi_addr,
1482 			    ce->sg_lo_addr);
1483 
1484 			ce = nce;
1485 		}
1486 
1487 		DNPRINTF(MPI_D_DMA, "%s:  %d: %d 0x%016llx\n", DEVNAME(sc),
1488 		    i, dmap->dm_segs[i].ds_len,
1489 		    (u_int64_t)dmap->dm_segs[i].ds_addr);
1490 
1491 		sge = nsge;
1492 
1493 		sge->sg_hdr = htole32(flags | dmap->dm_segs[i].ds_len);
1494 		addr = (u_int32_t)((u_int64_t)dmap->dm_segs[i].ds_addr >> 32);
1495 		sge->sg_hi_addr = htole32(addr);
1496 		addr = (u_int32_t)dmap->dm_segs[i].ds_addr;
1497 		sge->sg_lo_addr = htole32(addr);
1498 
1499 		DNPRINTF(MPI_D_DMA, "%s:  %d: 0x%08x 0x%08x 0x%08x\n",
1500 		    DEVNAME(sc), i, sge->sg_hdr, sge->sg_hi_addr,
1501 		    sge->sg_lo_addr);
1502 
1503 		nsge = sge + 1;
1504 	}
1505 
1506 	/* terminate list */
1507 	sge->sg_hdr |= htole32(MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
1508 	    MPI_SGE_FL_EOL);
1509 
1510 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1511 	    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
1512 	    BUS_DMASYNC_PREWRITE);
1513 
1514 	return (0);
1515 }
1516 
1517 void
1518 mpi_minphys(struct buf *bp, struct scsi_link *sl)
1519 {
1520 	/* XXX */
1521 	if (bp->b_bcount > MAXPHYS)
1522 		bp->b_bcount = MAXPHYS;
1523 	minphys(bp);
1524 }
1525 
1526 int
1527 mpi_scsi_probe(struct scsi_link *link)
1528 {
1529 	struct mpi_softc		*sc = link->adapter_softc;
1530 	struct mpi_ecfg_hdr		ehdr;
1531 	struct mpi_cfg_sas_dev_pg0	pg0;
1532 	u_int32_t			address;
1533 
1534 	if (sc->sc_porttype != MPI_PORTFACTS_PORTTYPE_SAS)
1535 		return (0);
1536 
1537 	address = MPI_CFG_SAS_DEV_ADDR_BUS | link->target;
1538 
1539 	if (mpi_ecfg_header(sc, MPI_CONFIG_REQ_EXTPAGE_TYPE_SAS_DEVICE, 0,
1540 	    address, &ehdr) != 0)
1541 		return (EIO);
1542 
1543 	if (mpi_ecfg_page(sc, address, &ehdr, 1, &pg0, sizeof(pg0)) != 0)
1544 		return (0);
1545 
1546 	DNPRINTF(MPI_D_MISC, "%s: mpi_scsi_probe sas dev pg 0 for target %d:\n",
1547 	    DEVNAME(sc), link->target);
1548 	DNPRINTF(MPI_D_MISC, "%s:  slot: 0x%04x enc_handle: 0x%04x\n",
1549 	    DEVNAME(sc), letoh16(pg0.slot), letoh16(pg0.enc_handle));
1550 	DNPRINTF(MPI_D_MISC, "%s:  sas_addr: 0x%016llx\n", DEVNAME(sc),
1551 	    letoh64(pg0.sas_addr));
1552 	DNPRINTF(MPI_D_MISC, "%s:  parent_dev_handle: 0x%04x phy_num: 0x%02x "
1553 	    "access_status: 0x%02x\n", DEVNAME(sc),
1554 	    letoh16(pg0.parent_dev_handle), pg0.phy_num, pg0.access_status);
1555 	DNPRINTF(MPI_D_MISC, "%s:  dev_handle: 0x%04x "
1556 	    "bus: 0x%02x target: 0x%02x\n", DEVNAME(sc),
1557 	    letoh16(pg0.dev_handle), pg0.bus, pg0.target);
1558 	DNPRINTF(MPI_D_MISC, "%s:  device_info: 0x%08x\n", DEVNAME(sc),
1559 	    letoh32(pg0.device_info));
1560 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%04x physical_port: 0x%02x\n",
1561 	    DEVNAME(sc), letoh16(pg0.flags), pg0.physical_port);
1562 
1563 	if (ISSET(letoh32(pg0.device_info),
1564 	    MPI_CFG_SAS_DEV_0_DEVINFO_ATAPI_DEVICE)) {
1565 		DNPRINTF(MPI_D_MISC, "%s: target %d is an ATAPI device\n",
1566 		    DEVNAME(sc), link->target);
1567 		link->flags |= SDEV_ATAPI;
1568 		link->quirks |= SDEV_ONLYBIG;
1569 	}
1570 
1571 	return (0);
1572 }
1573 
1574 u_int32_t
1575 mpi_read(struct mpi_softc *sc, bus_size_t r)
1576 {
1577 	u_int32_t			rv;
1578 
1579 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1580 	    BUS_SPACE_BARRIER_READ);
1581 	rv = bus_space_read_4(sc->sc_iot, sc->sc_ioh, r);
1582 
1583 	DNPRINTF(MPI_D_RW, "%s: mpi_read %#x %#x\n", DEVNAME(sc), r, rv);
1584 
1585 	return (rv);
1586 }
1587 
1588 void
1589 mpi_write(struct mpi_softc *sc, bus_size_t r, u_int32_t v)
1590 {
1591 	DNPRINTF(MPI_D_RW, "%s: mpi_write %#x %#x\n", DEVNAME(sc), r, v);
1592 
1593 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
1594 	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
1595 	    BUS_SPACE_BARRIER_WRITE);
1596 }
1597 
1598 int
1599 mpi_wait_eq(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1600     u_int32_t target)
1601 {
1602 	int				i;
1603 
1604 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_eq %#x %#x %#x\n", DEVNAME(sc), r,
1605 	    mask, target);
1606 
1607 	for (i = 0; i < 10000; i++) {
1608 		if ((mpi_read(sc, r) & mask) == target)
1609 			return (0);
1610 		delay(1000);
1611 	}
1612 
1613 	return (1);
1614 }
1615 
1616 int
1617 mpi_wait_ne(struct mpi_softc *sc, bus_size_t r, u_int32_t mask,
1618     u_int32_t target)
1619 {
1620 	int				i;
1621 
1622 	DNPRINTF(MPI_D_RW, "%s: mpi_wait_ne %#x %#x %#x\n", DEVNAME(sc), r,
1623 	    mask, target);
1624 
1625 	for (i = 0; i < 10000; i++) {
1626 		if ((mpi_read(sc, r) & mask) != target)
1627 			return (0);
1628 		delay(1000);
1629 	}
1630 
1631 	return (1);
1632 }
1633 
1634 int
1635 mpi_init(struct mpi_softc *sc)
1636 {
1637 	u_int32_t			db;
1638 	int				i;
1639 
1640 	/* spin until the IOC leaves the RESET state */
1641 	if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1642 	    MPI_DOORBELL_STATE_RESET) != 0) {
1643 		DNPRINTF(MPI_D_MISC, "%s: mpi_init timeout waiting to leave "
1644 		    "reset state\n", DEVNAME(sc));
1645 		return (1);
1646 	}
1647 
1648 	/* check current ownership */
1649 	db = mpi_read_db(sc);
1650 	if ((db & MPI_DOORBELL_WHOINIT) == MPI_DOORBELL_WHOINIT_PCIPEER) {
1651 		DNPRINTF(MPI_D_MISC, "%s: mpi_init initialised by pci peer\n",
1652 		    DEVNAME(sc));
1653 		return (0);
1654 	}
1655 
1656 	for (i = 0; i < 5; i++) {
1657 		switch (db & MPI_DOORBELL_STATE) {
1658 		case MPI_DOORBELL_STATE_READY:
1659 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is ready\n",
1660 			    DEVNAME(sc));
1661 			return (0);
1662 
1663 		case MPI_DOORBELL_STATE_OPER:
1664 		case MPI_DOORBELL_STATE_FAULT:
1665 			DNPRINTF(MPI_D_MISC, "%s: mpi_init ioc is being "
1666 			    "reset\n" , DEVNAME(sc));
1667 			if (mpi_reset_soft(sc) != 0)
1668 				mpi_reset_hard(sc);
1669 			break;
1670 
1671 		case MPI_DOORBELL_STATE_RESET:
1672 			DNPRINTF(MPI_D_MISC, "%s: mpi_init waiting to come "
1673 			    "out of reset\n", DEVNAME(sc));
1674 			if (mpi_wait_ne(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1675 			    MPI_DOORBELL_STATE_RESET) != 0)
1676 				return (1);
1677 			break;
1678 		}
1679 		db = mpi_read_db(sc);
1680 	}
1681 
1682 	return (1);
1683 }
1684 
1685 int
1686 mpi_reset_soft(struct mpi_softc *sc)
1687 {
1688 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_soft\n", DEVNAME(sc));
1689 
1690 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1691 		return (1);
1692 
1693 	mpi_write_db(sc,
1694 	    MPI_DOORBELL_FUNCTION(MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET));
1695 	if (mpi_wait_eq(sc, MPI_INTR_STATUS,
1696 	    MPI_INTR_STATUS_IOCDOORBELL, 0) != 0)
1697 		return (1);
1698 
1699 	if (mpi_wait_eq(sc, MPI_DOORBELL, MPI_DOORBELL_STATE,
1700 	    MPI_DOORBELL_STATE_READY) != 0)
1701 		return (1);
1702 
1703 	return (0);
1704 }
1705 
1706 int
1707 mpi_reset_hard(struct mpi_softc *sc)
1708 {
1709 	DNPRINTF(MPI_D_MISC, "%s: mpi_reset_hard\n", DEVNAME(sc));
1710 
1711 	/* enable diagnostic register */
1712 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1713 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_1);
1714 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_2);
1715 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_3);
1716 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_4);
1717 	mpi_write(sc, MPI_WRITESEQ, MPI_WRITESEQ_5);
1718 
1719 	/* reset ioc */
1720 	mpi_write(sc, MPI_HOSTDIAG, MPI_HOSTDIAG_RESET_ADAPTER);
1721 
1722 	delay(10000);
1723 
1724 	/* disable diagnostic register */
1725 	mpi_write(sc, MPI_WRITESEQ, 0xff);
1726 
1727 	/* restore pci bits? */
1728 
1729 	/* firmware bits? */
1730 	return (0);
1731 }
1732 
1733 int
1734 mpi_handshake_send(struct mpi_softc *sc, void *buf, size_t dwords)
1735 {
1736 	u_int32_t				*query = buf;
1737 	int					i;
1738 
1739 	/* make sure the doorbell is not in use. */
1740 	if (mpi_read_db(sc) & MPI_DOORBELL_INUSE)
1741 		return (1);
1742 
1743 	/* clear pending doorbell interrupts */
1744 	if (mpi_read_intr(sc) & MPI_INTR_STATUS_DOORBELL)
1745 		mpi_write_intr(sc, 0);
1746 
1747 	/*
1748 	 * first write the doorbell with the handshake function and the
1749 	 * dword count.
1750 	 */
1751 	mpi_write_db(sc, MPI_DOORBELL_FUNCTION(MPI_FUNCTION_HANDSHAKE) |
1752 	    MPI_DOORBELL_DWORDS(dwords));
1753 
1754 	/*
1755 	 * the doorbell used bit will be set because a doorbell function has
1756 	 * started. Wait for the interrupt and then ack it.
1757 	 */
1758 	if (mpi_wait_db_int(sc) != 0)
1759 		return (1);
1760 	mpi_write_intr(sc, 0);
1761 
1762 	/* poll for the acknowledgement. */
1763 	if (mpi_wait_db_ack(sc) != 0)
1764 		return (1);
1765 
1766 	/* write the query through the doorbell. */
1767 	for (i = 0; i < dwords; i++) {
1768 		mpi_write_db(sc, htole32(query[i]));
1769 		if (mpi_wait_db_ack(sc) != 0)
1770 			return (1);
1771 	}
1772 
1773 	return (0);
1774 }
1775 
1776 int
1777 mpi_handshake_recv_dword(struct mpi_softc *sc, u_int32_t *dword)
1778 {
1779 	u_int16_t				*words = (u_int16_t *)dword;
1780 	int					i;
1781 
1782 	for (i = 0; i < 2; i++) {
1783 		if (mpi_wait_db_int(sc) != 0)
1784 			return (1);
1785 		words[i] = letoh16(mpi_read_db(sc) & MPI_DOORBELL_DATA_MASK);
1786 		mpi_write_intr(sc, 0);
1787 	}
1788 
1789 	return (0);
1790 }
1791 
1792 int
1793 mpi_handshake_recv(struct mpi_softc *sc, void *buf, size_t dwords)
1794 {
1795 	struct mpi_msg_reply			*reply = buf;
1796 	u_int32_t				*dbuf = buf, dummy;
1797 	int					i;
1798 
1799 	/* get the first dword so we can read the length out of the header. */
1800 	if (mpi_handshake_recv_dword(sc, &dbuf[0]) != 0)
1801 		return (1);
1802 
1803 	DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dwords: %d reply: %d\n",
1804 	    DEVNAME(sc), dwords, reply->msg_length);
1805 
1806 	/*
1807 	 * the total length, in dwords, is in the message length field of the
1808 	 * reply header.
1809 	 */
1810 	for (i = 1; i < MIN(dwords, reply->msg_length); i++) {
1811 		if (mpi_handshake_recv_dword(sc, &dbuf[i]) != 0)
1812 			return (1);
1813 	}
1814 
1815 	/* if there's extra stuff to come off the ioc, discard it */
1816 	while (i++ < reply->msg_length) {
1817 		if (mpi_handshake_recv_dword(sc, &dummy) != 0)
1818 			return (1);
1819 		DNPRINTF(MPI_D_CMD, "%s: mpi_handshake_recv dummy read: "
1820 		    "0x%08x\n", DEVNAME(sc), dummy);
1821 	}
1822 
1823 	/* wait for the doorbell used bit to be reset and clear the intr */
1824 	if (mpi_wait_db_int(sc) != 0)
1825 		return (1);
1826 	mpi_write_intr(sc, 0);
1827 
1828 	return (0);
1829 }
1830 
1831 void
1832 mpi_empty_done(struct mpi_ccb *ccb)
1833 {
1834 	/* nothing to do */
1835 }
1836 
1837 int
1838 mpi_iocfacts(struct mpi_softc *sc)
1839 {
1840 	struct mpi_msg_iocfacts_request		ifq;
1841 	struct mpi_msg_iocfacts_reply		ifp;
1842 
1843 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts\n", DEVNAME(sc));
1844 
1845 	bzero(&ifq, sizeof(ifq));
1846 	bzero(&ifp, sizeof(ifp));
1847 
1848 	ifq.function = MPI_FUNCTION_IOC_FACTS;
1849 	ifq.chain_offset = 0;
1850 	ifq.msg_flags = 0;
1851 	ifq.msg_context = htole32(0xdeadbeef);
1852 
1853 	if (mpi_handshake_send(sc, &ifq, dwordsof(ifq)) != 0) {
1854 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts send failed\n",
1855 		    DEVNAME(sc));
1856 		return (1);
1857 	}
1858 
1859 	if (mpi_handshake_recv(sc, &ifp, dwordsof(ifp)) != 0) {
1860 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocfacts recv failed\n",
1861 		    DEVNAME(sc));
1862 		return (1);
1863 	}
1864 
1865 	DNPRINTF(MPI_D_MISC, "%s:  func: 0x%02x len: %d msgver: %d.%d\n",
1866 	    DEVNAME(sc), ifp.function, ifp.msg_length,
1867 	    ifp.msg_version_maj, ifp.msg_version_min);
1868 	DNPRINTF(MPI_D_MISC, "%s:  msgflags: 0x%02x iocnumber: 0x%02x "
1869 	    "hdrver: %d.%d\n", DEVNAME(sc), ifp.msg_flags,
1870 	    ifp.ioc_number, ifp.header_version_maj,
1871 	    ifp.header_version_min);
1872 	DNPRINTF(MPI_D_MISC, "%s:  message context: 0x%08x\n", DEVNAME(sc),
1873 	    letoh32(ifp.msg_context));
1874 	DNPRINTF(MPI_D_MISC, "%s:  iocstatus: 0x%04x ioexcept: 0x%04x\n",
1875 	    DEVNAME(sc), letoh16(ifp.ioc_status),
1876 	    letoh16(ifp.ioc_exceptions));
1877 	DNPRINTF(MPI_D_MISC, "%s:  iocloginfo: 0x%08x\n", DEVNAME(sc),
1878 	    letoh32(ifp.ioc_loginfo));
1879 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x%02x blocksize: %d whoinit: 0x%02x "
1880 	    "maxchdepth: %d\n", DEVNAME(sc), ifp.flags,
1881 	    ifp.block_size, ifp.whoinit, ifp.max_chain_depth);
1882 	DNPRINTF(MPI_D_MISC, "%s:  reqfrsize: %d replyqdepth: %d\n",
1883 	    DEVNAME(sc), letoh16(ifp.request_frame_size),
1884 	    letoh16(ifp.reply_queue_depth));
1885 	DNPRINTF(MPI_D_MISC, "%s:  productid: 0x%04x\n", DEVNAME(sc),
1886 	    letoh16(ifp.product_id));
1887 	DNPRINTF(MPI_D_MISC, "%s:  hostmfahiaddr: 0x%08x\n", DEVNAME(sc),
1888 	    letoh32(ifp.current_host_mfa_hi_addr));
1889 	DNPRINTF(MPI_D_MISC, "%s:  event_state: 0x%02x number_of_ports: %d "
1890 	    "global_credits: %d\n",
1891 	    DEVNAME(sc), ifp.event_state, ifp.number_of_ports,
1892 	    letoh16(ifp.global_credits));
1893 	DNPRINTF(MPI_D_MISC, "%s:  sensebufhiaddr: 0x%08x\n", DEVNAME(sc),
1894 	    letoh32(ifp.current_sense_buffer_hi_addr));
1895 	DNPRINTF(MPI_D_MISC, "%s:  maxbus: %d maxdev: %d replyfrsize: %d\n",
1896 	    DEVNAME(sc), ifp.max_buses, ifp.max_devices,
1897 	    letoh16(ifp.current_reply_frame_size));
1898 	DNPRINTF(MPI_D_MISC, "%s:  fw_image_size: %d\n", DEVNAME(sc),
1899 	    letoh32(ifp.fw_image_size));
1900 	DNPRINTF(MPI_D_MISC, "%s:  ioc_capabilities: 0x%08x\n", DEVNAME(sc),
1901 	    letoh32(ifp.ioc_capabilities));
1902 	DNPRINTF(MPI_D_MISC, "%s:  fw_version: %d.%d fw_version_unit: 0x%02x "
1903 	    "fw_version_dev: 0x%02x\n", DEVNAME(sc),
1904 	    ifp.fw_version_maj, ifp.fw_version_min,
1905 	    ifp.fw_version_unit, ifp.fw_version_dev);
1906 	DNPRINTF(MPI_D_MISC, "%s:  hi_priority_queue_depth: 0x%04x\n",
1907 	    DEVNAME(sc), letoh16(ifp.hi_priority_queue_depth));
1908 	DNPRINTF(MPI_D_MISC, "%s:  host_page_buffer_sge: hdr: 0x%08x "
1909 	    "addr 0x%08x %08x\n", DEVNAME(sc),
1910 	    letoh32(ifp.host_page_buffer_sge.sg_hdr),
1911 	    letoh32(ifp.host_page_buffer_sge.sg_hi_addr),
1912 	    letoh32(ifp.host_page_buffer_sge.sg_lo_addr));
1913 
1914 	sc->sc_maxcmds = letoh16(ifp.global_credits);
1915 	sc->sc_maxchdepth = ifp.max_chain_depth;
1916 	sc->sc_ioc_number = ifp.ioc_number;
1917 	if (sc->sc_flags & MPI_F_SPI)
1918 		sc->sc_buswidth = 16;
1919 	else
1920 		sc->sc_buswidth =
1921 		    (ifp.max_devices == 0) ? 256 : ifp.max_devices;
1922 	if (ifp.flags & MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT)
1923 		sc->sc_fw_len = letoh32(ifp.fw_image_size);
1924 
1925 	sc->sc_repq = MIN(MPI_REPLYQ_DEPTH, letoh16(ifp.reply_queue_depth));
1926 
1927 	/*
1928 	 * you can fit sg elements on the end of the io cmd if they fit in the
1929 	 * request frame size.
1930 	 */
1931 	sc->sc_first_sgl_len = ((letoh16(ifp.request_frame_size) * 4) -
1932 	    sizeof(struct mpi_msg_scsi_io)) / sizeof(struct mpi_sge);
1933 	DNPRINTF(MPI_D_MISC, "%s:   first sgl len: %d\n", DEVNAME(sc),
1934 	    sc->sc_first_sgl_len);
1935 
1936 	sc->sc_chain_len = (letoh16(ifp.request_frame_size) * 4) /
1937 	    sizeof(struct mpi_sge);
1938 	DNPRINTF(MPI_D_MISC, "%s:   chain len: %d\n", DEVNAME(sc),
1939 	    sc->sc_chain_len);
1940 
1941 	/* the sgl tailing the io cmd loses an entry to the chain element. */
1942 	sc->sc_max_sgl_len = MPI_MAX_SGL - 1;
1943 	/* the sgl chains lose an entry for each chain element */
1944 	sc->sc_max_sgl_len -= (MPI_MAX_SGL - sc->sc_first_sgl_len) /
1945 	    sc->sc_chain_len;
1946 	DNPRINTF(MPI_D_MISC, "%s:   max sgl len: %d\n", DEVNAME(sc),
1947 	    sc->sc_max_sgl_len);
1948 
1949 	/* XXX we're ignoring the max chain depth */
1950 
1951 	return (0);
1952 }
1953 
1954 int
1955 mpi_iocinit(struct mpi_softc *sc)
1956 {
1957 	struct mpi_msg_iocinit_request		iiq;
1958 	struct mpi_msg_iocinit_reply		iip;
1959 	u_int32_t				hi_addr;
1960 
1961 	DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit\n", DEVNAME(sc));
1962 
1963 	bzero(&iiq, sizeof(iiq));
1964 	bzero(&iip, sizeof(iip));
1965 
1966 	iiq.function = MPI_FUNCTION_IOC_INIT;
1967 	iiq.whoinit = MPI_WHOINIT_HOST_DRIVER;
1968 
1969 	iiq.max_devices = (sc->sc_buswidth == 256) ? 0 : sc->sc_buswidth;
1970 	iiq.max_buses = 1;
1971 
1972 	iiq.msg_context = htole32(0xd00fd00f);
1973 
1974 	iiq.reply_frame_size = htole16(MPI_REPLY_SIZE);
1975 
1976 	hi_addr = (u_int32_t)((u_int64_t)MPI_DMA_DVA(sc->sc_requests) >> 32);
1977 	iiq.host_mfa_hi_addr = htole32(hi_addr);
1978 	iiq.sense_buffer_hi_addr = htole32(hi_addr);
1979 
1980 	iiq.msg_version_maj = 0x01;
1981 	iiq.msg_version_min = 0x02;
1982 
1983 	iiq.hdr_version_unit = 0x0d;
1984 	iiq.hdr_version_dev = 0x00;
1985 
1986 	if (mpi_handshake_send(sc, &iiq, dwordsof(iiq)) != 0) {
1987 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit send failed\n",
1988 		    DEVNAME(sc));
1989 		return (1);
1990 	}
1991 
1992 	if (mpi_handshake_recv(sc, &iip, dwordsof(iip)) != 0) {
1993 		DNPRINTF(MPI_D_MISC, "%s: mpi_iocinit recv failed\n",
1994 		    DEVNAME(sc));
1995 		return (1);
1996 	}
1997 
1998 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d "
1999 	    "whoinit: 0x%02x\n", DEVNAME(sc), iip.function,
2000 	    iip.msg_length, iip.whoinit);
2001 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x max_buses: %d "
2002 	    "max_devices: %d flags: 0x%02x\n", DEVNAME(sc), iip.msg_flags,
2003 	    iip.max_buses, iip.max_devices, iip.flags);
2004 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2005 	    letoh32(iip.msg_context));
2006 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2007 	    letoh16(iip.ioc_status));
2008 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2009 	    letoh32(iip.ioc_loginfo));
2010 
2011 	return (0);
2012 }
2013 
2014 int
2015 mpi_portfacts(struct mpi_softc *sc)
2016 {
2017 	struct mpi_ccb				*ccb;
2018 	struct mpi_msg_portfacts_request	*pfq;
2019 	volatile struct mpi_msg_portfacts_reply	*pfp;
2020 	int					rv = 1;
2021 
2022 	DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts\n", DEVNAME(sc));
2023 
2024 	ccb = mpi_get_ccb(sc);
2025 	if (ccb == NULL) {
2026 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts ccb_get\n",
2027 		    DEVNAME(sc));
2028 		return (rv);
2029 	}
2030 
2031 	ccb->ccb_done = mpi_empty_done;
2032 	pfq = ccb->ccb_cmd;
2033 
2034 	pfq->function = MPI_FUNCTION_PORT_FACTS;
2035 	pfq->chain_offset = 0;
2036 	pfq->msg_flags = 0;
2037 	pfq->port_number = 0;
2038 	pfq->msg_context = htole32(ccb->ccb_id);
2039 
2040 	if (mpi_poll(sc, ccb, 50000) != 0) {
2041 		DNPRINTF(MPI_D_MISC, "%s: mpi_portfacts poll\n", DEVNAME(sc));
2042 		goto err;
2043 	}
2044 
2045 	if (ccb->ccb_rcb == NULL) {
2046 		DNPRINTF(MPI_D_MISC, "%s: empty portfacts reply\n",
2047 		    DEVNAME(sc));
2048 		goto err;
2049 	}
2050 	pfp = ccb->ccb_rcb->rcb_reply;
2051 
2052 	DNPRINTF(MPI_D_MISC, "%s:  function: 0x%02x msg_length: %d\n",
2053 	    DEVNAME(sc), pfp->function, pfp->msg_length);
2054 	DNPRINTF(MPI_D_MISC, "%s:  msg_flags: 0x%02x port_number: %d\n",
2055 	    DEVNAME(sc), pfp->msg_flags, pfp->port_number);
2056 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2057 	    letoh32(pfp->msg_context));
2058 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2059 	    letoh16(pfp->ioc_status));
2060 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2061 	    letoh32(pfp->ioc_loginfo));
2062 	DNPRINTF(MPI_D_MISC, "%s:  max_devices: %d port_type: 0x%02x\n",
2063 	    DEVNAME(sc), letoh16(pfp->max_devices), pfp->port_type);
2064 	DNPRINTF(MPI_D_MISC, "%s:  protocol_flags: 0x%04x port_scsi_id: %d\n",
2065 	    DEVNAME(sc), letoh16(pfp->protocol_flags),
2066 	    letoh16(pfp->port_scsi_id));
2067 	DNPRINTF(MPI_D_MISC, "%s:  max_persistent_ids: %d "
2068 	    "max_posted_cmd_buffers: %d\n", DEVNAME(sc),
2069 	    letoh16(pfp->max_persistent_ids),
2070 	    letoh16(pfp->max_posted_cmd_buffers));
2071 	DNPRINTF(MPI_D_MISC, "%s:  max_lan_buckets: %d\n", DEVNAME(sc),
2072 	    letoh16(pfp->max_lan_buckets));
2073 
2074 	sc->sc_porttype = pfp->port_type;
2075 	if (sc->sc_target == -1)
2076 		sc->sc_target = letoh16(pfp->port_scsi_id);
2077 
2078 	mpi_push_reply(sc, ccb->ccb_rcb);
2079 	rv = 0;
2080 err:
2081 	mpi_put_ccb(sc, ccb);
2082 
2083 	return (rv);
2084 }
2085 
2086 int
2087 mpi_cfg_coalescing(struct mpi_softc *sc)
2088 {
2089 	struct mpi_cfg_hdr		hdr;
2090 	struct mpi_cfg_ioc_pg1		pg;
2091 	u_int32_t			flags;
2092 
2093 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 1, 0, &hdr) != 0) {
2094 		DNPRINTF(MPI_D_MISC, "%s: unable to fetch IOC page 1 header\n",
2095 		    DEVNAME(sc));
2096 		return (1);
2097 	}
2098 
2099 	if (mpi_cfg_page(sc, 0, &hdr, 1, &pg, sizeof(pg)) != 0) {
2100 		DNPRINTF(MPI_D_MISC, "%s: unable to fetch IOC page 1\n",
2101 		    DEVNAME(sc));
2102 		return (1);
2103 	}
2104 
2105 	DNPRINTF(MPI_D_MISC, "%s: IOC page 1\n", DEVNAME(sc));
2106 	DNPRINTF(MPI_D_MISC, "%s:  flags: 0x08%x\n", DEVNAME(sc),
2107 	    letoh32(pg.flags));
2108 	DNPRINTF(MPI_D_MISC, "%s:  coalescing_timeout: %d\n", DEVNAME(sc),
2109 	    letoh32(pg.coalescing_timeout));
2110 	DNPRINTF(MPI_D_MISC, "%s:  coalescing_depth: %d pci_slot_num: %d\n",
2111 	    DEVNAME(sc), pg.coalescing_timeout, pg.pci_slot_num);
2112 
2113 	flags = letoh32(pg.flags);
2114 	if (!ISSET(flags, MPI_CFG_IOC_1_REPLY_COALESCING))
2115 		return (0);
2116 
2117 	CLR(pg.flags, htole32(MPI_CFG_IOC_1_REPLY_COALESCING));
2118 	if (mpi_cfg_page(sc, 0, &hdr, 0, &pg, sizeof(pg)) != 0) {
2119 		DNPRINTF(MPI_D_MISC, "%s: unable to clear coalescing\n",
2120 		    DEVNAME(sc));
2121 		return (1);
2122 	}
2123 
2124 	return (0);
2125 }
2126 
2127 int
2128 mpi_eventnotify(struct mpi_softc *sc)
2129 {
2130 	struct mpi_ccb				*ccb;
2131 	struct mpi_msg_event_request		*enq;
2132 
2133 	ccb = mpi_get_ccb(sc);
2134 	if (ccb == NULL) {
2135 		DNPRINTF(MPI_D_MISC, "%s: mpi_eventnotify ccb_get\n",
2136 		    DEVNAME(sc));
2137 		return (1);
2138 	}
2139 
2140 	ccb->ccb_done = mpi_eventnotify_done;
2141 	enq = ccb->ccb_cmd;
2142 
2143 	enq->function = MPI_FUNCTION_EVENT_NOTIFICATION;
2144 	enq->chain_offset = 0;
2145 	enq->event_switch = MPI_EVENT_SWITCH_ON;
2146 	enq->msg_context = htole32(ccb->ccb_id);
2147 
2148 	sc->sc_evt_ccb = ccb;
2149 	mpi_start(sc, ccb);
2150 	return (0);
2151 }
2152 
2153 void
2154 mpi_eventnotify_done(struct mpi_ccb *ccb)
2155 {
2156 	struct mpi_softc			*sc = ccb->ccb_sc;
2157 	struct mpi_msg_event_reply		*enp = ccb->ccb_rcb->rcb_reply;
2158 
2159 	DNPRINTF(MPI_D_EVT, "%s: mpi_eventnotify_done\n", DEVNAME(sc));
2160 
2161 	DNPRINTF(MPI_D_EVT, "%s:  function: 0x%02x msg_length: %d "
2162 	    "data_length: %d\n", DEVNAME(sc), enp->function, enp->msg_length,
2163 	    letoh16(enp->data_length));
2164 	DNPRINTF(MPI_D_EVT, "%s:  ack_required: %d msg_flags 0x%02x\n",
2165 	    DEVNAME(sc), enp->ack_required, enp->msg_flags);
2166 	DNPRINTF(MPI_D_EVT, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2167 	    letoh32(enp->msg_context));
2168 	DNPRINTF(MPI_D_EVT, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2169 	    letoh16(enp->ioc_status));
2170 	DNPRINTF(MPI_D_EVT, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2171 	    letoh32(enp->ioc_loginfo));
2172 	DNPRINTF(MPI_D_EVT, "%s:  event: 0x%08x\n", DEVNAME(sc),
2173 	    letoh32(enp->event));
2174 	DNPRINTF(MPI_D_EVT, "%s:  event_context: 0x%08x\n", DEVNAME(sc),
2175 	    letoh32(enp->event_context));
2176 
2177 	switch (letoh32(enp->event)) {
2178 	/* ignore these */
2179 	case MPI_EVENT_EVENT_CHANGE:
2180 	case MPI_EVENT_SAS_PHY_LINK_STATUS:
2181 		break;
2182 
2183 	case MPI_EVENT_SAS_DEVICE_STATUS_CHANGE:
2184 		if (sc->sc_scsibus == NULL)
2185 			break;
2186 
2187 		mpi_evt_sas(sc, ccb->ccb_rcb);
2188 		break;
2189 
2190 	default:
2191 		DNPRINTF(MPI_D_EVT, "%s:  unhandled event 0x%02x\n",
2192 		    DEVNAME(sc), letoh32(enp->event));
2193 		break;
2194 	}
2195 
2196 	if (enp->ack_required)
2197 		mpi_eventack(sc, enp);
2198 	mpi_push_reply(sc, ccb->ccb_rcb);
2199 
2200 #if 0
2201 	/* fc hbas have a bad habit of setting this without meaning it. */
2202 	if ((enp->msg_flags & MPI_EVENT_FLAGS_REPLY_KEPT) == 0) {
2203 		mpi_put_ccb(sc, ccb);
2204 	}
2205 #endif
2206 }
2207 
2208 void
2209 mpi_evt_sas(struct mpi_softc *sc, struct mpi_rcb *rcb)
2210 {
2211 	struct mpi_evt_sas_change		*ch;
2212 	u_int8_t				*data;
2213 
2214 	data = rcb->rcb_reply;
2215 	data += sizeof(struct mpi_msg_event_reply);
2216 	ch = (struct mpi_evt_sas_change *)data;
2217 
2218 	if (ch->bus != 0)
2219 		return;
2220 
2221 	switch (ch->reason) {
2222 	case MPI_EVT_SASCH_REASON_ADDED:
2223 	case MPI_EVT_SASCH_REASON_NO_PERSIST_ADDED:
2224 		if (scsi_req_probe(sc->sc_scsibus, ch->target, -1) != 0) {
2225 			printf("%s: unable to request attach of %d\n",
2226 			    DEVNAME(sc), ch->target);
2227 		}
2228 		break;
2229 
2230 	case MPI_EVT_SASCH_REASON_NOT_RESPONDING:
2231 		scsi_activate(sc->sc_scsibus, ch->target, -1, DVACT_DEACTIVATE);
2232 		if (scsi_req_detach(sc->sc_scsibus, ch->target, -1,
2233 		    DETACH_FORCE) != 0) {
2234 			printf("%s: unable to request detach of %d\n",
2235 			    DEVNAME(sc), ch->target);
2236 		}
2237 		break;
2238 
2239 	case MPI_EVT_SASCH_REASON_SMART_DATA:
2240 	case MPI_EVT_SASCH_REASON_UNSUPPORTED:
2241 	case MPI_EVT_SASCH_REASON_INTERNAL_RESET:
2242 		break;
2243 	default:
2244 		printf("%s: unknown reason for SAS device status change: "
2245 		    "0x%02x\n", DEVNAME(sc), ch->reason);
2246 		break;
2247 	}
2248 }
2249 
2250 void
2251 mpi_eventack(struct mpi_softc *sc, struct mpi_msg_event_reply *enp)
2252 {
2253 	struct mpi_ccb				*ccb;
2254 	struct mpi_msg_eventack_request		*eaq;
2255 
2256 	ccb = mpi_get_ccb(sc);
2257 	if (ccb == NULL) {
2258 		DNPRINTF(MPI_D_EVT, "%s: mpi_eventack ccb_get\n", DEVNAME(sc));
2259 		return;
2260 	}
2261 
2262 	ccb->ccb_done = mpi_eventack_done;
2263 	eaq = ccb->ccb_cmd;
2264 
2265 	eaq->function = MPI_FUNCTION_EVENT_ACK;
2266 	eaq->msg_context = htole32(ccb->ccb_id);
2267 
2268 	eaq->event = enp->event;
2269 	eaq->event_context = enp->event_context;
2270 
2271 	mpi_start(sc, ccb);
2272 	return;
2273 }
2274 
2275 void
2276 mpi_eventack_done(struct mpi_ccb *ccb)
2277 {
2278 	struct mpi_softc			*sc = ccb->ccb_sc;
2279 
2280 	DNPRINTF(MPI_D_EVT, "%s: event ack done\n", DEVNAME(sc));
2281 
2282 	mpi_push_reply(sc, ccb->ccb_rcb);
2283 	mpi_put_ccb(sc, ccb);
2284 }
2285 
2286 int
2287 mpi_portenable(struct mpi_softc *sc)
2288 {
2289 	struct mpi_ccb				*ccb;
2290 	struct mpi_msg_portenable_request	*peq;
2291 	int					rv = 0;
2292 
2293 	DNPRINTF(MPI_D_MISC, "%s: mpi_portenable\n", DEVNAME(sc));
2294 
2295 	ccb = mpi_get_ccb(sc);
2296 	if (ccb == NULL) {
2297 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable ccb_get\n",
2298 		    DEVNAME(sc));
2299 		return (1);
2300 	}
2301 
2302 	ccb->ccb_done = mpi_empty_done;
2303 	peq = ccb->ccb_cmd;
2304 
2305 	peq->function = MPI_FUNCTION_PORT_ENABLE;
2306 	peq->port_number = 0;
2307 	peq->msg_context = htole32(ccb->ccb_id);
2308 
2309 	if (mpi_poll(sc, ccb, 50000) != 0) {
2310 		DNPRINTF(MPI_D_MISC, "%s: mpi_portenable poll\n", DEVNAME(sc));
2311 		return (1);
2312 	}
2313 
2314 	if (ccb->ccb_rcb == NULL) {
2315 		DNPRINTF(MPI_D_MISC, "%s: empty portenable reply\n",
2316 		    DEVNAME(sc));
2317 		rv = 1;
2318 	} else
2319 		mpi_push_reply(sc, ccb->ccb_rcb);
2320 
2321 	mpi_put_ccb(sc, ccb);
2322 
2323 	return (rv);
2324 }
2325 
2326 int
2327 mpi_fwupload(struct mpi_softc *sc)
2328 {
2329 	struct mpi_ccb				*ccb;
2330 	struct {
2331 		struct mpi_msg_fwupload_request		req;
2332 		struct mpi_sge				sge;
2333 	} __packed				*bundle;
2334 	struct mpi_msg_fwupload_reply		*upp;
2335 	u_int64_t				addr;
2336 	int					rv = 0;
2337 
2338 	if (sc->sc_fw_len == 0)
2339 		return (0);
2340 
2341 	DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload\n", DEVNAME(sc));
2342 
2343 	sc->sc_fw = mpi_dmamem_alloc(sc, sc->sc_fw_len);
2344 	if (sc->sc_fw == NULL) {
2345 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload unable to allocate %d\n",
2346 		    DEVNAME(sc), sc->sc_fw_len);
2347 		return (1);
2348 	}
2349 
2350 	ccb = mpi_get_ccb(sc);
2351 	if (ccb == NULL) {
2352 		DNPRINTF(MPI_D_MISC, "%s: mpi_fwupload ccb_get\n",
2353 		    DEVNAME(sc));
2354 		goto err;
2355 	}
2356 
2357 	ccb->ccb_done = mpi_empty_done;
2358 	bundle = ccb->ccb_cmd;
2359 
2360 	bundle->req.function = MPI_FUNCTION_FW_UPLOAD;
2361 	bundle->req.msg_context = htole32(ccb->ccb_id);
2362 
2363 	bundle->req.image_type = MPI_FWUPLOAD_IMAGETYPE_IOC_FW;
2364 
2365 	bundle->req.tce.details_length = 12;
2366 	bundle->req.tce.image_size = htole32(sc->sc_fw_len);
2367 
2368 	bundle->sge.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2369 	    MPI_SGE_FL_SIZE_64 | MPI_SGE_FL_LAST | MPI_SGE_FL_EOB |
2370 	    MPI_SGE_FL_EOL | (u_int32_t)sc->sc_fw_len);
2371 	addr = MPI_DMA_DVA(sc->sc_fw);
2372 	bundle->sge.sg_hi_addr = htole32((u_int32_t)(addr >> 32));
2373 	bundle->sge.sg_lo_addr = htole32((u_int32_t)addr);
2374 
2375 	if (mpi_poll(sc, ccb, 50000) != 0) {
2376 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n", DEVNAME(sc));
2377 		goto err;
2378 	}
2379 
2380 	if (ccb->ccb_rcb == NULL)
2381 		panic("%s: unable to do fw upload\n", DEVNAME(sc));
2382 	upp = ccb->ccb_rcb->rcb_reply;
2383 
2384 	if (letoh16(upp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2385 		rv = 1;
2386 
2387 	mpi_push_reply(sc, ccb->ccb_rcb);
2388 	mpi_put_ccb(sc, ccb);
2389 
2390 	return (rv);
2391 
2392 err:
2393 	mpi_dmamem_free(sc, sc->sc_fw);
2394 	return (1);
2395 }
2396 
2397 void
2398 mpi_get_raid(struct mpi_softc *sc)
2399 {
2400 	struct mpi_cfg_hdr		hdr;
2401 	struct mpi_cfg_ioc_pg2		*vol_page;
2402 	struct mpi_cfg_raid_vol		*vol_list, *vol;
2403 	size_t				pagelen;
2404 	u_int32_t			capabilities;
2405 	struct scsi_link		*link;
2406 	int				i;
2407 
2408 	DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid\n", DEVNAME(sc));
2409 
2410 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_IOC, 2, 0, &hdr) != 0) {
2411 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch header"
2412 		    "for IOC page 2\n", DEVNAME(sc));
2413 		return;
2414 	}
2415 
2416 	pagelen = hdr.page_length * 4; /* dwords to bytes */
2417 	vol_page = malloc(pagelen, M_TEMP, M_WAITOK|M_CANFAIL);
2418 	if (vol_page == NULL) {
2419 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to allocate "
2420 		    "space for ioc config page 2\n", DEVNAME(sc));
2421 		return;
2422 	}
2423 	vol_list = (struct mpi_cfg_raid_vol *)(vol_page + 1);
2424 
2425 	if (mpi_cfg_page(sc, 0, &hdr, 1, vol_page, pagelen) != 0) {
2426 		DNPRINTF(MPI_D_RAID, "%s: mpi_get_raid unable to fetch IOC "
2427 		    "page 2\n", DEVNAME(sc));
2428 		goto out;
2429 	}
2430 
2431 	capabilities = letoh32(vol_page->capabilities);
2432 
2433 	DNPRINTF(MPI_D_RAID, "%s:  capabilities: 0x08%x\n", DEVNAME(sc),
2434 	    letoh32(vol_page->capabilities));
2435 	DNPRINTF(MPI_D_RAID, "%s:  active_vols: %d max_vols: %d "
2436 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
2437 	    vol_page->active_vols, vol_page->max_vols,
2438 	    vol_page->active_physdisks, vol_page->max_physdisks);
2439 
2440 	/* don't walk list if there are no RAID capability */
2441 	if (capabilities == 0xdeadbeef) {
2442 		printf("%s: deadbeef in raid configuration\n", DEVNAME(sc));
2443 		goto out;
2444 	}
2445 
2446 	if ((capabilities & MPI_CFG_IOC_2_CAPABILITIES_RAID) == 0 ||
2447 	    (vol_page->active_vols == 0))
2448 		goto out;
2449 
2450 	sc->sc_flags |= MPI_F_RAID;
2451 
2452 	for (i = 0; i < vol_page->active_vols; i++) {
2453 		vol = &vol_list[i];
2454 
2455 		DNPRINTF(MPI_D_RAID, "%s:   id: %d bus: %d ioc: %d pg: %d\n",
2456 		    DEVNAME(sc), vol->vol_id, vol->vol_bus, vol->vol_ioc,
2457 		    vol->vol_page);
2458 		DNPRINTF(MPI_D_RAID, "%s:   type: 0x%02x flags: 0x%02x\n",
2459 		    DEVNAME(sc), vol->vol_type, vol->flags);
2460 
2461 		if (vol->vol_ioc != sc->sc_ioc_number || vol->vol_bus != 0)
2462 			continue;
2463 
2464 		link = sc->sc_scsibus->sc_link[vol->vol_id][0];
2465 		if (link == NULL)
2466 			continue;
2467 
2468 		link->flags |= SDEV_VIRTUAL;
2469 	}
2470 
2471 out:
2472 	free(vol_page, M_TEMP);
2473 }
2474 
2475 int
2476 mpi_req_cfg_header(struct mpi_softc *sc, u_int8_t type, u_int8_t number,
2477     u_int32_t address, int flags, void *p)
2478 {
2479 	struct mpi_ccb				*ccb;
2480 	struct mpi_msg_config_request		*cq;
2481 	struct mpi_msg_config_reply		*cp;
2482 	struct mpi_cfg_hdr			*hdr = p;
2483 	struct mpi_ecfg_hdr			*ehdr = p;
2484 	int					etype = 0;
2485 	int					rv = 0;
2486 
2487 	DNPRINTF(MPI_D_MISC, "%s: mpi_req_cfg_header type: %#x number: %x "
2488 	    "address: 0x%08x flags: 0x%b\n", DEVNAME(sc), type, number,
2489 	    address, flags, MPI_PG_FMT);
2490 
2491 	ccb = scsi_io_get(&sc->sc_iopool,
2492 	    ISSET(flags, MPI_PG_POLL) ? SCSI_NOSLEEP : 0);
2493 	if (ccb == NULL) {
2494 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header ccb_get\n",
2495 		    DEVNAME(sc));
2496 		return (1);
2497 	}
2498 
2499 	if (ISSET(flags, MPI_PG_EXTENDED)) {
2500 		etype = type;
2501 		type = MPI_CONFIG_REQ_PAGE_TYPE_EXTENDED;
2502 	}
2503 
2504 	cq = ccb->ccb_cmd;
2505 
2506 	cq->function = MPI_FUNCTION_CONFIG;
2507 	cq->msg_context = htole32(ccb->ccb_id);
2508 
2509 	cq->action = MPI_CONFIG_REQ_ACTION_PAGE_HEADER;
2510 
2511 	cq->config_header.page_number = number;
2512 	cq->config_header.page_type = type;
2513 	cq->ext_page_type = etype;
2514 	cq->page_address = htole32(address);
2515 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2516 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL);
2517 
2518 	ccb->ccb_done = mpi_empty_done;
2519 	if (ISSET(flags, MPI_PG_POLL)) {
2520 		if (mpi_poll(sc, ccb, 50000) != 0) {
2521 			DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n",
2522 			    DEVNAME(sc));
2523 			return (1);
2524 		}
2525 	} else
2526 		mpi_wait(sc, ccb);
2527 
2528 	if (ccb->ccb_rcb == NULL)
2529 		panic("%s: unable to fetch config header\n", DEVNAME(sc));
2530 	cp = ccb->ccb_rcb->rcb_reply;
2531 
2532 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2533 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2534 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2535 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2536 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2537 	    cp->msg_flags);
2538 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2539 	    letoh32(cp->msg_context));
2540 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2541 	    letoh16(cp->ioc_status));
2542 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2543 	    letoh32(cp->ioc_loginfo));
2544 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2545 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2546 	    cp->config_header.page_version,
2547 	    cp->config_header.page_length,
2548 	    cp->config_header.page_number,
2549 	    cp->config_header.page_type);
2550 
2551 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2552 		rv = 1;
2553 	else if (ISSET(flags, MPI_PG_EXTENDED)) {
2554 		bzero(ehdr, sizeof(*ehdr));
2555 		ehdr->page_version = cp->config_header.page_version;
2556 		ehdr->page_number = cp->config_header.page_number;
2557 		ehdr->page_type = cp->config_header.page_type;
2558 		ehdr->ext_page_length = cp->ext_page_length;
2559 		ehdr->ext_page_type = cp->ext_page_type;
2560 	} else
2561 		*hdr = cp->config_header;
2562 
2563 	mpi_push_reply(sc, ccb->ccb_rcb);
2564 	scsi_io_put(&sc->sc_iopool, ccb);
2565 
2566 	return (rv);
2567 }
2568 
2569 int
2570 mpi_req_cfg_page(struct mpi_softc *sc, u_int32_t address, int flags,
2571     void *p, int read, void *page, size_t len)
2572 {
2573 	struct mpi_ccb				*ccb;
2574 	struct mpi_msg_config_request		*cq;
2575 	struct mpi_msg_config_reply		*cp;
2576 	struct mpi_cfg_hdr			*hdr = p;
2577 	struct mpi_ecfg_hdr			*ehdr = p;
2578 	u_int64_t				dva;
2579 	char					*kva;
2580 	int					page_length;
2581 	int					rv = 0;
2582 
2583 	DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page address: %d read: %d type: %x\n",
2584 	    DEVNAME(sc), address, read, hdr->page_type);
2585 
2586 	page_length = ISSET(flags, MPI_PG_EXTENDED) ?
2587 	    letoh16(ehdr->ext_page_length) : hdr->page_length;
2588 
2589 	if (len > MPI_REQUEST_SIZE - sizeof(struct mpi_msg_config_request) ||
2590 	    len < page_length * 4)
2591 		return (1);
2592 
2593 	ccb = scsi_io_get(&sc->sc_iopool,
2594 	    ISSET(flags, MPI_PG_POLL) ? SCSI_NOSLEEP : 0);
2595 	if (ccb == NULL) {
2596 		DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_page ccb_get\n", DEVNAME(sc));
2597 		return (1);
2598 	}
2599 
2600 	cq = ccb->ccb_cmd;
2601 
2602 	cq->function = MPI_FUNCTION_CONFIG;
2603 	cq->msg_context = htole32(ccb->ccb_id);
2604 
2605 	cq->action = (read ? MPI_CONFIG_REQ_ACTION_PAGE_READ_CURRENT :
2606 	    MPI_CONFIG_REQ_ACTION_PAGE_WRITE_CURRENT);
2607 
2608 	if (ISSET(flags, MPI_PG_EXTENDED)) {
2609 		cq->config_header.page_version = ehdr->page_version;
2610 		cq->config_header.page_number = ehdr->page_number;
2611 		cq->config_header.page_type = ehdr->page_type;
2612 		cq->ext_page_len = ehdr->ext_page_length;
2613 		cq->ext_page_type = ehdr->ext_page_type;
2614 	} else
2615 		cq->config_header = *hdr;
2616 	cq->config_header.page_type &= MPI_CONFIG_REQ_PAGE_TYPE_MASK;
2617 	cq->page_address = htole32(address);
2618 	cq->page_buffer.sg_hdr = htole32(MPI_SGE_FL_TYPE_SIMPLE |
2619 	    MPI_SGE_FL_LAST | MPI_SGE_FL_EOB | MPI_SGE_FL_EOL |
2620 	    (page_length * 4) |
2621 	    (read ? MPI_SGE_FL_DIR_IN : MPI_SGE_FL_DIR_OUT));
2622 
2623 	/* bounce the page via the request space to avoid more bus_dma games */
2624 	dva = ccb->ccb_cmd_dva + sizeof(struct mpi_msg_config_request);
2625 
2626 	cq->page_buffer.sg_hi_addr = htole32((u_int32_t)(dva >> 32));
2627 	cq->page_buffer.sg_lo_addr = htole32((u_int32_t)dva);
2628 
2629 	kva = ccb->ccb_cmd;
2630 	kva += sizeof(struct mpi_msg_config_request);
2631 	if (!read)
2632 		bcopy(page, kva, len);
2633 
2634 	ccb->ccb_done = mpi_empty_done;
2635 	if (ISSET(flags, MPI_PG_POLL)) {
2636 		if (mpi_poll(sc, ccb, 50000) != 0) {
2637 			DNPRINTF(MPI_D_MISC, "%s: mpi_cfg_header poll\n",
2638 			    DEVNAME(sc));
2639 			return (1);
2640 		}
2641 	} else
2642 		mpi_wait(sc, ccb);
2643 
2644 	if (ccb->ccb_rcb == NULL) {
2645 		mpi_put_ccb(sc, ccb);
2646 		return (1);
2647 	}
2648 	cp = ccb->ccb_rcb->rcb_reply;
2649 
2650 	DNPRINTF(MPI_D_MISC, "%s:  action: 0x%02x msg_length: %d function: "
2651 	    "0x%02x\n", DEVNAME(sc), cp->action, cp->msg_length, cp->function);
2652 	DNPRINTF(MPI_D_MISC, "%s:  ext_page_length: %d ext_page_type: 0x%02x "
2653 	    "msg_flags: 0x%02x\n", DEVNAME(sc),
2654 	    letoh16(cp->ext_page_length), cp->ext_page_type,
2655 	    cp->msg_flags);
2656 	DNPRINTF(MPI_D_MISC, "%s:  msg_context: 0x%08x\n", DEVNAME(sc),
2657 	    letoh32(cp->msg_context));
2658 	DNPRINTF(MPI_D_MISC, "%s:  ioc_status: 0x%04x\n", DEVNAME(sc),
2659 	    letoh16(cp->ioc_status));
2660 	DNPRINTF(MPI_D_MISC, "%s:  ioc_loginfo: 0x%08x\n", DEVNAME(sc),
2661 	    letoh32(cp->ioc_loginfo));
2662 	DNPRINTF(MPI_D_MISC, "%s:  page_version: 0x%02x page_length: %d "
2663 	    "page_number: 0x%02x page_type: 0x%02x\n", DEVNAME(sc),
2664 	    cp->config_header.page_version,
2665 	    cp->config_header.page_length,
2666 	    cp->config_header.page_number,
2667 	    cp->config_header.page_type);
2668 
2669 	if (letoh16(cp->ioc_status) != MPI_IOCSTATUS_SUCCESS)
2670 		rv = 1;
2671 	else if (read)
2672 		bcopy(kva, page, len);
2673 
2674 	mpi_push_reply(sc, ccb->ccb_rcb);
2675 	scsi_io_put(&sc->sc_iopool, ccb);
2676 
2677 	return (rv);
2678 }
2679 
2680 int
2681 mpi_scsi_ioctl(struct scsi_link *link, u_long cmd, caddr_t addr, int flag,
2682     struct proc *p)
2683 {
2684 	struct mpi_softc	*sc = (struct mpi_softc *)link->adapter_softc;
2685 
2686 	DNPRINTF(MPI_D_IOCTL, "%s: mpi_scsi_ioctl\n", DEVNAME(sc));
2687 
2688 	if (sc->sc_ioctl)
2689 		return (sc->sc_ioctl(link->adapter_softc, cmd, addr));
2690 	else
2691 		return (ENOTTY);
2692 }
2693 
2694 #if NBIO > 0
2695 int
2696 mpi_bio_get_pg0_raid(struct mpi_softc *sc, int id)
2697 {
2698 	int			len, rv = EINVAL;
2699 	u_int32_t		address;
2700 	struct mpi_cfg_hdr	hdr;
2701 	struct mpi_cfg_raid_vol_pg0 *rpg0;
2702 
2703 	/* get IOC page 2 */
2704 	if (mpi_req_cfg_page(sc, 0, 0, &sc->sc_cfg_hdr, 1, sc->sc_vol_page,
2705 	    sc->sc_cfg_hdr.page_length * 4) != 0) {
2706 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_bio_get_pg0_raid unable to "
2707 		    "fetch IOC page 2\n", DEVNAME(sc));
2708 		goto done;
2709 	}
2710 
2711 	/* XXX return something else than EINVAL to indicate within hs range */
2712 	if (id > sc->sc_vol_page->active_vols) {
2713 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_bio_get_pg0_raid invalid vol "
2714 		    "id: %d\n", DEVNAME(sc), id);
2715 		goto done;
2716 	}
2717 
2718 	/* replace current buffer with new one */
2719 	len = sizeof *rpg0 + sc->sc_vol_page->max_physdisks *
2720 	    sizeof(struct mpi_cfg_raid_vol_pg0_physdisk);
2721 	rpg0 = malloc(len, M_DEVBUF, M_WAITOK | M_CANFAIL);
2722 	if (rpg0 == NULL) {
2723 		printf("%s: can't get memory for RAID page 0, "
2724 		    "bio disabled\n", DEVNAME(sc));
2725 		goto done;
2726 	}
2727 	if (sc->sc_rpg0)
2728 		free(sc->sc_rpg0, M_DEVBUF);
2729 	sc->sc_rpg0 = rpg0;
2730 
2731 	/* get raid vol page 0 */
2732 	address = sc->sc_vol_list[id].vol_id |
2733 	    (sc->sc_vol_list[id].vol_bus << 8);
2734 	if (mpi_req_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_VOL, 0,
2735 	    address, 0, &hdr) != 0)
2736 		goto done;
2737 	if (mpi_req_cfg_page(sc, address, 0, &hdr, 1, rpg0, len)) {
2738 		DNPRINTF(MPI_D_RAID, "%s: can't get RAID vol cfg page 0\n",
2739 		    DEVNAME(sc));
2740 		goto done;
2741 	}
2742 
2743 	rv = 0;
2744 done:
2745 	return (rv);
2746 }
2747 
2748 int
2749 mpi_ioctl(struct device *dev, u_long cmd, caddr_t addr)
2750 {
2751 	struct mpi_softc	*sc = (struct mpi_softc *)dev;
2752 	int error = 0;
2753 
2754 	DNPRINTF(MPI_D_IOCTL, "%s: mpi_ioctl ", DEVNAME(sc));
2755 
2756 	/* make sure we have bio enabled */
2757 	if (sc->sc_ioctl != mpi_ioctl)
2758 		return (EINVAL);
2759 
2760 	rw_enter_write(&sc->sc_lock);
2761 
2762 	switch (cmd) {
2763 	case BIOCINQ:
2764 		DNPRINTF(MPI_D_IOCTL, "inq\n");
2765 		error = mpi_ioctl_inq(sc, (struct bioc_inq *)addr);
2766 		break;
2767 
2768 	case BIOCVOL:
2769 		DNPRINTF(MPI_D_IOCTL, "vol\n");
2770 		error = mpi_ioctl_vol(sc, (struct bioc_vol *)addr);
2771 		break;
2772 
2773 	case BIOCDISK:
2774 		DNPRINTF(MPI_D_IOCTL, "disk\n");
2775 		error = mpi_ioctl_disk(sc, (struct bioc_disk *)addr);
2776 		break;
2777 
2778 	case BIOCALARM:
2779 		DNPRINTF(MPI_D_IOCTL, "alarm\n");
2780 		break;
2781 
2782 	case BIOCBLINK:
2783 		DNPRINTF(MPI_D_IOCTL, "blink\n");
2784 		break;
2785 
2786 	case BIOCSETSTATE:
2787 		DNPRINTF(MPI_D_IOCTL, "setstate\n");
2788 		error = mpi_ioctl_setstate(sc, (struct bioc_setstate *)addr);
2789 		break;
2790 
2791 	default:
2792 		DNPRINTF(MPI_D_IOCTL, " invalid ioctl\n");
2793 		error = EINVAL;
2794 	}
2795 
2796 	rw_exit_write(&sc->sc_lock);
2797 
2798 	return (error);
2799 }
2800 
2801 int
2802 mpi_ioctl_inq(struct mpi_softc *sc, struct bioc_inq *bi)
2803 {
2804 	if (!(sc->sc_flags & MPI_F_RAID)) {
2805 		bi->bi_novol = 0;
2806 		bi->bi_nodisk = 0;
2807 	}
2808 
2809 	if (mpi_cfg_page(sc, 0, &sc->sc_cfg_hdr, 1, sc->sc_vol_page,
2810 	    sc->sc_cfg_hdr.page_length * 4) != 0) {
2811 		DNPRINTF(MPI_D_IOCTL, "%s: mpi_get_raid unable to fetch IOC "
2812 		    "page 2\n", DEVNAME(sc));
2813 		return (EINVAL);
2814 	}
2815 
2816 	DNPRINTF(MPI_D_IOCTL, "%s:  active_vols: %d max_vols: %d "
2817 	    "active_physdisks: %d max_physdisks: %d\n", DEVNAME(sc),
2818 	    sc->sc_vol_page->active_vols, sc->sc_vol_page->max_vols,
2819 	    sc->sc_vol_page->active_physdisks, sc->sc_vol_page->max_physdisks);
2820 
2821 	bi->bi_novol = sc->sc_vol_page->active_vols;
2822 	bi->bi_nodisk = sc->sc_vol_page->active_physdisks;
2823 	strlcpy(bi->bi_dev, DEVNAME(sc), sizeof(bi->bi_dev));
2824 
2825 	return (0);
2826 }
2827 
2828 int
2829 mpi_ioctl_vol(struct mpi_softc *sc, struct bioc_vol *bv)
2830 {
2831 	int			i, vol, id, rv = EINVAL;
2832 	struct device		*dev;
2833 	struct scsi_link	*link;
2834 	struct mpi_cfg_raid_vol_pg0 *rpg0;
2835 
2836 	id = bv->bv_volid;
2837 	if (mpi_bio_get_pg0_raid(sc, id))
2838 		goto done;
2839 
2840 	if (id > sc->sc_vol_page->active_vols)
2841 		return (EINVAL); /* XXX deal with hot spares */
2842 
2843 	rpg0 = sc->sc_rpg0;
2844 	if (rpg0 == NULL)
2845 		goto done;
2846 
2847 	/* determine status */
2848 	switch (rpg0->volume_state) {
2849 	case MPI_CFG_RAID_VOL_0_STATE_OPTIMAL:
2850 		bv->bv_status = BIOC_SVONLINE;
2851 		break;
2852 	case MPI_CFG_RAID_VOL_0_STATE_DEGRADED:
2853 		bv->bv_status = BIOC_SVDEGRADED;
2854 		break;
2855 	case MPI_CFG_RAID_VOL_0_STATE_FAILED:
2856 	case MPI_CFG_RAID_VOL_0_STATE_MISSING:
2857 		bv->bv_status = BIOC_SVOFFLINE;
2858 		break;
2859 	default:
2860 		bv->bv_status = BIOC_SVINVALID;
2861 	}
2862 
2863 	/* override status if scrubbing or something */
2864 	if (rpg0->volume_status & MPI_CFG_RAID_VOL_0_STATUS_RESYNCING)
2865 		bv->bv_status = BIOC_SVREBUILD;
2866 
2867 	bv->bv_size = (u_quad_t)letoh32(rpg0->max_lba) * 512;
2868 
2869 	switch (sc->sc_vol_list[id].vol_type) {
2870 	case MPI_CFG_RAID_TYPE_RAID_IS:
2871 		bv->bv_level = 0;
2872 		break;
2873 	case MPI_CFG_RAID_TYPE_RAID_IME:
2874 	case MPI_CFG_RAID_TYPE_RAID_IM:
2875 		bv->bv_level = 1;
2876 		break;
2877 	case MPI_CFG_RAID_TYPE_RAID_5:
2878 		bv->bv_level = 5;
2879 		break;
2880 	case MPI_CFG_RAID_TYPE_RAID_6:
2881 		bv->bv_level = 6;
2882 		break;
2883 	case MPI_CFG_RAID_TYPE_RAID_10:
2884 		bv->bv_level = 10;
2885 		break;
2886 	case MPI_CFG_RAID_TYPE_RAID_50:
2887 		bv->bv_level = 50;
2888 		break;
2889 	default:
2890 		bv->bv_level = -1;
2891 	}
2892 
2893 	bv->bv_nodisk = rpg0->num_phys_disks;
2894 
2895 	for (i = 0, vol = -1; i < sc->sc_buswidth; i++) {
2896 		link = sc->sc_scsibus->sc_link[i][0];
2897 		if (link == NULL)
2898 			continue;
2899 
2900 		/* skip if not a virtual disk */
2901 		if (!(link->flags & SDEV_VIRTUAL))
2902 			continue;
2903 
2904 		vol++;
2905 		/* are we it? */
2906 		if (vol == bv->bv_volid) {
2907 			dev = link->device_softc;
2908 			memcpy(bv->bv_vendor, link->inqdata.vendor,
2909 			    sizeof bv->bv_vendor);
2910 			bv->bv_vendor[sizeof(bv->bv_vendor) - 1] = '\0';
2911 			strlcpy(bv->bv_dev, dev->dv_xname, sizeof bv->bv_dev);
2912 			break;
2913 		}
2914 	}
2915 	rv = 0;
2916 done:
2917 	return (rv);
2918 }
2919 
2920 int
2921 mpi_ioctl_disk(struct mpi_softc *sc, struct bioc_disk *bd)
2922 {
2923 	int			pdid, id, rv = EINVAL;
2924 	u_int32_t		address;
2925 	struct mpi_cfg_hdr	hdr;
2926 	struct mpi_cfg_raid_vol_pg0 *rpg0;
2927 	struct mpi_cfg_raid_vol_pg0_physdisk *physdisk;
2928 	struct mpi_cfg_raid_physdisk_pg0 pdpg0;
2929 
2930 	id = bd->bd_volid;
2931 	if (mpi_bio_get_pg0_raid(sc, id))
2932 		goto done;
2933 
2934 	if (id > sc->sc_vol_page->active_vols)
2935 		return (EINVAL); /* XXX deal with hot spares */
2936 
2937 	rpg0 = sc->sc_rpg0;
2938 	if (rpg0 == NULL)
2939 		goto done;
2940 
2941 	pdid = bd->bd_diskid;
2942 	if (pdid > rpg0->num_phys_disks)
2943 		goto done;
2944 	physdisk = (struct mpi_cfg_raid_vol_pg0_physdisk *)(rpg0 + 1);
2945 	physdisk += pdid;
2946 
2947 	/* get raid phys disk page 0 */
2948 	address = physdisk->phys_disk_num;
2949 	if (mpi_cfg_header(sc, MPI_CONFIG_REQ_PAGE_TYPE_RAID_PD, 0, address,
2950 	    &hdr) != 0)
2951 		goto done;
2952 	if (mpi_cfg_page(sc, address, &hdr, 1, &pdpg0, sizeof pdpg0)) {
2953 		bd->bd_status = BIOC_SDFAILED;
2954 		return (0);
2955 	}
2956 	bd->bd_channel = pdpg0.phys_disk_bus;
2957 	bd->bd_target = pdpg0.phys_disk_id;
2958 	bd->bd_lun = 0;
2959 	bd->bd_size = (u_quad_t)letoh32(pdpg0.max_lba) * 512;
2960 	strlcpy(bd->bd_vendor, (char *)pdpg0.vendor_id, sizeof(bd->bd_vendor));
2961 
2962 	switch (pdpg0.phys_disk_state) {
2963 	case MPI_CFG_RAID_PHYDISK_0_STATE_ONLINE:
2964 		bd->bd_status = BIOC_SDONLINE;
2965 		break;
2966 	case MPI_CFG_RAID_PHYDISK_0_STATE_MISSING:
2967 	case MPI_CFG_RAID_PHYDISK_0_STATE_FAILED:
2968 		bd->bd_status = BIOC_SDFAILED;
2969 		break;
2970 	case MPI_CFG_RAID_PHYDISK_0_STATE_HOSTFAIL:
2971 	case MPI_CFG_RAID_PHYDISK_0_STATE_OTHER:
2972 	case MPI_CFG_RAID_PHYDISK_0_STATE_OFFLINE:
2973 		bd->bd_status = BIOC_SDOFFLINE;
2974 		break;
2975 	case MPI_CFG_RAID_PHYDISK_0_STATE_INIT:
2976 		bd->bd_status = BIOC_SDSCRUB;
2977 		break;
2978 	case MPI_CFG_RAID_PHYDISK_0_STATE_INCOMPAT:
2979 	default:
2980 		bd->bd_status = BIOC_SDINVALID;
2981 		break;
2982 	}
2983 
2984 	/* XXX figure this out */
2985 	/* bd_serial[32]; */
2986 	/* bd_procdev[16]; */
2987 
2988 	rv = 0;
2989 done:
2990 	return (rv);
2991 }
2992 
2993 int
2994 mpi_ioctl_setstate(struct mpi_softc *sc, struct bioc_setstate *bs)
2995 {
2996 	return (ENOTTY);
2997 }
2998 
2999 #ifndef SMALL_KERNEL
3000 int
3001 mpi_create_sensors(struct mpi_softc *sc)
3002 {
3003 	struct device		*dev;
3004 	struct scsi_link	*link;
3005 	int			i, vol;
3006 
3007 	/* count volumes */
3008 	for (i = 0, vol = 0; i < sc->sc_buswidth; i++) {
3009 		link = sc->sc_scsibus->sc_link[i][0];
3010 		if (link == NULL)
3011 			continue;
3012 		/* skip if not a virtual disk */
3013 		if (!(link->flags & SDEV_VIRTUAL))
3014 			continue;
3015 
3016 		vol++;
3017 	}
3018 	if (vol == 0)
3019 		return (0);
3020 
3021 	sc->sc_sensors = malloc(sizeof(struct ksensor) * vol,
3022 	    M_DEVBUF, M_WAITOK|M_ZERO);
3023 	if (sc->sc_sensors == NULL)
3024 		return (1);
3025 
3026 	strlcpy(sc->sc_sensordev.xname, DEVNAME(sc),
3027 	    sizeof(sc->sc_sensordev.xname));
3028 
3029 	for (i = 0, vol= 0; i < sc->sc_buswidth; i++) {
3030 		link = sc->sc_scsibus->sc_link[i][0];
3031 		if (link == NULL)
3032 			continue;
3033 		/* skip if not a virtual disk */
3034 		if (!(link->flags & SDEV_VIRTUAL))
3035 			continue;
3036 
3037 		dev = link->device_softc;
3038 		strlcpy(sc->sc_sensors[vol].desc, dev->dv_xname,
3039 		    sizeof(sc->sc_sensors[vol].desc));
3040 		sc->sc_sensors[vol].type = SENSOR_DRIVE;
3041 		sc->sc_sensors[vol].status = SENSOR_S_UNKNOWN;
3042 		sensor_attach(&sc->sc_sensordev, &sc->sc_sensors[vol]);
3043 
3044 		vol++;
3045 	}
3046 
3047 	if (sensor_task_register(sc, mpi_refresh_sensors, 10) == NULL)
3048 		goto bad;
3049 
3050 	sensordev_install(&sc->sc_sensordev);
3051 
3052 	return (0);
3053 
3054 bad:
3055 	free(sc->sc_sensors, M_DEVBUF);
3056 	return (1);
3057 }
3058 
3059 void
3060 mpi_refresh_sensors(void *arg)
3061 {
3062 	int			i, vol;
3063 	struct scsi_link	*link;
3064 	struct mpi_softc	*sc = arg;
3065 	struct mpi_cfg_raid_vol_pg0 *rpg0;
3066 
3067 	rw_enter_write(&sc->sc_lock);
3068 
3069 	for (i = 0, vol = 0; i < sc->sc_buswidth; i++) {
3070 		link = sc->sc_scsibus->sc_link[i][0];
3071 		if (link == NULL)
3072 			continue;
3073 		/* skip if not a virtual disk */
3074 		if (!(link->flags & SDEV_VIRTUAL))
3075 			continue;
3076 
3077 		if (mpi_bio_get_pg0_raid(sc, vol))
3078 			continue;
3079 
3080 		rpg0 = sc->sc_rpg0;
3081 		if (rpg0 == NULL)
3082 			goto done;
3083 
3084 		/* determine status */
3085 		switch (rpg0->volume_state) {
3086 		case MPI_CFG_RAID_VOL_0_STATE_OPTIMAL:
3087 			sc->sc_sensors[vol].value = SENSOR_DRIVE_ONLINE;
3088 			sc->sc_sensors[vol].status = SENSOR_S_OK;
3089 			break;
3090 		case MPI_CFG_RAID_VOL_0_STATE_DEGRADED:
3091 			sc->sc_sensors[vol].value = SENSOR_DRIVE_PFAIL;
3092 			sc->sc_sensors[vol].status = SENSOR_S_WARN;
3093 			break;
3094 		case MPI_CFG_RAID_VOL_0_STATE_FAILED:
3095 		case MPI_CFG_RAID_VOL_0_STATE_MISSING:
3096 			sc->sc_sensors[vol].value = SENSOR_DRIVE_FAIL;
3097 			sc->sc_sensors[vol].status = SENSOR_S_CRIT;
3098 			break;
3099 		default:
3100 			sc->sc_sensors[vol].value = 0; /* unknown */
3101 			sc->sc_sensors[vol].status = SENSOR_S_UNKNOWN;
3102 		}
3103 
3104 		/* override status if scrubbing or something */
3105 		if (rpg0->volume_status & MPI_CFG_RAID_VOL_0_STATUS_RESYNCING) {
3106 			sc->sc_sensors[vol].value = SENSOR_DRIVE_REBUILD;
3107 			sc->sc_sensors[vol].status = SENSOR_S_WARN;
3108 		}
3109 
3110 		vol++;
3111 	}
3112 done:
3113 	rw_exit_write(&sc->sc_lock);
3114 }
3115 #endif /* SMALL_KERNEL */
3116 #endif /* NBIO > 0 */
3117