195 	struct mfi_ccb		*ccb;  in mfi_get_ccb()  local
200 	ccb = SLIST_FIRST(&sc->sc_ccb_freeq);  in mfi_get_ccb()
201 	if (ccb != NULL) {  in mfi_get_ccb()
203 		ccb->ccb_state = MFI_CCB_READY;  in mfi_get_ccb()
207 	DNPRINTF(MFI_D_CCB, "%s: mfi_get_ccb: %p\n", DEVNAME(sc), ccb);  in mfi_get_ccb()
210 	return (ccb);  in mfi_get_ccb()
217 	struct mfi_ccb		*ccb = io;  in mfi_put_ccb()  local
219 	DNPRINTF(MFI_D_CCB, "%s: mfi_put_ccb: %p\n", DEVNAME(sc), ccb);  in mfi_put_ccb()
223 	SLIST_INSERT_HEAD(&sc->sc_ccb_freeq, ccb, ccb_link);  in mfi_put_ccb()
229 mfi_scrub_ccb(struct mfi_ccb *ccb)  in mfi_scrub_ccb()  argument
231 	struct mfi_frame_header	*hdr = &ccb->ccb_frame->mfr_header;  in mfi_scrub_ccb()
235 	ccb->ccb_state = MFI_CCB_FREE;  in mfi_scrub_ccb()
236 	ccb->ccb_cookie = NULL;  in mfi_scrub_ccb()
237 	ccb->ccb_flags = 0;  in mfi_scrub_ccb()
238 	ccb->ccb_done = NULL;  in mfi_scrub_ccb()
239 	ccb->ccb_direction = 0;  in mfi_scrub_ccb()
240 	ccb->ccb_frame_size = 0;  in mfi_scrub_ccb()
241 	ccb->ccb_extra_frames = 0;  in mfi_scrub_ccb()
242 	ccb->ccb_sgl = NULL;  in mfi_scrub_ccb()
243 	ccb->ccb_data = NULL;  in mfi_scrub_ccb()
244 	ccb->ccb_len = 0;  in mfi_scrub_ccb()
250 	struct mfi_ccb		*ccb;  in mfi_init_ccb()  local
260 		ccb = &sc->sc_ccb[i];  in mfi_init_ccb()
263 		ccb->ccb_frame = (union mfi_frame *)  in mfi_init_ccb()
265 		ccb->ccb_pframe =  in mfi_init_ccb()
267 		ccb->ccb_pframe_offset = sc->sc_frames_size * i;  in mfi_init_ccb()
268 		ccb->ccb_frame->mfr_header.mfh_context = i;  in mfi_init_ccb()
271 		ccb->ccb_sense = (struct mfi_sense *)  in mfi_init_ccb()
273 		ccb->ccb_psense =  in mfi_init_ccb()
279 		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &ccb->ccb_dmamap);  in mfi_init_ccb()
288 		    ccb->ccb_frame->mfr_header.mfh_context, ccb,  in mfi_init_ccb()
289 		    ccb->ccb_frame, ccb->ccb_pframe,  in mfi_init_ccb()
290 		    ccb->ccb_sense, ccb->ccb_psense,  in mfi_init_ccb()
291 		    ccb->ccb_dmamap);  in mfi_init_ccb()
294 		mfi_put_ccb(sc, ccb);  in mfi_init_ccb()
300 	while ((ccb = mfi_get_ccb(sc)) != NULL)  in mfi_init_ccb()
301 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);  in mfi_init_ccb()
454 	struct mfi_ccb		*ccb;  in mfi_initialize_firmware()  local
461 	ccb = scsi_io_get(&sc->sc_iopool, 0);  in mfi_initialize_firmware()
462 	mfi_scrub_ccb(ccb);  in mfi_initialize_firmware()
464 	init = &ccb->ccb_frame->mfr_init;  in mfi_initialize_firmware()
481 	init->mif_qinfo_new_addr = htole64(ccb->ccb_pframe + MFI_FRAME_SIZE);  in mfi_initialize_firmware()
487 	ccb->ccb_done = mfi_empty_done;  in mfi_initialize_firmware()
488 	mfi_poll(sc, ccb);  in mfi_initialize_firmware()
492 	mfi_put_ccb(sc, ccb);  in mfi_initialize_firmware()
498 mfi_empty_done(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_empty_done()  argument
878 mfi_poll(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_poll()  argument
885 	hdr = &ccb->ccb_frame->mfr_header;  in mfi_poll()
889 	mfi_start(sc, ccb);  in mfi_poll()
895 		    ccb->ccb_pframe_offset, sc->sc_frames_size,  in mfi_poll()
904 			ccb->ccb_flags |= MFI_CCB_F_ERR;  in mfi_poll()
909 		    ccb->ccb_pframe_offset, sc->sc_frames_size,  in mfi_poll()
913 	if (ccb->ccb_len > 0) {  in mfi_poll()
914 		bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 0,  in mfi_poll()
915 		    ccb->ccb_dmamap->dm_mapsize,  in mfi_poll()
916 		    (ccb->ccb_direction & MFI_DATA_IN) ?  in mfi_poll()
919 		bus_dmamap_unload(sc->sc_dmat, ccb->ccb_dmamap);  in mfi_poll()
922 	ccb->ccb_done(sc, ccb);  in mfi_poll()
926 mfi_exec(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_exec()  argument
933 	if (ccb->ccb_cookie != NULL || ccb->ccb_done != NULL)  in mfi_exec()
937 	ccb->ccb_cookie = &m;  in mfi_exec()
938 	ccb->ccb_done = mfi_exec_done;  in mfi_exec()
940 	mfi_start(sc, ccb);  in mfi_exec()
943 	while (ccb->ccb_cookie != NULL)  in mfi_exec()
944 		msleep_nsec(ccb, &m, PRIBIO, "mfiexec", INFSLP);  in mfi_exec()
949 mfi_exec_done(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_exec_done()  argument
951 	struct mutex *m = ccb->ccb_cookie;  in mfi_exec_done()
954 	ccb->ccb_cookie = NULL;  in mfi_exec_done()
955 	wakeup_one(ccb);  in mfi_exec_done()
964 	struct mfi_ccb		*ccb;  in mfi_intr()  local
991 			ccb = &sc->sc_ccb[ctx];  in mfi_intr()
994 			mfi_done(sc, ccb);  in mfi_intr()
1013 mfi_scsi_io(struct mfi_softc *sc, struct mfi_ccb *ccb,  in mfi_scsi_io()  argument
1025 	io = &ccb->ccb_frame->mfr_io;  in mfi_scsi_io()
1028 		ccb->ccb_direction = MFI_DATA_IN;  in mfi_scsi_io()
1031 		ccb->ccb_direction = MFI_DATA_OUT;  in mfi_scsi_io()
1039 	io->mif_sense_addr = htole64(ccb->ccb_psense);  in mfi_scsi_io()
1041 	ccb->ccb_done = mfi_scsi_xs_done;  in mfi_scsi_io()
1042 	ccb->ccb_cookie = xs;  in mfi_scsi_io()
1043 	ccb->ccb_frame_size = MFI_IO_FRAME_SIZE;  in mfi_scsi_io()
1044 	ccb->ccb_sgl = &io->mif_sgl;  in mfi_scsi_io()
1045 	ccb->ccb_data = xs->data;  in mfi_scsi_io()
1046 	ccb->ccb_len = xs->datalen;  in mfi_scsi_io()
1048 	if (mfi_create_sgl(sc, ccb, (xs->flags & SCSI_NOSLEEP) ?  in mfi_scsi_io()
1056 mfi_scsi_xs_done(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_scsi_xs_done()  argument
1058 	struct scsi_xfer	*xs = ccb->ccb_cookie;  in mfi_scsi_xs_done()
1059 	struct mfi_frame_header	*hdr = &ccb->ccb_frame->mfr_header;  in mfi_scsi_xs_done()
1062 	    DEVNAME(sc), ccb, ccb->ccb_frame);  in mfi_scsi_xs_done()
1073 		memcpy(&xs->sense, ccb->ccb_sense, sizeof(xs->sense));  in mfi_scsi_xs_done()
1090 			    &xs->sense, ccb->ccb_sense);  in mfi_scsi_xs_done()
1092 			memcpy(&xs->sense, ccb->ccb_sense,  in mfi_scsi_xs_done()
1105 mfi_scsi_ld(struct mfi_softc *sc, struct mfi_ccb *ccb, struct scsi_xfer *xs)  in mfi_scsi_ld()  argument
1113 	pf = &ccb->ccb_frame->mfr_pass;  in mfi_scsi_ld()
1122 	pf->mpf_sense_addr = htole64(ccb->ccb_psense);  in mfi_scsi_ld()
1127 	ccb->ccb_done = mfi_scsi_xs_done;  in mfi_scsi_ld()
1128 	ccb->ccb_cookie = xs;  in mfi_scsi_ld()
1129 	ccb->ccb_frame_size = MFI_PASS_FRAME_SIZE;  in mfi_scsi_ld()
1130 	ccb->ccb_sgl = &pf->mpf_sgl;  in mfi_scsi_ld()
1133 		ccb->ccb_direction = xs->flags & SCSI_DATA_IN ?  in mfi_scsi_ld()
1136 		ccb->ccb_direction = MFI_DATA_NONE;  in mfi_scsi_ld()
1139 		ccb->ccb_data = xs->data;  in mfi_scsi_ld()
1140 		ccb->ccb_len = xs->datalen;  in mfi_scsi_ld()
1142 		if (mfi_create_sgl(sc, ccb, (xs->flags & SCSI_NOSLEEP) ?  in mfi_scsi_ld()
1155 	struct mfi_ccb		*ccb = xs->io;  in mfi_scsi_cmd()  local
1175 	mfi_scrub_ccb(ccb);  in mfi_scsi_cmd()
1186 		if (mfi_scsi_io(sc, ccb, xs, blockno, blockcnt))  in mfi_scsi_cmd()
1196 		if (mfi_scsi_io(sc, ccb, xs, blockno, blockcnt))  in mfi_scsi_cmd()
1205 		if (mfi_scsi_io(sc, ccb, xs, blockno, blockcnt))  in mfi_scsi_cmd()
1211 		if (mfi_do_mgmt(sc, ccb, MR_DCMD_CTRL_CACHE_FLUSH,  in mfi_scsi_cmd()
1219 		if (mfi_scsi_ld(sc, ccb, xs))  in mfi_scsi_cmd()
1227 		mfi_poll(sc, ccb);  in mfi_scsi_cmd()
1229 		mfi_start(sc, ccb);  in mfi_scsi_cmd()
1242 mfi_default_sgd_load(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_default_sgd_load()  argument
1244 	struct mfi_frame_header	*hdr = &ccb->ccb_frame->mfr_header;  in mfi_default_sgd_load()
1245 	union mfi_sgl		*sgl = ccb->ccb_sgl;  in mfi_default_sgd_load()
1246 	bus_dma_segment_t	*sgd = ccb->ccb_dmamap->dm_segs;  in mfi_default_sgd_load()
1251 	for (i = 0; i < ccb->ccb_dmamap->dm_nsegs; i++) {  in mfi_default_sgd_load()
1265 	return (ccb->ccb_dmamap->dm_nsegs *  in mfi_default_sgd_load()
1270 mfi_create_sgl(struct mfi_softc *sc, struct mfi_ccb *ccb, int flags)  in mfi_create_sgl()  argument
1272 	struct mfi_frame_header	*hdr = &ccb->ccb_frame->mfr_header;  in mfi_create_sgl()
1276 	    ccb->ccb_data);  in mfi_create_sgl()
1278 	if (!ccb->ccb_data) {  in mfi_create_sgl()
1283 	error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap,  in mfi_create_sgl()
1284 	    ccb->ccb_data, ccb->ccb_len, NULL, flags);  in mfi_create_sgl()
1294 	ccb->ccb_frame_size += mfi_sgd_load(sc, ccb);  in mfi_create_sgl()
1296 	if (ccb->ccb_direction == MFI_DATA_IN) {  in mfi_create_sgl()
1298 		bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 0,  in mfi_create_sgl()
1299 		    ccb->ccb_dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);  in mfi_create_sgl()
1302 		bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 0,  in mfi_create_sgl()
1303 		    ccb->ccb_dmamap->dm_mapsize, BUS_DMASYNC_PREWRITE);  in mfi_create_sgl()
1306 	hdr->mfh_sg_count = ccb->ccb_dmamap->dm_nsegs;  in mfi_create_sgl()
1307 	ccb->ccb_extra_frames = (ccb->ccb_frame_size - 1) / MFI_FRAME_SIZE;  in mfi_create_sgl()
1313 	    ccb->ccb_frame_size,  in mfi_create_sgl()
1315 	    ccb->ccb_dmamap->dm_nsegs,  in mfi_create_sgl()
1316 	    ccb->ccb_extra_frames);  in mfi_create_sgl()
1325 	struct mfi_ccb *ccb;  in mfi_mgmt()  local
1328 	ccb = scsi_io_get(&sc->sc_iopool, 0);  in mfi_mgmt()
1329 	mfi_scrub_ccb(ccb);  in mfi_mgmt()
1330 	rv = mfi_do_mgmt(sc, ccb, opc, dir, len, buf, mbox);  in mfi_mgmt()
1331 	scsi_io_put(&sc->sc_iopool, ccb);  in mfi_mgmt()
1337 mfi_do_mgmt(struct mfi_softc *sc, struct mfi_ccb *ccb, uint32_t opc,  in mfi_do_mgmt()  argument
1350 	dcmd = &ccb->ccb_frame->mfr_dcmd;  in mfi_do_mgmt()
1357 	ccb->ccb_direction = dir;  in mfi_do_mgmt()
1359 	ccb->ccb_frame_size = MFI_DCMD_FRAME_SIZE;  in mfi_do_mgmt()
1369 		ccb->ccb_data = dma_buf;  in mfi_do_mgmt()
1370 		ccb->ccb_len = len;  in mfi_do_mgmt()
1371 		ccb->ccb_sgl = &dcmd->mdf_sgl;  in mfi_do_mgmt()
1373 		if (mfi_create_sgl(sc, ccb, cold ? BUS_DMA_NOWAIT :  in mfi_do_mgmt()
1381 		ccb->ccb_done = mfi_empty_done;  in mfi_do_mgmt()
1382 		mfi_poll(sc, ccb);  in mfi_do_mgmt()
1384 		mfi_exec(sc, ccb);  in mfi_do_mgmt()
2521 mfi_start(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_start()  argument
2524 	    ccb->ccb_pframe_offset, sc->sc_frames_size,  in mfi_start()
2527 	mfi_post(sc, ccb);  in mfi_start()
2531 mfi_done(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_done()  argument
2534 	    ccb->ccb_pframe_offset, sc->sc_frames_size,  in mfi_done()
2537 	if (ccb->ccb_len > 0) {  in mfi_done()
2538 		bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap,  in mfi_done()
2539 		    0, ccb->ccb_dmamap->dm_mapsize,  in mfi_done()
2540 		    (ccb->ccb_direction == MFI_DATA_IN) ?  in mfi_done()
2543 		bus_dmamap_unload(sc->sc_dmat, ccb->ccb_dmamap);  in mfi_done()
2546 	ccb->ccb_done(sc, ccb);  in mfi_done()
2577 mfi_xscale_post(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_xscale_post()  argument
2579 	mfi_write(sc, MFI_IQP, (ccb->ccb_pframe >> 3) |  in mfi_xscale_post()
2580 	    ccb->ccb_extra_frames);  in mfi_xscale_post()
2612 mfi_ppc_post(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_ppc_post()  argument
2614 	mfi_write(sc, MFI_IQP, 0x1 | ccb->ccb_pframe |  in mfi_ppc_post()
2615 	    (ccb->ccb_extra_frames << 1));  in mfi_ppc_post()
2647 mfi_gen2_post(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_gen2_post()  argument
2649 	mfi_write(sc, MFI_IQP, 0x1 | ccb->ccb_pframe |  in mfi_gen2_post()
2650 	    (ccb->ccb_extra_frames << 1));  in mfi_gen2_post()
2681 mfi_skinny_post(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_skinny_post()  argument
2683 	mfi_write(sc, MFI_IQPL, 0x1 | ccb->ccb_pframe |  in mfi_skinny_post()
2684 	    (ccb->ccb_extra_frames << 1));  in mfi_skinny_post()
2689 mfi_skinny_sgd_load(struct mfi_softc *sc, struct mfi_ccb *ccb)  in mfi_skinny_sgd_load()  argument
2691 	struct mfi_frame_header	*hdr = &ccb->ccb_frame->mfr_header;  in mfi_skinny_sgd_load()
2692 	union mfi_sgl		*sgl = ccb->ccb_sgl;  in mfi_skinny_sgd_load()
2693 	bus_dma_segment_t	*sgd = ccb->ccb_dmamap->dm_segs;  in mfi_skinny_sgd_load()
2701 		for (i = 0; i < ccb->ccb_dmamap->dm_nsegs; i++) {  in mfi_skinny_sgd_load()
2708 		return (ccb->ccb_dmamap->dm_nsegs * sizeof(sgl->sg_skinny));  in mfi_skinny_sgd_load()
2710 		return (mfi_default_sgd_load(sc, ccb));  in mfi_skinny_sgd_load()
2745 	struct mfi_ccb *ccb = xs->io;  in mfi_pd_scsi_cmd()  local
2746 	struct mfi_pass_frame *pf = &ccb->ccb_frame->mfr_pass;  in mfi_pd_scsi_cmd()
2751 	mfi_scrub_ccb(ccb);  in mfi_pd_scsi_cmd()
2761 	pf->mpf_sense_addr = htole64(ccb->ccb_psense);  in mfi_pd_scsi_cmd()
2766 	ccb->ccb_done = mfi_scsi_xs_done;  in mfi_pd_scsi_cmd()
2767 	ccb->ccb_cookie = xs;  in mfi_pd_scsi_cmd()
2768 	ccb->ccb_frame_size = MFI_PASS_FRAME_SIZE;  in mfi_pd_scsi_cmd()
2769 	ccb->ccb_sgl = &pf->mpf_sgl;  in mfi_pd_scsi_cmd()
2772 		ccb->ccb_direction = xs->flags & SCSI_DATA_IN ?  in mfi_pd_scsi_cmd()
2775 		ccb->ccb_direction = MFI_DATA_NONE;  in mfi_pd_scsi_cmd()
2778 		ccb->ccb_data = xs->data;  in mfi_pd_scsi_cmd()
2779 		ccb->ccb_len = xs->datalen;  in mfi_pd_scsi_cmd()
2781 		if (mfi_create_sgl(sc, ccb, (xs->flags & SCSI_NOSLEEP) ?  in mfi_pd_scsi_cmd()
2787 		mfi_poll(sc, ccb);  in mfi_pd_scsi_cmd()
2789 		mfi_start(sc, ccb);  in mfi_pd_scsi_cmd()