11709Smlf /*
21709Smlf * CDDL HEADER START
31709Smlf *
41709Smlf * The contents of this file are subject to the terms of the
51709Smlf * Common Development and Distribution License, Version 1.0 only
61709Smlf * (the "License"). You may not use this file except in compliance
71709Smlf * with the License.
81709Smlf *
91709Smlf * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
101709Smlf * or http://www.opensolaris.org/os/licensing.
111709Smlf * See the License for the specific language governing permissions
121709Smlf * and limitations under the License.
131709Smlf *
141709Smlf * When distributing Covered Code, include this CDDL HEADER in each
151709Smlf * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
161709Smlf * If applicable, add the following below this CDDL HEADER, with the
171709Smlf * fields enclosed by brackets "[]" replaced with your own identifying
181709Smlf * information: Portions Copyright [yyyy] [name of copyright owner]
191709Smlf *
201709Smlf * CDDL HEADER END
211709Smlf */
221709Smlf /*
23*7542SRichard.Bean@Sun.COM * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
241709Smlf * Use is subject to license terms.
251709Smlf */
261709Smlf
271709Smlf /*
281709Smlf * Device Strategy
291709Smlf */
301709Smlf #include <sys/dktp/cm.h>
311709Smlf #include <sys/kstat.h>
321709Smlf
331709Smlf #include <sys/dktp/quetypes.h>
341709Smlf #include <sys/dktp/queue.h>
351709Smlf #include <sys/dktp/tgcom.h>
361709Smlf #include <sys/dktp/fctypes.h>
371709Smlf #include <sys/dktp/flowctrl.h>
381709Smlf #include <sys/param.h>
391709Smlf #include <vm/page.h>
401709Smlf #include <sys/modctl.h>
411709Smlf
421709Smlf /*
431709Smlf * Object Management
441709Smlf */
451709Smlf
461709Smlf static struct buf *qmerge_nextbp(struct que_data *qfp, struct buf *bp_merge,
471709Smlf int *can_merge);
481709Smlf
491709Smlf static struct modlmisc modlmisc = {
501709Smlf &mod_miscops, /* Type of module */
51*7542SRichard.Bean@Sun.COM "Device Strategy Objects"
521709Smlf };
531709Smlf
541709Smlf static struct modlinkage modlinkage = {
551709Smlf MODREV_1,
561709Smlf &modlmisc,
571709Smlf NULL
581709Smlf };
591709Smlf
601709Smlf int
_init(void)611709Smlf _init(void)
621709Smlf {
631709Smlf return (mod_install(&modlinkage));
641709Smlf }
651709Smlf
661709Smlf int
_fini(void)671709Smlf _fini(void)
681709Smlf {
691709Smlf return (mod_remove(&modlinkage));
701709Smlf }
711709Smlf
721709Smlf int
_info(struct modinfo * modinfop)731709Smlf _info(struct modinfo *modinfop)
741709Smlf {
751709Smlf return (mod_info(&modlinkage, modinfop));
761709Smlf }
771709Smlf
781709Smlf
791709Smlf /*
801709Smlf * Common Flow Control functions
811709Smlf */
821709Smlf
831709Smlf /*
841709Smlf * Local static data
851709Smlf */
861709Smlf #ifdef FLC_DEBUG
871709Smlf #define DENT 0x0001
881709Smlf #define DERR 0x0002
891709Smlf #define DIO 0x0004
901709Smlf static int flc_debug = DENT|DERR|DIO;
911709Smlf
921709Smlf #include <sys/thread.h>
931709Smlf static int flc_malloc_intr = 0;
941709Smlf #endif /* FLC_DEBUG */
951709Smlf
961709Smlf static int flc_kstat = 1;
971709Smlf
981709Smlf static struct flc_obj *fc_create(struct flc_objops *fcopsp);
991709Smlf static int fc_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp,
1001709Smlf void *lkarg);
1011709Smlf static int fc_free(struct flc_obj *flcobjp);
1021709Smlf static int fc_start_kstat(opaque_t queuep, char *devtype, int instance);
1031709Smlf static int fc_stop_kstat(opaque_t queuep);
1041709Smlf
1051709Smlf static struct flc_obj *
fc_create(struct flc_objops * fcopsp)1061709Smlf fc_create(struct flc_objops *fcopsp)
1071709Smlf {
1081709Smlf struct flc_obj *flcobjp;
1091709Smlf struct fc_data *fcdp;
1101709Smlf
1111709Smlf flcobjp = kmem_zalloc((sizeof (*flcobjp) + sizeof (*fcdp)), KM_NOSLEEP);
1121709Smlf if (!flcobjp)
1131709Smlf return (NULL);
1141709Smlf
1151709Smlf fcdp = (struct fc_data *)(flcobjp+1);
1161709Smlf flcobjp->flc_data = (opaque_t)fcdp;
1171709Smlf flcobjp->flc_ops = fcopsp;
1181709Smlf
1191709Smlf return ((opaque_t)flcobjp);
1201709Smlf }
1211709Smlf
1221709Smlf static int dmult_maxcnt = DMULT_MAXCNT;
1231709Smlf
1241709Smlf static int
fc_init(opaque_t queuep,opaque_t tgcom_objp,opaque_t que_objp,void * lkarg)1251709Smlf fc_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, void *lkarg)
1261709Smlf {
1271709Smlf struct fc_data *fcdp = (struct fc_data *)queuep;
1281709Smlf
1291709Smlf mutex_init(&fcdp->ds_mutex, NULL, MUTEX_DRIVER, lkarg);
1301709Smlf
1311709Smlf fcdp->ds_queobjp = que_objp;
1321709Smlf fcdp->ds_tgcomobjp = tgcom_objp;
1331709Smlf fcdp->ds_waitcnt = dmult_maxcnt;
1341709Smlf
1351709Smlf QUE_INIT(que_objp, lkarg);
1361709Smlf TGCOM_INIT(tgcom_objp);
1371709Smlf return (DDI_SUCCESS);
1381709Smlf }
1391709Smlf
1401709Smlf static int
fc_free(struct flc_obj * flcobjp)1411709Smlf fc_free(struct flc_obj *flcobjp)
1421709Smlf {
1431709Smlf struct fc_data *fcdp;
1441709Smlf
1451709Smlf fcdp = (struct fc_data *)flcobjp->flc_data;
1461709Smlf if (fcdp->ds_queobjp)
1471709Smlf QUE_FREE(fcdp->ds_queobjp);
1481709Smlf if (fcdp->ds_tgcomobjp) {
1491709Smlf TGCOM_FREE(fcdp->ds_tgcomobjp);
1501709Smlf mutex_destroy(&fcdp->ds_mutex);
1511709Smlf }
1521709Smlf kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp)));
1531709Smlf return (0);
1541709Smlf }
1551709Smlf
1561709Smlf /*ARGSUSED*/
1571709Smlf static int
fc_start_kstat(opaque_t queuep,char * devtype,int instance)1581709Smlf fc_start_kstat(opaque_t queuep, char *devtype, int instance)
1591709Smlf {
1601709Smlf struct fc_data *fcdp = (struct fc_data *)queuep;
1611709Smlf if (!flc_kstat)
1621709Smlf return (0);
1631709Smlf
1641709Smlf if (!fcdp->ds_kstat) {
1651709Smlf if (fcdp->ds_kstat = kstat_create("cmdk", instance, NULL,
1661709Smlf "disk", KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT)) {
1671709Smlf kstat_install(fcdp->ds_kstat);
1681709Smlf }
1691709Smlf }
1701709Smlf return (0);
1711709Smlf }
1721709Smlf
1731709Smlf static int
fc_stop_kstat(opaque_t queuep)1741709Smlf fc_stop_kstat(opaque_t queuep)
1751709Smlf {
1761709Smlf struct fc_data *fcdp = (struct fc_data *)queuep;
1771709Smlf
1781709Smlf if (fcdp->ds_kstat) {
1791709Smlf kstat_delete(fcdp->ds_kstat);
1801709Smlf fcdp->ds_kstat = NULL;
1811709Smlf }
1821709Smlf return (0);
1831709Smlf }
1841709Smlf
1851709Smlf
1861709Smlf /*
1871709Smlf * Single Command per Device
1881709Smlf */
1891709Smlf /*
1901709Smlf * Local Function Prototypes
1911709Smlf */
1921709Smlf static int dsngl_restart();
1931709Smlf
1941709Smlf static int dsngl_enque(opaque_t, struct buf *);
1951709Smlf static int dsngl_deque(opaque_t, struct buf *);
1961709Smlf
1971709Smlf struct flc_objops dsngl_ops = {
1981709Smlf fc_init,
1991709Smlf fc_free,
2001709Smlf dsngl_enque,
2011709Smlf dsngl_deque,
2021709Smlf fc_start_kstat,
2031709Smlf fc_stop_kstat,
2041709Smlf 0, 0
2051709Smlf };
2061709Smlf
2071709Smlf struct flc_obj *
dsngl_create()2081709Smlf dsngl_create()
2091709Smlf {
2101709Smlf return (fc_create((struct flc_objops *)&dsngl_ops));
2111709Smlf }
2121709Smlf
2131709Smlf static int
dsngl_enque(opaque_t queuep,struct buf * in_bp)2141709Smlf dsngl_enque(opaque_t queuep, struct buf *in_bp)
2151709Smlf {
2161709Smlf struct fc_data *dsnglp = (struct fc_data *)queuep;
2171709Smlf opaque_t tgcom_objp;
2181709Smlf opaque_t que_objp;
2191709Smlf
2201709Smlf que_objp = dsnglp->ds_queobjp;
2211709Smlf tgcom_objp = dsnglp->ds_tgcomobjp;
2221709Smlf
2231709Smlf if (!in_bp)
2241709Smlf return (0);
2251709Smlf mutex_enter(&dsnglp->ds_mutex);
2261709Smlf if (dsnglp->ds_bp || dsnglp->ds_outcnt) {
2271709Smlf QUE_ADD(que_objp, in_bp);
2281709Smlf if (dsnglp->ds_kstat) {
2291709Smlf kstat_waitq_enter(KSTAT_IO_PTR(dsnglp->ds_kstat));
2301709Smlf }
2311709Smlf mutex_exit(&dsnglp->ds_mutex);
2321709Smlf return (0);
2331709Smlf }
2341709Smlf if (dsnglp->ds_kstat) {
2351709Smlf kstat_waitq_enter(KSTAT_IO_PTR(dsnglp->ds_kstat));
2361709Smlf }
2371709Smlf if (TGCOM_PKT(tgcom_objp, in_bp, dsngl_restart,
2381709Smlf (caddr_t)dsnglp) != DDI_SUCCESS) {
2391709Smlf
2401709Smlf dsnglp->ds_bp = in_bp;
2411709Smlf mutex_exit(&dsnglp->ds_mutex);
2421709Smlf return (0);
2431709Smlf }
2441709Smlf dsnglp->ds_outcnt++;
2451709Smlf if (dsnglp->ds_kstat)
2461709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(dsnglp->ds_kstat));
2471709Smlf mutex_exit(&dsnglp->ds_mutex);
2481709Smlf TGCOM_TRANSPORT(tgcom_objp, in_bp);
2491709Smlf return (0);
2501709Smlf }
2511709Smlf
2521709Smlf static int
dsngl_deque(opaque_t queuep,struct buf * in_bp)2531709Smlf dsngl_deque(opaque_t queuep, struct buf *in_bp)
2541709Smlf {
2551709Smlf struct fc_data *dsnglp = (struct fc_data *)queuep;
2561709Smlf opaque_t tgcom_objp;
2571709Smlf opaque_t que_objp;
2581709Smlf struct buf *bp;
2591709Smlf
2601709Smlf que_objp = dsnglp->ds_queobjp;
2611709Smlf tgcom_objp = dsnglp->ds_tgcomobjp;
2621709Smlf
2631709Smlf mutex_enter(&dsnglp->ds_mutex);
2641709Smlf if (in_bp) {
2651709Smlf dsnglp->ds_outcnt--;
2661709Smlf if (dsnglp->ds_kstat) {
2671709Smlf if (in_bp->b_flags & B_READ) {
2681709Smlf KSTAT_IO_PTR(dsnglp->ds_kstat)->reads++;
2691709Smlf KSTAT_IO_PTR(dsnglp->ds_kstat)->nread +=
2701709Smlf (in_bp->b_bcount - in_bp->b_resid);
2711709Smlf } else {
2721709Smlf KSTAT_IO_PTR(dsnglp->ds_kstat)->writes++;
2731709Smlf KSTAT_IO_PTR(dsnglp->ds_kstat)->nwritten +=
2741709Smlf (in_bp->b_bcount - in_bp->b_resid);
2751709Smlf }
2761709Smlf kstat_runq_exit(KSTAT_IO_PTR(dsnglp->ds_kstat));
2771709Smlf }
2781709Smlf }
2791709Smlf for (;;) {
2801709Smlf if (!dsnglp->ds_bp)
2811709Smlf dsnglp->ds_bp = QUE_DEL(que_objp);
2821709Smlf if (!dsnglp->ds_bp ||
2831709Smlf (TGCOM_PKT(tgcom_objp, dsnglp->ds_bp, dsngl_restart,
2841709Smlf (caddr_t)dsnglp) != DDI_SUCCESS) ||
2851709Smlf dsnglp->ds_outcnt) {
2861709Smlf mutex_exit(&dsnglp->ds_mutex);
2871709Smlf return (0);
2881709Smlf }
2891709Smlf dsnglp->ds_outcnt++;
2901709Smlf bp = dsnglp->ds_bp;
2911709Smlf dsnglp->ds_bp = QUE_DEL(que_objp);
2921709Smlf if (dsnglp->ds_kstat)
2931709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(dsnglp->ds_kstat));
2941709Smlf mutex_exit(&dsnglp->ds_mutex);
2951709Smlf
2961709Smlf TGCOM_TRANSPORT(tgcom_objp, bp);
2971709Smlf
2981709Smlf if (!mutex_tryenter(&dsnglp->ds_mutex))
2991709Smlf return (0);
3001709Smlf }
3011709Smlf }
3021709Smlf
3031709Smlf static int
dsngl_restart(struct fc_data * dsnglp)3041709Smlf dsngl_restart(struct fc_data *dsnglp)
3051709Smlf {
3061709Smlf (void) dsngl_deque(dsnglp, NULL);
3071709Smlf return (-1);
3081709Smlf }
3091709Smlf
3101709Smlf
3111709Smlf /*
3121709Smlf * Multiple Commands per Device
3131709Smlf */
3141709Smlf /*
3151709Smlf * Local Function Prototypes
3161709Smlf */
3171709Smlf static int dmult_restart();
3181709Smlf
3191709Smlf static int dmult_enque(opaque_t, struct buf *);
3201709Smlf static int dmult_deque(opaque_t, struct buf *);
3211709Smlf
3221709Smlf struct flc_objops dmult_ops = {
3231709Smlf fc_init,
3241709Smlf fc_free,
3251709Smlf dmult_enque,
3261709Smlf dmult_deque,
3271709Smlf fc_start_kstat,
3281709Smlf fc_stop_kstat,
3291709Smlf 0, 0
3301709Smlf };
3311709Smlf
3321709Smlf struct flc_obj *
dmult_create()3331709Smlf dmult_create()
3341709Smlf {
3351709Smlf return (fc_create((struct flc_objops *)&dmult_ops));
3361709Smlf
3371709Smlf }
3381709Smlf
3391709Smlf
3401709Smlf /*
3411709Smlf * Some of the object management functions QUE_ADD() and QUE_DEL()
3421709Smlf * do not accquire lock.
3431709Smlf * They depend on dmult_enque(), dmult_deque() to do all locking.
3441709Smlf * If this changes we have to grab locks in qmerge_add() and qmerge_del().
3451709Smlf */
3461709Smlf static int
dmult_enque(opaque_t queuep,struct buf * in_bp)3471709Smlf dmult_enque(opaque_t queuep, struct buf *in_bp)
3481709Smlf {
3491709Smlf struct fc_data *dmultp = (struct fc_data *)queuep;
3501709Smlf opaque_t tgcom_objp;
3511709Smlf opaque_t que_objp;
3521709Smlf
3531709Smlf que_objp = dmultp->ds_queobjp;
3541709Smlf tgcom_objp = dmultp->ds_tgcomobjp;
3551709Smlf
3561709Smlf if (!in_bp)
3571709Smlf return (0);
3581709Smlf mutex_enter(&dmultp->ds_mutex);
3591709Smlf if ((dmultp->ds_outcnt >= dmultp->ds_waitcnt) || dmultp->ds_bp) {
3601709Smlf QUE_ADD(que_objp, in_bp);
3611709Smlf if (dmultp->ds_kstat) {
3621709Smlf kstat_waitq_enter(KSTAT_IO_PTR(dmultp->ds_kstat));
3631709Smlf }
3641709Smlf mutex_exit(&dmultp->ds_mutex);
3651709Smlf return (0);
3661709Smlf }
3671709Smlf if (dmultp->ds_kstat) {
3681709Smlf kstat_waitq_enter(KSTAT_IO_PTR(dmultp->ds_kstat));
3691709Smlf }
3701709Smlf
3711709Smlf if (TGCOM_PKT(tgcom_objp, in_bp, dmult_restart,
3721709Smlf (caddr_t)dmultp) != DDI_SUCCESS) {
3731709Smlf
3741709Smlf dmultp->ds_bp = in_bp;
3751709Smlf mutex_exit(&dmultp->ds_mutex);
3761709Smlf return (0);
3771709Smlf }
3781709Smlf dmultp->ds_outcnt++;
3791709Smlf if (dmultp->ds_kstat)
3801709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(dmultp->ds_kstat));
3811709Smlf mutex_exit(&dmultp->ds_mutex);
3821709Smlf
3831709Smlf TGCOM_TRANSPORT(tgcom_objp, in_bp);
3841709Smlf return (0);
3851709Smlf }
3861709Smlf
3871709Smlf static int
dmult_deque(opaque_t queuep,struct buf * in_bp)3881709Smlf dmult_deque(opaque_t queuep, struct buf *in_bp)
3891709Smlf {
3901709Smlf struct fc_data *dmultp = (struct fc_data *)queuep;
3911709Smlf opaque_t tgcom_objp;
3921709Smlf opaque_t que_objp;
3931709Smlf struct buf *bp;
3941709Smlf
3951709Smlf que_objp = dmultp->ds_queobjp;
3961709Smlf tgcom_objp = dmultp->ds_tgcomobjp;
3971709Smlf
3981709Smlf mutex_enter(&dmultp->ds_mutex);
3991709Smlf if (in_bp) {
4001709Smlf dmultp->ds_outcnt--;
4011709Smlf if (dmultp->ds_kstat) {
4021709Smlf if (in_bp->b_flags & B_READ) {
4031709Smlf KSTAT_IO_PTR(dmultp->ds_kstat)->reads++;
4041709Smlf KSTAT_IO_PTR(dmultp->ds_kstat)->nread +=
4051709Smlf (in_bp->b_bcount - in_bp->b_resid);
4061709Smlf } else {
4071709Smlf KSTAT_IO_PTR(dmultp->ds_kstat)->writes++;
4081709Smlf KSTAT_IO_PTR(dmultp->ds_kstat)->nwritten +=
4091709Smlf (in_bp->b_bcount - in_bp->b_resid);
4101709Smlf }
4111709Smlf kstat_runq_exit(KSTAT_IO_PTR(dmultp->ds_kstat));
4121709Smlf }
4131709Smlf }
4141709Smlf
4151709Smlf for (;;) {
4161709Smlf
4171709Smlf #ifdef FLC_DEBUG
4181709Smlf if ((curthread->t_intr) && (!dmultp->ds_bp) &&
4191709Smlf (!dmultp->ds_outcnt))
4201709Smlf flc_malloc_intr++;
4211709Smlf #endif
4221709Smlf
4231709Smlf if (!dmultp->ds_bp)
4241709Smlf dmultp->ds_bp = QUE_DEL(que_objp);
4251709Smlf if (!dmultp->ds_bp ||
4261709Smlf (TGCOM_PKT(tgcom_objp, dmultp->ds_bp, dmult_restart,
4271709Smlf (caddr_t)dmultp) != DDI_SUCCESS) ||
4281709Smlf (dmultp->ds_outcnt >= dmultp->ds_waitcnt)) {
4291709Smlf mutex_exit(&dmultp->ds_mutex);
4301709Smlf return (0);
4311709Smlf }
4321709Smlf dmultp->ds_outcnt++;
4331709Smlf bp = dmultp->ds_bp;
4341709Smlf dmultp->ds_bp = QUE_DEL(que_objp);
4351709Smlf
4361709Smlf if (dmultp->ds_kstat)
4371709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(dmultp->ds_kstat));
4381709Smlf
4391709Smlf mutex_exit(&dmultp->ds_mutex);
4401709Smlf
4411709Smlf TGCOM_TRANSPORT(tgcom_objp, bp);
4421709Smlf
4431709Smlf if (!mutex_tryenter(&dmultp->ds_mutex))
4441709Smlf return (0);
4451709Smlf }
4461709Smlf }
4471709Smlf
4481709Smlf static int
dmult_restart(struct fc_data * dmultp)4491709Smlf dmult_restart(struct fc_data *dmultp)
4501709Smlf {
4511709Smlf (void) dmult_deque(dmultp, NULL);
4521709Smlf return (-1);
4531709Smlf }
4541709Smlf
4551709Smlf /*
4561709Smlf * Duplexed Commands per Device: Read Queue and Write Queue
4571709Smlf */
4581709Smlf /*
4591709Smlf * Local Function Prototypes
4601709Smlf */
4611709Smlf static int duplx_restart();
4621709Smlf
4631709Smlf static int duplx_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp,
4641709Smlf void *lkarg);
4651709Smlf static int duplx_free(struct flc_obj *flcobjp);
4661709Smlf static int duplx_enque(opaque_t queuep, struct buf *bp);
4671709Smlf static int duplx_deque(opaque_t queuep, struct buf *bp);
4681709Smlf
4691709Smlf struct flc_objops duplx_ops = {
4701709Smlf duplx_init,
4711709Smlf duplx_free,
4721709Smlf duplx_enque,
4731709Smlf duplx_deque,
4741709Smlf fc_start_kstat,
4751709Smlf fc_stop_kstat,
4761709Smlf 0, 0
4771709Smlf };
4781709Smlf
4791709Smlf struct flc_obj *
duplx_create()4801709Smlf duplx_create()
4811709Smlf {
4821709Smlf struct flc_obj *flcobjp;
4831709Smlf struct duplx_data *fcdp;
4841709Smlf
4851709Smlf flcobjp = kmem_zalloc((sizeof (*flcobjp) + sizeof (*fcdp)), KM_NOSLEEP);
4861709Smlf if (!flcobjp)
4871709Smlf return (NULL);
4881709Smlf
4891709Smlf fcdp = (struct duplx_data *)(flcobjp+1);
4901709Smlf flcobjp->flc_data = (opaque_t)fcdp;
4911709Smlf flcobjp->flc_ops = &duplx_ops;
4921709Smlf
4931709Smlf fcdp->ds_writeq.fc_qobjp = qfifo_create();
4941709Smlf if (!(fcdp->ds_writeq.fc_qobjp = qfifo_create())) {
4951709Smlf kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp)));
4961709Smlf return (NULL);
4971709Smlf }
4981709Smlf return (flcobjp);
4991709Smlf }
5001709Smlf
5011709Smlf static int
duplx_free(struct flc_obj * flcobjp)5021709Smlf duplx_free(struct flc_obj *flcobjp)
5031709Smlf {
5041709Smlf struct duplx_data *fcdp;
5051709Smlf
5061709Smlf fcdp = (struct duplx_data *)flcobjp->flc_data;
5071709Smlf if (fcdp->ds_writeq.fc_qobjp) {
5081709Smlf QUE_FREE(fcdp->ds_writeq.fc_qobjp);
5091709Smlf }
5101709Smlf if (fcdp->ds_readq.fc_qobjp)
5111709Smlf QUE_FREE(fcdp->ds_readq.fc_qobjp);
5121709Smlf if (fcdp->ds_tgcomobjp) {
5131709Smlf TGCOM_FREE(fcdp->ds_tgcomobjp);
5141709Smlf mutex_destroy(&fcdp->ds_mutex);
5151709Smlf }
5161709Smlf kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp)));
5171709Smlf return (0);
5181709Smlf }
5191709Smlf
5201709Smlf static int
duplx_init(opaque_t queuep,opaque_t tgcom_objp,opaque_t que_objp,void * lkarg)5211709Smlf duplx_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, void *lkarg)
5221709Smlf {
5231709Smlf struct duplx_data *fcdp = (struct duplx_data *)queuep;
5241709Smlf fcdp->ds_tgcomobjp = tgcom_objp;
5251709Smlf fcdp->ds_readq.fc_qobjp = que_objp;
5261709Smlf
5271709Smlf QUE_INIT(que_objp, lkarg);
5281709Smlf QUE_INIT(fcdp->ds_writeq.fc_qobjp, lkarg);
5291709Smlf TGCOM_INIT(tgcom_objp);
5301709Smlf
5311709Smlf mutex_init(&fcdp->ds_mutex, NULL, MUTEX_DRIVER, lkarg);
5321709Smlf
5331709Smlf fcdp->ds_writeq.fc_maxcnt = DUPLX_MAXCNT;
5341709Smlf fcdp->ds_readq.fc_maxcnt = DUPLX_MAXCNT;
5351709Smlf
5361709Smlf /* queues point to each other for round robin */
5371709Smlf fcdp->ds_readq.next = &fcdp->ds_writeq;
5381709Smlf fcdp->ds_writeq.next = &fcdp->ds_readq;
5391709Smlf
5401709Smlf return (DDI_SUCCESS);
5411709Smlf }
5421709Smlf
5431709Smlf static int
duplx_enque(opaque_t queuep,struct buf * in_bp)5441709Smlf duplx_enque(opaque_t queuep, struct buf *in_bp)
5451709Smlf {
5461709Smlf struct duplx_data *duplxp = (struct duplx_data *)queuep;
5471709Smlf opaque_t tgcom_objp;
5481709Smlf struct fc_que *activeq;
5491709Smlf struct buf *bp;
5501709Smlf
5511709Smlf mutex_enter(&duplxp->ds_mutex);
5521709Smlf if (in_bp) {
5531709Smlf if (duplxp->ds_kstat) {
5541709Smlf kstat_waitq_enter(KSTAT_IO_PTR(duplxp->ds_kstat));
5551709Smlf }
5561709Smlf if (in_bp->b_flags & B_READ)
5571709Smlf activeq = &duplxp->ds_readq;
5581709Smlf else
5591709Smlf activeq = &duplxp->ds_writeq;
5601709Smlf
5611709Smlf QUE_ADD(activeq->fc_qobjp, in_bp);
5621709Smlf } else {
5631709Smlf activeq = &duplxp->ds_readq;
5641709Smlf }
5651709Smlf
5661709Smlf tgcom_objp = duplxp->ds_tgcomobjp;
5671709Smlf
5681709Smlf for (;;) {
5691709Smlf if (!activeq->fc_bp)
5701709Smlf activeq->fc_bp = QUE_DEL(activeq->fc_qobjp);
5711709Smlf if (!activeq->fc_bp ||
5721709Smlf (TGCOM_PKT(tgcom_objp, activeq->fc_bp, duplx_restart,
5731709Smlf (caddr_t)duplxp) != DDI_SUCCESS) ||
5741709Smlf (activeq->fc_outcnt >= activeq->fc_maxcnt)) {
5751709Smlf
5761709Smlf /* switch read/write queues */
5771709Smlf activeq = activeq->next;
5781709Smlf if (!activeq->fc_bp)
5791709Smlf activeq->fc_bp = QUE_DEL(activeq->fc_qobjp);
5801709Smlf if (!activeq->fc_bp ||
5811709Smlf (TGCOM_PKT(tgcom_objp, activeq->fc_bp,
5821709Smlf duplx_restart, (caddr_t)duplxp) != DDI_SUCCESS) ||
5831709Smlf (activeq->fc_outcnt >= activeq->fc_maxcnt)) {
5841709Smlf mutex_exit(&duplxp->ds_mutex);
5851709Smlf return (0);
5861709Smlf }
5871709Smlf }
5881709Smlf
5891709Smlf activeq->fc_outcnt++;
5901709Smlf bp = activeq->fc_bp;
5911709Smlf activeq->fc_bp = NULL;
5921709Smlf
5931709Smlf if (duplxp->ds_kstat)
5941709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(duplxp->ds_kstat));
5951709Smlf mutex_exit(&duplxp->ds_mutex);
5961709Smlf
5971709Smlf TGCOM_TRANSPORT(tgcom_objp, bp);
5981709Smlf
5991709Smlf if (!mutex_tryenter(&duplxp->ds_mutex))
6001709Smlf return (0);
6011709Smlf
6021709Smlf activeq = activeq->next;
6031709Smlf }
6041709Smlf }
6051709Smlf
6061709Smlf static int
duplx_deque(opaque_t queuep,struct buf * in_bp)6071709Smlf duplx_deque(opaque_t queuep, struct buf *in_bp)
6081709Smlf {
6091709Smlf struct duplx_data *duplxp = (struct duplx_data *)queuep;
6101709Smlf opaque_t tgcom_objp;
6111709Smlf struct fc_que *activeq;
6121709Smlf struct buf *bp;
6131709Smlf
6141709Smlf mutex_enter(&duplxp->ds_mutex);
6151709Smlf
6161709Smlf tgcom_objp = duplxp->ds_tgcomobjp;
6171709Smlf
6181709Smlf if (in_bp->b_flags & B_READ)
6191709Smlf activeq = &duplxp->ds_readq;
6201709Smlf else
6211709Smlf activeq = &duplxp->ds_writeq;
6221709Smlf activeq->fc_outcnt--;
6231709Smlf
6241709Smlf if (duplxp->ds_kstat) {
6251709Smlf if (in_bp->b_flags & B_READ) {
6261709Smlf KSTAT_IO_PTR(duplxp->ds_kstat)->reads++;
6271709Smlf KSTAT_IO_PTR(duplxp->ds_kstat)->nread +=
6281709Smlf (in_bp->b_bcount - in_bp->b_resid);
6291709Smlf } else {
6301709Smlf KSTAT_IO_PTR(duplxp->ds_kstat)->writes++;
6311709Smlf KSTAT_IO_PTR(duplxp->ds_kstat)->nwritten +=
6321709Smlf (in_bp->b_bcount - in_bp->b_resid);
6331709Smlf }
6341709Smlf kstat_runq_exit(KSTAT_IO_PTR(duplxp->ds_kstat));
6351709Smlf }
6361709Smlf
6371709Smlf for (;;) {
6381709Smlf
6391709Smlf /* if needed, try to pull request off a queue */
6401709Smlf if (!activeq->fc_bp)
6411709Smlf activeq->fc_bp = QUE_DEL(activeq->fc_qobjp);
6421709Smlf
6431709Smlf if (!activeq->fc_bp ||
6441709Smlf (TGCOM_PKT(tgcom_objp, activeq->fc_bp, duplx_restart,
6451709Smlf (caddr_t)duplxp) != DDI_SUCCESS) ||
6461709Smlf (activeq->fc_outcnt >= activeq->fc_maxcnt)) {
6471709Smlf
6481709Smlf activeq = activeq->next;
6491709Smlf if (!activeq->fc_bp)
6501709Smlf activeq->fc_bp = QUE_DEL(activeq->fc_qobjp);
6511709Smlf
6521709Smlf if (!activeq->fc_bp ||
6531709Smlf (TGCOM_PKT(tgcom_objp, activeq->fc_bp,
6541709Smlf duplx_restart, (caddr_t)duplxp) != DDI_SUCCESS) ||
6551709Smlf (activeq->fc_outcnt >= activeq->fc_maxcnt)) {
6561709Smlf mutex_exit(&duplxp->ds_mutex);
6571709Smlf return (0);
6581709Smlf }
6591709Smlf }
6601709Smlf
6611709Smlf activeq->fc_outcnt++;
6621709Smlf bp = activeq->fc_bp;
6631709Smlf activeq->fc_bp = NULL;
6641709Smlf
6651709Smlf if (duplxp->ds_kstat)
6661709Smlf kstat_waitq_to_runq(KSTAT_IO_PTR(duplxp->ds_kstat));
6671709Smlf
6681709Smlf mutex_exit(&duplxp->ds_mutex);
6691709Smlf
6701709Smlf TGCOM_TRANSPORT(tgcom_objp, bp);
6711709Smlf
6721709Smlf if (!mutex_tryenter(&duplxp->ds_mutex))
6731709Smlf return (0);
6741709Smlf
6751709Smlf activeq = activeq->next;
6761709Smlf }
6771709Smlf }
6781709Smlf
6791709Smlf static int
duplx_restart(struct duplx_data * duplxp)6801709Smlf duplx_restart(struct duplx_data *duplxp)
6811709Smlf {
6821709Smlf (void) duplx_enque(duplxp, NULL);
6831709Smlf return (-1);
6841709Smlf }
6851709Smlf
6861709Smlf /*
6871709Smlf * Tagged queueing flow control
6881709Smlf */
6891709Smlf /*
6901709Smlf * Local Function Prototypes
6911709Smlf */
6921709Smlf
6931709Smlf struct flc_objops adapt_ops = {
6941709Smlf fc_init,
6951709Smlf fc_free,
6961709Smlf dmult_enque,
6971709Smlf dmult_deque,
6981709Smlf fc_start_kstat,
6991709Smlf fc_stop_kstat,
7001709Smlf 0, 0
7011709Smlf };
7021709Smlf
7031709Smlf struct flc_obj *
adapt_create()7041709Smlf adapt_create()
7051709Smlf {
7061709Smlf return (fc_create((struct flc_objops *)&adapt_ops));
7071709Smlf
7081709Smlf }
7091709Smlf
7101709Smlf /*
7111709Smlf * Common Queue functions
7121709Smlf */
7131709Smlf
7141709Smlf /*
7151709Smlf * Local static data
7161709Smlf */
7171709Smlf #ifdef Q_DEBUG
7181709Smlf #define DENT 0x0001
7191709Smlf #define DERR 0x0002
7201709Smlf #define DIO 0x0004
7211709Smlf static int que_debug = DENT|DERR|DIO;
7221709Smlf
7231709Smlf #endif /* Q_DEBUG */
7241709Smlf /*
7251709Smlf * Local Function Prototypes
7261709Smlf */
7271709Smlf static struct que_obj *que_create(struct que_objops *qopsp);
7281709Smlf static int que_init(struct que_data *qfp, void *lkarg);
7291709Smlf static int que_free(struct que_obj *queobjp);
7301709Smlf static struct buf *que_del(struct que_data *qfp);
7311709Smlf
7321709Smlf static struct que_obj *
que_create(struct que_objops * qopsp)7331709Smlf que_create(struct que_objops *qopsp)
7341709Smlf {
7351709Smlf struct que_data *qfp;
7361709Smlf struct que_obj *queobjp;
7371709Smlf
7381709Smlf queobjp = kmem_zalloc((sizeof (*queobjp) + sizeof (*qfp)), KM_NOSLEEP);
7391709Smlf if (!queobjp)
7401709Smlf return (NULL);
7411709Smlf
7421709Smlf queobjp->que_ops = qopsp;
7431709Smlf qfp = (struct que_data *)(queobjp+1);
7441709Smlf queobjp->que_data = (opaque_t)qfp;
7451709Smlf
7461709Smlf return ((opaque_t)queobjp);
7471709Smlf }
7481709Smlf
7491709Smlf static int
que_init(struct que_data * qfp,void * lkarg)7501709Smlf que_init(struct que_data *qfp, void *lkarg)
7511709Smlf {
7521709Smlf mutex_init(&qfp->q_mutex, NULL, MUTEX_DRIVER, lkarg);
7531709Smlf return (DDI_SUCCESS);
7541709Smlf }
7551709Smlf
7561709Smlf static int
que_free(struct que_obj * queobjp)7571709Smlf que_free(struct que_obj *queobjp)
7581709Smlf {
7591709Smlf struct que_data *qfp;
7601709Smlf
7611709Smlf qfp = (struct que_data *)queobjp->que_data;
7621709Smlf mutex_destroy(&qfp->q_mutex);
7631709Smlf kmem_free(queobjp, (sizeof (*queobjp) + sizeof (struct que_data)));
7641709Smlf return (0);
7651709Smlf }
7661709Smlf
7671709Smlf static struct buf *
que_del(struct que_data * qfp)7681709Smlf que_del(struct que_data *qfp)
7691709Smlf {
7701709Smlf struct buf *bp;
7711709Smlf
7721709Smlf bp = qfp->q_tab.b_actf;
7731709Smlf if (bp) {
7741709Smlf qfp->q_tab.b_actf = bp->av_forw;
7751709Smlf if (!qfp->q_tab.b_actf)
7761709Smlf qfp->q_tab.b_actl = NULL;
7771709Smlf bp->av_forw = 0;
7781709Smlf }
7791709Smlf return (bp);
7801709Smlf }
7811709Smlf
7821709Smlf
7831709Smlf
7841709Smlf /*
7851709Smlf * Qmerge
7861709Smlf * Local Function Prototypes
7871709Smlf */
7881709Smlf static int qmerge_add(), qmerge_free();
7891709Smlf static struct buf *qmerge_del(struct que_data *qfp);
7901709Smlf
7911709Smlf struct que_objops qmerge_ops = {
7921709Smlf que_init,
7931709Smlf qmerge_free,
7941709Smlf qmerge_add,
7951709Smlf qmerge_del,
7961709Smlf 0, 0
7971709Smlf };
7981709Smlf
7991709Smlf /* fields in diskhd */
8001709Smlf #define hd_cnt b_back
8011709Smlf #define hd_private b_forw
8021709Smlf #define hd_flags b_flags
8031709Smlf #define hd_sync_next av_forw
8041709Smlf #define hd_async_next av_back
8051709Smlf
8061709Smlf #define hd_sync2async sync_async_ratio
8071709Smlf
8081709Smlf #define QNEAR_FORWARD 0x01
8091709Smlf #define QNEAR_BACKWARD 0x02
8101709Smlf #define QNEAR_ASYNCONLY 0x04
8111709Smlf #define QNEAR_ASYNCALSO 0x08
8121709Smlf
8131709Smlf #define DBLK(bp) ((unsigned long)(bp)->b_private)
8141709Smlf
8151709Smlf #define BP_LT_BP(a, b) (DBLK(a) < DBLK(b))
8161709Smlf #define BP_GT_BP(a, b) (DBLK(a) > DBLK(b))
8171709Smlf #define BP_LT_HD(a, b) (DBLK(a) < (unsigned long)((b)->hd_private))
8181709Smlf #define BP_GT_HD(a, b) (DBLK(a) > (unsigned long)((b)->hd_private))
8191709Smlf #define QNEAR_ASYNC (QNEAR_ASYNCONLY|QNEAR_ASYNCALSO)
8201709Smlf
8211709Smlf #define SYNC2ASYNC(a) ((a)->q_tab.hd_cnt)
8221709Smlf
8231709Smlf
8241709Smlf /*
8251709Smlf * qmerge implements a two priority queue, the low priority queue holding ASYNC
8261709Smlf * write requests, while the rest are queued in the high priority sync queue.
8271709Smlf * Requests on the async queue would be merged if possible.
8281709Smlf * By default qmerge2wayscan is 1, indicating an elevator algorithm. When
8291709Smlf * this variable is set to zero, it has the following side effects.
8301709Smlf * 1. We assume fairness is the number one issue.
8311709Smlf * 2. The next request to be picked indicates current head position.
8321709Smlf *
8331709Smlf * qmerge_sync2async indicates the ratio of scans of high prioriy
8341709Smlf * sync queue to low priority async queue.
8351709Smlf *
8361709Smlf * When qmerge variables have the following values it defaults to qsort
8371709Smlf *
8381709Smlf * qmerge1pri = 1, qmerge2wayscan = 0, qmerge_max_merge = 0
8391709Smlf *
8401709Smlf */
8411709Smlf static int qmerge_max_merge = 128 * 1024;
8421709Smlf static intptr_t qmerge_sync2async = 4;
8431709Smlf static int qmerge2wayscan = 1;
8441709Smlf static int qmerge1pri = 0;
8451709Smlf static int qmerge_merge = 0;
8461709Smlf
8471709Smlf /*
8481709Smlf * Local static data
8491709Smlf */
8501709Smlf struct que_obj *
qmerge_create()8511709Smlf qmerge_create()
8521709Smlf {
8531709Smlf struct que_data *qfp;
8541709Smlf struct que_obj *queobjp;
8551709Smlf
8561709Smlf queobjp = kmem_zalloc((sizeof (*queobjp) + sizeof (*qfp)), KM_NOSLEEP);
8571709Smlf if (!queobjp)
8581709Smlf return (NULL);
8591709Smlf
8601709Smlf queobjp->que_ops = &qmerge_ops;
8611709Smlf qfp = (struct que_data *)(queobjp+1);
8621709Smlf qfp->q_tab.hd_private = qfp->q_tab.hd_private = 0;
8631709Smlf qfp->q_tab.hd_sync_next = qfp->q_tab.hd_async_next = NULL;
8641709Smlf qfp->q_tab.hd_cnt = (void *)qmerge_sync2async;
8651709Smlf queobjp->que_data = (opaque_t)qfp;
8661709Smlf
8671709Smlf return ((opaque_t)queobjp);
8681709Smlf }
8691709Smlf
8701709Smlf static int
qmerge_free(struct que_obj * queobjp)8711709Smlf qmerge_free(struct que_obj *queobjp)
8721709Smlf {
8731709Smlf struct que_data *qfp;
8741709Smlf
8751709Smlf qfp = (struct que_data *)queobjp->que_data;
8761709Smlf mutex_destroy(&qfp->q_mutex);
8771709Smlf kmem_free(queobjp, (sizeof (*queobjp) + sizeof (*qfp)));
8781709Smlf return (0);
8791709Smlf }
8801709Smlf
8811709Smlf static int
qmerge_can_merge(bp1,bp2)8821709Smlf qmerge_can_merge(bp1, bp2)
8831709Smlf struct buf *bp1, *bp2;
8841709Smlf {
8851709Smlf const int paw_flags = B_PAGEIO | B_ASYNC | B_WRITE;
8861709Smlf
8871709Smlf if ((bp1->b_un.b_addr != 0) || (bp2->b_un.b_addr != 0) ||
8881709Smlf ((bp1->b_flags & (paw_flags | B_REMAPPED)) != paw_flags) ||
8891709Smlf ((bp2->b_flags & (paw_flags | B_REMAPPED)) != paw_flags) ||
8901709Smlf (bp1->b_bcount & PAGEOFFSET) || (bp2->b_bcount & PAGEOFFSET) ||
8911709Smlf (bp1->b_bcount + bp2->b_bcount > qmerge_max_merge))
8921709Smlf return (0);
8931709Smlf
8941709Smlf if ((DBLK(bp2) + bp2->b_bcount / DEV_BSIZE == DBLK(bp1)) ||
8951709Smlf (DBLK(bp1) + bp1->b_bcount / DEV_BSIZE == DBLK(bp2)))
8961709Smlf return (1);
8971709Smlf else
8981709Smlf return (0);
8991709Smlf }
9001709Smlf
9011709Smlf static void
qmerge_mergesetup(bp_merge,bp)9021709Smlf qmerge_mergesetup(bp_merge, bp)
9031709Smlf struct buf *bp_merge, *bp;
9041709Smlf {
9051709Smlf struct buf *bp1;
9061709Smlf struct page *pp, *pp_merge, *pp_merge_prev;
9071709Smlf int forward;
9081709Smlf
9091709Smlf qmerge_merge++;
9101709Smlf forward = DBLK(bp_merge) < DBLK(bp);
9111709Smlf
9121709Smlf bp_merge->b_bcount += bp->b_bcount;
9131709Smlf
9141709Smlf pp = bp->b_pages;
9151709Smlf pp_merge = bp_merge->b_pages;
9161709Smlf
9171709Smlf pp_merge_prev = pp_merge->p_prev;
9181709Smlf
9191709Smlf pp_merge->p_prev->p_next = pp;
9201709Smlf pp_merge->p_prev = pp->p_prev;
9211709Smlf pp->p_prev->p_next = pp_merge;
9221709Smlf pp->p_prev = pp_merge_prev;
9231709Smlf
9241709Smlf bp1 = bp_merge->b_forw;
9251709Smlf
9261709Smlf bp1->av_back->av_forw = bp;
9271709Smlf bp->av_back = bp1->av_back;
9281709Smlf bp1->av_back = bp;
9291709Smlf bp->av_forw = bp1;
9301709Smlf
9311709Smlf if (!forward) {
9321709Smlf bp_merge->b_forw = bp;
9331709Smlf bp_merge->b_pages = pp;
9341709Smlf bp_merge->b_private = bp->b_private;
9351709Smlf }
9361709Smlf }
9371709Smlf
9381709Smlf static void
que_insert(struct que_data * qfp,struct buf * bp)9391709Smlf que_insert(struct que_data *qfp, struct buf *bp)
9401709Smlf {
9411709Smlf struct buf *bp1, *bp_start, *lowest_bp, *highest_bp;
9421709Smlf uintptr_t highest_blk, lowest_blk;
9431709Smlf struct buf **async_bpp, **sync_bpp, **bpp;
9441709Smlf struct diskhd *dp = &qfp->q_tab;
9451709Smlf
9461709Smlf sync_bpp = &dp->hd_sync_next;
9471709Smlf async_bpp = &dp->hd_async_next;
9481709Smlf /*
9491709Smlf * The ioctl used by the format utility requires that bp->av_back be
9501709Smlf * preserved.
9511709Smlf */
9521709Smlf if (bp->av_back)
9531709Smlf bp->b_error = (intptr_t)bp->av_back;
9541709Smlf if (!qmerge1pri &&
9551709Smlf ((bp->b_flags & (B_ASYNC|B_READ|B_FREE)) == B_ASYNC)) {
9561709Smlf bpp = &dp->hd_async_next;
9571709Smlf } else {
9581709Smlf bpp = &dp->hd_sync_next;
9591709Smlf }
9601709Smlf
9611709Smlf
9621709Smlf if ((bp1 = *bpp) == NULL) {
9631709Smlf *bpp = bp;
9641709Smlf bp->av_forw = bp->av_back = bp;
9651709Smlf if ((bpp == async_bpp) && (*sync_bpp == NULL)) {
9661709Smlf dp->hd_flags |= QNEAR_ASYNCONLY;
9671709Smlf } else if (bpp == sync_bpp) {
9681709Smlf dp->hd_flags &= ~QNEAR_ASYNCONLY;
9691709Smlf if (*async_bpp) {
9701709Smlf dp->hd_flags |= QNEAR_ASYNCALSO;
9711709Smlf }
9721709Smlf }
9731709Smlf return;
9741709Smlf }
9751709Smlf bp_start = bp1;
9761709Smlf if (DBLK(bp) < DBLK(bp1)) {
9771709Smlf lowest_blk = DBLK(bp1);
9781709Smlf lowest_bp = bp1;
9791709Smlf do {
9801709Smlf if (DBLK(bp) > DBLK(bp1)) {
9811709Smlf bp->av_forw = bp1->av_forw;
9821709Smlf bp1->av_forw->av_back = bp;
9831709Smlf bp1->av_forw = bp;
9841709Smlf bp->av_back = bp1;
9851709Smlf
9861709Smlf if (((bpp == async_bpp) &&
9871709Smlf (dp->hd_flags & QNEAR_ASYNC)) ||
9881709Smlf (bpp == sync_bpp)) {
9891709Smlf if (!(dp->hd_flags & QNEAR_BACKWARD) &&
9901709Smlf BP_GT_HD(bp, dp)) {
9911709Smlf *bpp = bp;
9921709Smlf }
9931709Smlf }
9941709Smlf return;
9951709Smlf } else if (DBLK(bp1) < lowest_blk) {
9961709Smlf lowest_bp = bp1;
9971709Smlf lowest_blk = DBLK(bp1);
9981709Smlf }
9991709Smlf } while ((DBLK(bp1->av_back) < DBLK(bp1)) &&
10001709Smlf ((bp1 = bp1->av_back) != bp_start));
10011709Smlf bp->av_forw = lowest_bp;
10021709Smlf lowest_bp->av_back->av_forw = bp;
10031709Smlf bp->av_back = lowest_bp->av_back;
10041709Smlf lowest_bp->av_back = bp;
10051709Smlf if ((bpp == async_bpp) && !(dp->hd_flags & QNEAR_ASYNC)) {
10061709Smlf *bpp = bp;
10071709Smlf } else if (!(dp->hd_flags & QNEAR_BACKWARD) &&
10081709Smlf BP_GT_HD(bp, dp)) {
10091709Smlf *bpp = bp;
10101709Smlf }
10111709Smlf } else {
10121709Smlf highest_blk = DBLK(bp1);
10131709Smlf highest_bp = bp1;
10141709Smlf do {
10151709Smlf if (DBLK(bp) < DBLK(bp1)) {
10161709Smlf bp->av_forw = bp1;
10171709Smlf bp1->av_back->av_forw = bp;
10181709Smlf bp->av_back = bp1->av_back;
10191709Smlf bp1->av_back = bp;
10201709Smlf if (((bpp == async_bpp) &&
10211709Smlf (dp->hd_flags & QNEAR_ASYNC)) ||
10221709Smlf (bpp == sync_bpp)) {
10231709Smlf if ((dp->hd_flags & QNEAR_BACKWARD) &&
10241709Smlf BP_LT_HD(bp, dp)) {
10251709Smlf *bpp = bp;
10261709Smlf }
10271709Smlf }
10281709Smlf return;
10291709Smlf } else if (DBLK(bp1) > highest_blk) {
10301709Smlf highest_bp = bp1;
10311709Smlf highest_blk = DBLK(bp1);
10321709Smlf }
10331709Smlf } while ((DBLK(bp1->av_forw) > DBLK(bp1)) &&
10341709Smlf ((bp1 = bp1->av_forw) != bp_start));
10351709Smlf bp->av_back = highest_bp;
10361709Smlf highest_bp->av_forw->av_back = bp;
10371709Smlf bp->av_forw = highest_bp->av_forw;
10381709Smlf highest_bp->av_forw = bp;
10391709Smlf
10401709Smlf if (((bpp == sync_bpp) ||
10411709Smlf ((bpp == async_bpp) && (dp->hd_flags & QNEAR_ASYNC))) &&
10421709Smlf (dp->hd_flags & QNEAR_BACKWARD) && (BP_LT_HD(bp, dp)))
10431709Smlf *bpp = bp;
10441709Smlf }
10451709Smlf }
10461709Smlf
10471709Smlf /*
10481709Smlf * dmult_enque() holds dmultp->ds_mutex lock, so we dont grab
10491709Smlf * lock here. If dmult_enque() changes we will have to visit
10501709Smlf * this function again
10511709Smlf */
10521709Smlf static int
qmerge_add(struct que_data * qfp,struct buf * bp)10531709Smlf qmerge_add(struct que_data *qfp, struct buf *bp)
10541709Smlf {
10551709Smlf
10561709Smlf que_insert(qfp, bp);
10571709Smlf return (++qfp->q_cnt);
10581709Smlf }
10591709Smlf
10601709Smlf static int
qmerge_iodone(struct buf * bp)10611709Smlf qmerge_iodone(struct buf *bp)
10621709Smlf {
10631709Smlf struct buf *bp1;
10641709Smlf struct page *pp, *pp1, *tmp_pp;
10651709Smlf
10661709Smlf if (bp->b_flags & B_REMAPPED)
10671709Smlf bp_mapout(bp);
10681709Smlf
10691709Smlf bp1 = bp->b_forw;
10701709Smlf do {
10711709Smlf bp->b_forw = bp1->av_forw;
10721709Smlf bp1->av_forw->av_back = bp1->av_back;
10731709Smlf bp1->av_back->av_forw = bp1->av_forw;
10741709Smlf pp = (page_t *)bp1->b_pages;
10751709Smlf pp1 = bp->b_forw->b_pages;
10761709Smlf
10771709Smlf tmp_pp = pp->p_prev;
10781709Smlf pp->p_prev = pp1->p_prev;
10791709Smlf pp->p_prev->p_next = pp;
10801709Smlf
10811709Smlf pp1->p_prev = tmp_pp;
10821709Smlf pp1->p_prev->p_next = pp1;
10831709Smlf
10841709Smlf if (bp->b_flags & B_ERROR) {
10851709Smlf bp1->b_error = bp->b_error;
10861709Smlf bp1->b_flags |= B_ERROR;
10871709Smlf }
10881709Smlf
10891709Smlf biodone(bp1);
10901709Smlf } while ((bp1 = bp->b_forw) != bp->b_forw->av_forw);
10911709Smlf
10921709Smlf biodone(bp1);
10931709Smlf kmem_free(bp, sizeof (*bp));
10941709Smlf return (0);
10951709Smlf }
10961709Smlf
10971709Smlf
10981709Smlf
10991709Smlf
11001709Smlf static struct buf *
qmerge_nextbp(struct que_data * qfp,struct buf * bp_merge,int * can_merge)11011709Smlf qmerge_nextbp(struct que_data *qfp, struct buf *bp_merge, int *can_merge)
11021709Smlf {
11031709Smlf intptr_t private, cnt;
11041709Smlf int flags;
11051709Smlf struct buf *sync_bp, *async_bp, *bp;
11061709Smlf struct buf **sync_bpp, **async_bpp, **bpp;
11071709Smlf struct diskhd *dp = &qfp->q_tab;
11081709Smlf
11091709Smlf if (qfp->q_cnt == 0) {
11101709Smlf return (NULL);
11111709Smlf }
11121709Smlf flags = qfp->q_tab.hd_flags;
11131709Smlf sync_bpp = &qfp->q_tab.hd_sync_next;
11141709Smlf async_bpp = &qfp->q_tab.hd_async_next;
11151709Smlf
11161709Smlf begin_nextbp:
11171709Smlf if (flags & QNEAR_ASYNCONLY) {
11181709Smlf bp = *async_bpp;
11191709Smlf private = DBLK(bp);
11201709Smlf if (bp_merge && !qmerge_can_merge(bp, bp_merge)) {
11211709Smlf return (NULL);
11221709Smlf } else if (bp->av_forw == bp) {
11231709Smlf bp->av_forw = bp->av_back = NULL;
11241709Smlf flags &= ~(QNEAR_ASYNCONLY | QNEAR_BACKWARD);
11251709Smlf private = 0;
11261709Smlf } else if (flags & QNEAR_BACKWARD) {
11271709Smlf if (DBLK(bp) < DBLK(bp->av_back)) {
11281709Smlf flags &= ~QNEAR_BACKWARD;
11291709Smlf private = 0;
11301709Smlf }
11311709Smlf } else if (DBLK(bp) > DBLK(bp->av_forw)) {
11321709Smlf if (qmerge2wayscan) {
11331709Smlf flags |= QNEAR_BACKWARD;
11341709Smlf } else {
11351709Smlf private = 0;
11361709Smlf }
11371709Smlf } else if (qmerge2wayscan == 0) {
11381709Smlf private = DBLK(bp->av_forw);
11391709Smlf }
11401709Smlf bpp = async_bpp;
11411709Smlf
11421709Smlf } else if (flags & QNEAR_ASYNCALSO) {
11431709Smlf sync_bp = *sync_bpp;
11441709Smlf async_bp = *async_bpp;
11451709Smlf if (flags & QNEAR_BACKWARD) {
11461709Smlf if (BP_GT_HD(sync_bp, dp) && BP_GT_HD(async_bp, dp)) {
11471709Smlf flags &= ~(QNEAR_BACKWARD|QNEAR_ASYNCALSO);
11481709Smlf *sync_bpp = sync_bp->av_forw;
11491709Smlf *async_bpp = async_bp->av_forw;
11501709Smlf SYNC2ASYNC(qfp) = (void *)qmerge_sync2async;
11511709Smlf qfp->q_tab.hd_private = 0;
11521709Smlf goto begin_nextbp;
11531709Smlf }
11541709Smlf if (BP_LT_HD(async_bp, dp) && BP_LT_HD(sync_bp, dp)) {
11551709Smlf if (BP_GT_BP(async_bp, sync_bp)) {
11561709Smlf bpp = async_bpp;
11571709Smlf bp = *async_bpp;
11581709Smlf } else {
11591709Smlf bpp = sync_bpp;
11601709Smlf bp = *sync_bpp;
11611709Smlf }
11621709Smlf } else if (BP_LT_HD(async_bp, dp)) {
11631709Smlf bpp = async_bpp;
11641709Smlf bp = *async_bpp;
11651709Smlf } else {
11661709Smlf bpp = sync_bpp;
11671709Smlf bp = *sync_bpp;
11681709Smlf }
11691709Smlf } else {
11701709Smlf if (BP_LT_HD(sync_bp, dp) && BP_LT_HD(async_bp, dp)) {
11711709Smlf if (qmerge2wayscan) {
11721709Smlf flags |= QNEAR_BACKWARD;
11731709Smlf *sync_bpp = sync_bp->av_back;
11741709Smlf *async_bpp = async_bp->av_back;
11751709Smlf goto begin_nextbp;
11761709Smlf } else {
11771709Smlf flags &= ~QNEAR_ASYNCALSO;
11781709Smlf SYNC2ASYNC(qfp) =
11791709Smlf (void *)qmerge_sync2async;
11801709Smlf qfp->q_tab.hd_private = 0;
11811709Smlf goto begin_nextbp;
11821709Smlf }
11831709Smlf }
11841709Smlf if (BP_GT_HD(async_bp, dp) && BP_GT_HD(sync_bp, dp)) {
11851709Smlf if (BP_LT_BP(async_bp, sync_bp)) {
11861709Smlf bpp = async_bpp;
11871709Smlf bp = *async_bpp;
11881709Smlf } else {
11891709Smlf bpp = sync_bpp;
11901709Smlf bp = *sync_bpp;
11911709Smlf }
11921709Smlf } else if (BP_GT_HD(async_bp, dp)) {
11931709Smlf bpp = async_bpp;
11941709Smlf bp = *async_bpp;
11951709Smlf } else {
11961709Smlf bpp = sync_bpp;
11971709Smlf bp = *sync_bpp;
11981709Smlf }
11991709Smlf }
12001709Smlf if (bp_merge && !qmerge_can_merge(bp, bp_merge)) {
12011709Smlf return (NULL);
12021709Smlf } else if (bp->av_forw == bp) {
12031709Smlf bp->av_forw = bp->av_back = NULL;
12041709Smlf flags &= ~QNEAR_ASYNCALSO;
12051709Smlf if (bpp == async_bpp) {
12061709Smlf SYNC2ASYNC(qfp) = (void *)qmerge_sync2async;
12071709Smlf } else {
12081709Smlf flags |= QNEAR_ASYNCONLY;
12091709Smlf }
12101709Smlf }
12111709Smlf private = DBLK(bp);
12121709Smlf } else {
12131709Smlf bp = *sync_bpp;
12141709Smlf private = DBLK(bp);
12151709Smlf if (bp_merge && !qmerge_can_merge(bp, bp_merge)) {
12161709Smlf return (NULL);
12171709Smlf } else if (bp->av_forw == bp) {
12181709Smlf private = 0;
12191709Smlf SYNC2ASYNC(qfp) = (void *)qmerge_sync2async;
12201709Smlf bp->av_forw = bp->av_back = NULL;
12211709Smlf flags &= ~QNEAR_BACKWARD;
12221709Smlf if (*async_bpp)
12231709Smlf flags |= QNEAR_ASYNCONLY;
12241709Smlf } else if (flags & QNEAR_BACKWARD) {
12251709Smlf if (DBLK(bp) < DBLK(bp->av_back)) {
12261709Smlf flags &= ~QNEAR_BACKWARD;
12271709Smlf cnt = (intptr_t)SYNC2ASYNC(qfp);
12281709Smlf if (cnt > 0) {
12291709Smlf cnt--;
12301709Smlf SYNC2ASYNC(qfp) = (void *)cnt;
12311709Smlf } else {
12321709Smlf if (*async_bpp)
12331709Smlf flags |= QNEAR_ASYNCALSO;
12341709Smlf SYNC2ASYNC(qfp) =
12351709Smlf (void *)qmerge_sync2async;
12361709Smlf }
12371709Smlf private = 0;
12381709Smlf }
12391709Smlf } else if (DBLK(bp) > DBLK(bp->av_forw)) {
12401709Smlf private = 0;
12411709Smlf if (qmerge2wayscan) {
12421709Smlf flags |= QNEAR_BACKWARD;
12431709Smlf private = DBLK(bp);
12441709Smlf } else {
12451709Smlf cnt = (intptr_t)SYNC2ASYNC(qfp);
12461709Smlf if (cnt > 0) {
12471709Smlf cnt--;
12481709Smlf SYNC2ASYNC(qfp) = (void *)cnt;
12491709Smlf } else {
12501709Smlf if (*async_bpp)
12511709Smlf flags |= QNEAR_ASYNCALSO;
12521709Smlf SYNC2ASYNC(qfp) =
12531709Smlf (void *)qmerge_sync2async;
12541709Smlf }
12551709Smlf }
12561709Smlf } else if (qmerge2wayscan == 0) {
12571709Smlf private = DBLK(bp->av_forw);
12581709Smlf }
12591709Smlf bpp = sync_bpp;
12601709Smlf }
12611709Smlf
12621709Smlf if (bp->av_forw) {
12631709Smlf *can_merge = !(bp->b_flags & B_READ);
12641709Smlf if (flags & QNEAR_BACKWARD) {
12651709Smlf *bpp = bp->av_back;
12661709Smlf if ((DBLK(bp->av_back) +
12671709Smlf bp->av_back->b_bcount / DEV_BSIZE) != DBLK(bp))
12681709Smlf *can_merge = 0;
12691709Smlf } else {
12701709Smlf *bpp = bp->av_forw;
12711709Smlf if ((DBLK(bp) + bp->b_bcount / DEV_BSIZE) !=
12721709Smlf DBLK(bp->av_forw))
12731709Smlf *can_merge = 0;
12741709Smlf }
12751709Smlf bp->av_forw->av_back = bp->av_back;
12761709Smlf bp->av_back->av_forw = bp->av_forw;
12771709Smlf bp->av_forw = bp->av_back = NULL;
12781709Smlf } else {
12791709Smlf *bpp = NULL;
12801709Smlf *can_merge = 0;
12811709Smlf }
12821709Smlf qfp->q_tab.hd_private = (void *)private;
12831709Smlf qfp->q_cnt--;
12841709Smlf qfp->q_tab.hd_flags = flags;
12851709Smlf if (bp->b_error) {
12861709Smlf bp->av_back = (void *)(intptr_t)bp->b_error;
12871709Smlf bp->b_error = 0;
12881709Smlf }
12891709Smlf return (bp);
12901709Smlf }
12911709Smlf
12921709Smlf static struct buf *
qmerge_del(struct que_data * qfp)12931709Smlf qmerge_del(struct que_data *qfp)
12941709Smlf {
12951709Smlf struct buf *bp, *next_bp, *bp_merge;
12961709Smlf int alloc_mergebp, merge;
12971709Smlf
12981709Smlf if (qfp->q_cnt == 0) {
12991709Smlf return (NULL);
13001709Smlf }
13011709Smlf
13021709Smlf bp_merge = bp = qmerge_nextbp(qfp, NULL, &merge);
13031709Smlf alloc_mergebp = 1;
13041709Smlf while (merge && (next_bp = qmerge_nextbp(qfp, bp_merge, &merge))) {
13051709Smlf if (alloc_mergebp) {
13061709Smlf bp_merge = kmem_alloc(sizeof (*bp_merge), KM_NOSLEEP);
13071709Smlf if (bp_merge == NULL) {
13081709Smlf mutex_exit(&qfp->q_mutex);
13091709Smlf return (bp);
13101709Smlf }
13111709Smlf bcopy(bp, bp_merge, sizeof (*bp_merge));
13121709Smlf bp_merge->b_iodone = qmerge_iodone;
13131709Smlf bp_merge->b_forw = bp;
13141709Smlf bp_merge->b_back = (struct buf *)qfp;
13151709Smlf bp->av_forw = bp->av_back = bp;
13161709Smlf alloc_mergebp = 0;
13171709Smlf }
13181709Smlf qmerge_mergesetup(bp_merge, next_bp);
13191709Smlf }
13201709Smlf return (bp_merge);
13211709Smlf }
13221709Smlf
13231709Smlf
13241709Smlf /*
13251709Smlf * FIFO Queue functions
13261709Smlf */
13271709Smlf /*
13281709Smlf * Local Function Prototypes
13291709Smlf */
13301709Smlf static int qfifo_add();
13311709Smlf
13321709Smlf struct que_objops qfifo_ops = {
13331709Smlf que_init,
13341709Smlf que_free,
13351709Smlf qfifo_add,
13361709Smlf que_del,
13371709Smlf 0, 0
13381709Smlf };
13391709Smlf
13401709Smlf /*
13411709Smlf * Local static data
13421709Smlf */
13431709Smlf struct que_obj *
qfifo_create()13441709Smlf qfifo_create()
13451709Smlf {
13461709Smlf return (que_create((struct que_objops *)&qfifo_ops));
13471709Smlf }
13481709Smlf
13491709Smlf static int
qfifo_add(struct que_data * qfp,struct buf * bp)13501709Smlf qfifo_add(struct que_data *qfp, struct buf *bp)
13511709Smlf {
13521709Smlf
13531709Smlf if (!qfp->q_tab.b_actf)
13541709Smlf qfp->q_tab.b_actf = bp;
13551709Smlf else
13561709Smlf qfp->q_tab.b_actl->av_forw = bp;
13571709Smlf qfp->q_tab.b_actl = bp;
13581709Smlf bp->av_forw = NULL;
13591709Smlf return (0);
13601709Smlf }
13611709Smlf
13621709Smlf /*
13631709Smlf * One-Way-Scan Queue functions
13641709Smlf */
13651709Smlf /*
13661709Smlf * Local Function Prototypes
13671709Smlf */
13681709Smlf static int qsort_add();
13691709Smlf static struct buf *qsort_del();
13701709Smlf static void oneway_scan_binary(struct diskhd *dp, struct buf *bp);
13711709Smlf
13721709Smlf struct que_objops qsort_ops = {
13731709Smlf que_init,
13741709Smlf que_free,
13751709Smlf qsort_add,
13761709Smlf qsort_del,
13771709Smlf 0, 0
13781709Smlf };
13791709Smlf
13801709Smlf /*
13811709Smlf * Local static data
13821709Smlf */
13831709Smlf struct que_obj *
qsort_create()13841709Smlf qsort_create()
13851709Smlf {
13861709Smlf return (que_create((struct que_objops *)&qsort_ops));
13871709Smlf }
13881709Smlf
13891709Smlf static int
qsort_add(struct que_data * qfp,struct buf * bp)13901709Smlf qsort_add(struct que_data *qfp, struct buf *bp)
13911709Smlf {
13921709Smlf qfp->q_cnt++;
13931709Smlf oneway_scan_binary(&qfp->q_tab, bp);
13941709Smlf return (0);
13951709Smlf }
13961709Smlf
13971709Smlf
13981709Smlf #define b_pasf b_forw
13991709Smlf #define b_pasl b_back
14001709Smlf static void
oneway_scan_binary(struct diskhd * dp,struct buf * bp)14011709Smlf oneway_scan_binary(struct diskhd *dp, struct buf *bp)
14021709Smlf {
14031709Smlf struct buf *ap;
14041709Smlf
14051709Smlf ap = dp->b_actf;
14061709Smlf if (ap == NULL) {
14071709Smlf dp->b_actf = bp;
14081709Smlf bp->av_forw = NULL;
14091709Smlf return;
14101709Smlf }
14111709Smlf if (DBLK(bp) < DBLK(ap)) {
14121709Smlf ap = dp->b_pasf;
14131709Smlf if ((ap == NULL) || (DBLK(bp) < DBLK(ap))) {
14141709Smlf dp->b_pasf = bp;
14151709Smlf bp->av_forw = ap;
14161709Smlf return;
14171709Smlf }
14181709Smlf }
14191709Smlf while (ap->av_forw) {
14201709Smlf if (DBLK(bp) < DBLK(ap->av_forw))
14211709Smlf break;
14221709Smlf ap = ap->av_forw;
14231709Smlf }
14241709Smlf bp->av_forw = ap->av_forw;
14251709Smlf ap->av_forw = bp;
14261709Smlf }
14271709Smlf
14281709Smlf static struct buf *
qsort_del(struct que_data * qfp)14291709Smlf qsort_del(struct que_data *qfp)
14301709Smlf {
14311709Smlf struct buf *bp;
14321709Smlf
14331709Smlf if (qfp->q_cnt == 0) {
14341709Smlf return (NULL);
14351709Smlf }
14361709Smlf qfp->q_cnt--;
14371709Smlf bp = qfp->q_tab.b_actf;
14381709Smlf qfp->q_tab.b_actf = bp->av_forw;
14391709Smlf bp->av_forw = 0;
14401709Smlf if (!qfp->q_tab.b_actf && qfp->q_tab.b_pasf) {
14411709Smlf qfp->q_tab.b_actf = qfp->q_tab.b_pasf;
14421709Smlf qfp->q_tab.b_pasf = NULL;
14431709Smlf }
14441709Smlf return (bp);
14451709Smlf }
14461709Smlf
14471709Smlf /*
14481709Smlf * Tagged queueing
14491709Smlf */
14501709Smlf /*
14511709Smlf * Local Function Prototypes
14521709Smlf */
14531709Smlf
14541709Smlf struct que_objops qtag_ops = {
14551709Smlf que_init,
14561709Smlf que_free,
14571709Smlf qsort_add,
14581709Smlf qsort_del,
14591709Smlf 0, 0
14601709Smlf };
14611709Smlf
14621709Smlf /*
14631709Smlf * Local static data
14641709Smlf */
14651709Smlf struct que_obj *
qtag_create()14661709Smlf qtag_create()
14671709Smlf {
14681709Smlf return (que_create((struct que_objops *)&qtag_ops));
14691709Smlf }
1470