Lines Matching full:isc
608 struct cam_iosched_softc *isc = arg;
612 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc);
615 delta = now - isc->last_time;
616 isc->this_frac = (uint32_t)delta >> 16; /* Note: discards seconds -- should be 0 harmless if not */
617 isc->last_time = now;
619 cam_iosched_cl_maybe_steer(&isc->cl);
621 cam_iosched_limiter_tick(&isc->read_stats);
622 cam_iosched_limiter_tick(&isc->write_stats);
623 cam_iosched_limiter_tick(&isc->trim_stats);
625 isc->schedfnc(isc->periph);
628 * isc->load is an EMA of the pending I/Os at each tick. The number of
648 pending = isc->read_stats.pending + isc->write_stats.pending /* + isc->trim_stats.pending */;
649 pending += !!(isc->read_stats.state_flags & IOP_RATE_LIMITED) * isc->read_stats.queued +
650 !!(isc->write_stats.state_flags & IOP_RATE_LIMITED) * isc->write_stats.queued /* +
651 !!(isc->trim_stats.state_flags & IOP_RATE_LIMITED) * isc->trim_stats.queued */ ;
653 pending /= isc->periph->path->device->ccbq.total_openings;
655 isc->load = (pending + (isc->load << 13) - isc->load) >> 13; /* see above: 13 -> 16139 / 200/s = ~81s ~1 minute */
657 isc->total_ticks++;
661 cam_iosched_cl_init(struct control_loop *clp, struct cam_iosched_softc *isc)
665 clp->softc = isc;
676 struct cam_iosched_softc *isc;
680 isc = clp->softc;
681 now = isc->last_time;
688 if (isc->write_stats.current != isc->write_stats.max)
690 isc->write_stats.current, isc->write_stats.max);
691 isc->read_stats.current = isc->read_stats.max;
692 isc->write_stats.current = isc->write_stats.max;
693 isc->trim_stats.current = isc->trim_stats.max;
696 old = isc->write_stats.current;
697 lat = isc->read_stats.ema;
724 if (lat < clp->lolat || isc->read_stats.total - clp->last_count < 10)
725 isc->write_stats.current = isc->write_stats.current *
728 isc->write_stats.current = isc->write_stats.current *
730 clp->last_count = isc->read_stats.total;
736 if (isc->write_stats.current < isc->write_stats.min)
737 isc->write_stats.current = isc->write_stats.min;
738 if (isc->write_stats.current > isc->write_stats.max)
739 isc->write_stats.current = isc->write_stats.max;
740 if (old != isc->write_stats.current && iosched_debug)
742 old, isc->write_stats.current,
764 cam_iosched_io_metric_update(struct cam_iosched_softc *isc,
769 cam_iosched_has_flagged_work(struct cam_iosched_softc *isc)
771 return !!(isc->flags & CAM_IOSCHED_FLAG_WORK_FLAGS);
775 cam_iosched_has_io(struct cam_iosched_softc *isc)
779 struct bio *rbp = bioq_first(&isc->bio_queue);
780 struct bio *wbp = bioq_first(&isc->write_queue);
782 cam_iosched_limiter_caniop(&isc->write_stats, wbp) == 0;
784 cam_iosched_limiter_caniop(&isc->read_stats, rbp) == 0;
786 printf("can write %d: pending_writes %d max_writes %d\n", can_write, isc->write_stats.pending, isc->write_stats.max);
787 printf("can read %d: read_stats.pending %d max_reads %d\n", can_read, isc->read_stats.pending, isc->read_stats.max);
788 printf("Queued reads %d writes %d\n", isc->read_stats.queued, isc->write_stats.queued);
793 return bioq_first(&isc->bio_queue) != NULL;
797 cam_iosched_has_more_trim(struct cam_iosched_softc *isc)
801 bp = bioq_first(&isc->trim_queue);
808 if (bp == NULL || cam_iosched_limiter_caniop(&isc->trim_stats, bp) != 0)
818 if (isc->trim_goal > 0) {
819 if (isc->queued_trims >= isc->trim_goal)
821 if (isc->queued_trims > 0 &&
822 isc->trim_ticks > 0 &&
823 ticks - isc->last_trim_tick > isc->trim_ticks)
829 return !(isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) && bp != NULL;
832 #define cam_iosched_sort_queue(isc) ((isc)->sort_io_queue >= 0 ? \
833 (isc)->sort_io_queue : cam_sort_io_queues)
836 cam_iosched_has_work(struct cam_iosched_softc *isc)
840 printf("has work: %d %d %d\n", cam_iosched_has_io(isc),
841 cam_iosched_has_more_trim(isc),
842 cam_iosched_has_flagged_work(isc));
845 return cam_iosched_has_io(isc) ||
846 cam_iosched_has_more_trim(isc) ||
847 cam_iosched_has_flagged_work(isc);
852 cam_iosched_iop_stats_init(struct cam_iosched_softc *isc, struct iop_stats *ios)
867 ios->softc = isc;
876 struct cam_iosched_softc *isc;
881 isc = ios->softc;
893 cam_periph_lock(isc->periph);
902 cam_periph_unlock(isc->periph);
906 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc);
907 isc->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE;
909 cam_periph_unlock(isc->periph);
913 cam_periph_unlock(isc->periph);
922 struct cam_iosched_softc *isc;
927 isc = clp->softc;
942 cam_periph_lock(isc->periph);
944 cam_periph_unlock(isc->periph);
1013 cam_iosched_iop_stats_sysctl_init(struct cam_iosched_softc *isc, struct iop_stats *ios, char *name)
1018 ios->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx,
1019 SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, name,
1107 cam_iosched_cl_sysctl_init(struct cam_iosched_softc *isc)
1113 clp = &isc->cl;
1114 clp->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx,
1115 SYSCTL_CHILDREN(isc->sysctl_tree), OID_AUTO, "control",
1163 struct cam_iosched_softc *isc;
1165 isc = malloc(sizeof(*isc), M_CAMSCHED, M_NOWAIT | M_ZERO);
1166 if (isc == NULL)
1168 isc->disk = dp;
1169 isc->schedfnc = schedfnc;
1172 printf("CAM IOSCHEDULER Allocating entry at %p\n", isc);
1174 isc->sort_io_queue = -1;
1175 bioq_init(&isc->bio_queue);
1176 bioq_init(&isc->trim_queue);
1179 bioq_init(&isc->write_queue);
1180 isc->read_bias = default_read_bias;
1181 isc->current_read_bias = 0;
1182 isc->quanta = min(hz, 200);
1183 cam_iosched_iop_stats_init(isc, &isc->read_stats);
1184 cam_iosched_iop_stats_init(isc, &isc->write_stats);
1185 cam_iosched_iop_stats_init(isc, &isc->trim_stats);
1186 isc->trim_stats.max = 1; /* Trims are special: one at a time for now */
1187 isc->last_time = sbinuptime();
1188 callout_init_mtx(&isc->ticker, cam_periph_mtx(periph), 0);
1189 isc->periph = periph;
1190 cam_iosched_cl_init(&isc->cl, isc);
1191 callout_reset(&isc->ticker, hz / isc->quanta, cam_iosched_ticker, isc);
1192 isc->flags |= CAM_IOSCHED_FLAG_CALLOUT_ACTIVE;
1195 *iscp = isc;
1205 cam_iosched_fini(struct cam_iosched_softc *isc)
1207 if (isc) {
1208 cam_iosched_flush(isc, NULL, ENXIO);
1210 cam_iosched_iop_stats_fini(&isc->read_stats);
1211 cam_iosched_iop_stats_fini(&isc->write_stats);
1212 cam_iosched_iop_stats_fini(&isc->trim_stats);
1213 cam_iosched_cl_sysctl_fini(&isc->cl);
1214 if (isc->sysctl_tree)
1215 if (sysctl_ctx_free(&isc->sysctl_ctx) != 0)
1217 if (isc->flags & CAM_IOSCHED_FLAG_CALLOUT_ACTIVE) {
1218 callout_drain(&isc->ticker);
1219 isc->flags &= ~ CAM_IOSCHED_FLAG_CALLOUT_ACTIVE;
1222 free(isc, M_CAMSCHED);
1230 void cam_iosched_sysctl_init(struct cam_iosched_softc *isc,
1238 &isc->sort_io_queue, 0,
1242 &isc->trim_goal, 0,
1246 &isc->trim_goal, 0,
1253 isc->sysctl_tree = SYSCTL_ADD_NODE(&isc->sysctl_ctx,
1256 n = SYSCTL_CHILDREN(isc->sysctl_tree);
1257 ctx = &isc->sysctl_ctx;
1259 cam_iosched_iop_stats_sysctl_init(isc, &isc->read_stats, "read");
1260 cam_iosched_iop_stats_sysctl_init(isc, &isc->write_stats, "write");
1261 cam_iosched_iop_stats_sysctl_init(isc, &isc->trim_stats, "trim");
1262 cam_iosched_cl_sysctl_init(isc);
1266 &isc->read_bias, default_read_bias,
1271 &isc->quanta, 0, cam_iosched_quanta_sysctl, "I",
1276 &isc->total_ticks, 0,
1281 &isc->load, 0,
1286 &isc->max_lat, 0,
1292 cam_iosched_set_latfcn(struct cam_iosched_softc *isc,
1296 isc->latfcn = fnp;
1297 isc->latarg = argp;
1323 cam_iosched_set_trim_goal(struct cam_iosched_softc *isc, int goal)
1326 isc->trim_goal = goal;
1330 cam_iosched_set_trim_ticks(struct cam_iosched_softc *isc, int trim_ticks)
1333 isc->trim_ticks = trim_ticks;
1342 cam_iosched_flush(struct cam_iosched_softc *isc, struct devstat *stp, int err)
1344 bioq_flush(&isc->bio_queue, stp, err);
1345 bioq_flush(&isc->trim_queue, stp, err);
1348 bioq_flush(&isc->write_queue, stp, err);
1354 cam_iosched_get_write(struct cam_iosched_softc *isc)
1367 bp = bioq_first(&isc->write_queue);
1378 if (bioq_first(&isc->bio_queue) && isc->current_read_bias) {
1383 isc->current_read_bias, isc->write_stats.queued,
1384 isc->read_stats.queued);
1385 isc->current_read_bias--;
1393 if (cam_iosched_limiter_iop(&isc->write_stats, bp) != 0) {
1396 isc->write_stats.state_flags |= IOP_RATE_LIMITED;
1404 isc->current_read_bias = isc->read_bias;
1405 bioq_remove(&isc->write_queue, bp);
1407 isc->write_stats.queued--;
1408 isc->write_stats.total++;
1409 isc->write_stats.pending++;
1413 isc->write_stats.state_flags &= ~IOP_RATE_LIMITED;
1422 cam_iosched_put_back_trim(struct cam_iosched_softc *isc, struct bio *bp)
1424 bioq_insert_head(&isc->trim_queue, bp);
1425 if (isc->queued_trims == 0)
1426 isc->last_trim_tick = ticks;
1427 isc->queued_trims++;
1429 isc->trim_stats.queued++;
1430 isc->trim_stats.total--; /* since we put it back, don't double count */
1431 isc->trim_stats.pending--;
1443 cam_iosched_next_trim(struct cam_iosched_softc *isc)
1447 bp = bioq_first(&isc->trim_queue);
1450 bioq_remove(&isc->trim_queue, bp);
1451 isc->queued_trims--;
1452 isc->last_trim_tick = ticks; /* Reset the tick timer when we take trims */
1454 isc->trim_stats.queued--;
1455 isc->trim_stats.total++;
1456 isc->trim_stats.pending++;
1469 cam_iosched_get_trim(struct cam_iosched_softc *isc)
1475 if (!cam_iosched_has_more_trim(isc))
1478 bp = bioq_first(&isc->trim_queue);
1488 if (bioq_first(&isc->bio_queue) && isc->current_read_bias) {
1492 isc->current_read_bias, isc->trim_stats.queued,
1493 isc->read_stats.queued);
1494 isc->current_read_bias--;
1501 isc->current_read_bias = isc->read_bias;
1514 if (cam_iosched_limiter_iop(&isc->trim_stats, bp) != 0) {
1517 isc->trim_stats.state_flags |= IOP_RATE_LIMITED;
1520 isc->current_read_bias = isc->read_bias;
1521 isc->trim_stats.state_flags &= ~IOP_RATE_LIMITED;
1524 return cam_iosched_next_trim(isc);
1556 cam_iosched_next_bio(struct cam_iosched_softc *isc)
1567 if ((bp = cam_iosched_get_trim(isc)) != NULL)
1577 if ((bp = cam_iosched_get_write(isc)) != NULL)
1586 for (bp = bioq_first(&isc->bio_queue); bp != NULL;
1595 if (cam_iosched_rate_limited(&isc->read_stats) ||
1596 cam_iosched_limiter_iop(&isc->read_stats, bp) != 0) {
1597 isc->read_stats.state_flags |= IOP_RATE_LIMITED;
1600 isc->read_stats.state_flags &= ~IOP_RATE_LIMITED;
1610 if (cam_iosched_rate_limited(&isc->write_stats) ||
1611 cam_iosched_limiter_iop(&isc->write_stats, bp) != 0) {
1612 isc->write_stats.state_flags |= IOP_RATE_LIMITED;
1615 isc->write_stats.state_flags &= ~IOP_RATE_LIMITED;
1625 bp = bioq_first(&isc->bio_queue);
1629 bioq_remove(&isc->bio_queue, bp);
1633 isc->read_stats.queued--;
1634 isc->read_stats.total++;
1635 isc->read_stats.pending++;
1637 isc->write_stats.queued--;
1638 isc->write_stats.total++;
1639 isc->write_stats.pending++;
1655 cam_iosched_queue_work(struct cam_iosched_softc *isc, struct bio *bp)
1675 while (bioq_first(&isc->trim_queue) &&
1677 nbp = bioq_takefirst(&isc->trim_queue);
1698 if (bp->bio_cmd == BIO_FLUSH && isc->trim_ticks > 0)
1699 isc->last_trim_tick = ticks - isc->trim_ticks - 1;
1706 bioq_insert_tail(&isc->trim_queue, bp);
1707 if (isc->queued_trims == 0)
1708 isc->last_trim_tick = ticks;
1709 isc->queued_trims++;
1711 isc->trim_stats.in++;
1712 isc->trim_stats.queued++;
1716 else if (do_dynamic_iosched && isc->read_bias != 0 &&
1718 if (cam_iosched_sort_queue(isc))
1719 bioq_disksort(&isc->write_queue, bp);
1721 bioq_insert_tail(&isc->write_queue, bp);
1725 isc->write_stats.in++;
1726 isc->write_stats.queued++;
1731 if (cam_iosched_sort_queue(isc))
1732 bioq_disksort(&isc->bio_queue, bp);
1734 bioq_insert_tail(&isc->bio_queue, bp);
1739 isc->read_stats.in++;
1740 isc->read_stats.queued++;
1742 isc->write_stats.in++;
1743 isc->write_stats.queued++;
1753 cam_iosched_schedule(struct cam_iosched_softc *isc, struct cam_periph *periph)
1756 if (cam_iosched_has_work(isc))
1764 cam_iosched_trim_done(struct cam_iosched_softc *isc)
1767 isc->flags &= ~CAM_IOSCHED_FLAG_TRIM_ACTIVE;
1775 cam_iosched_bio_complete(struct cam_iosched_softc *isc, struct bio *bp,
1786 retval = cam_iosched_limiter_iodone(&isc->write_stats, bp);
1788 isc->write_stats.errs++;
1789 isc->write_stats.out++;
1790 isc->write_stats.pending--;
1792 retval = cam_iosched_limiter_iodone(&isc->read_stats, bp);
1794 isc->read_stats.errs++;
1795 isc->read_stats.out++;
1796 isc->read_stats.pending--;
1799 isc->trim_stats.errs++;
1800 isc->trim_stats.out++;
1801 isc->trim_stats.pending--;
1813 cam_iosched_io_metric_update(isc, sim_latency, bp);
1819 if (isc->latfcn && isc->max_lat != 0 && sim_latency > isc->max_lat)
1820 isc->latfcn(isc->latarg, sim_latency, bp);
1832 cam_iosched_submit_trim(struct cam_iosched_softc *isc)
1835 isc->flags |= CAM_IOSCHED_FLAG_TRIM_ACTIVE;
1842 cam_iosched_set_sort_queue(struct cam_iosched_softc *isc, int val)
1845 isc->sort_io_queue = val;
1849 cam_iosched_has_work_flags(struct cam_iosched_softc *isc, uint32_t flags)
1851 return isc->flags & flags;
1855 cam_iosched_set_work_flags(struct cam_iosched_softc *isc, uint32_t flags)
1857 isc->flags |= flags;
1861 cam_iosched_clr_work_flags(struct cam_iosched_softc *isc, uint32_t flags)
1863 isc->flags &= ~flags;
2052 cam_iosched_io_metric_update(struct cam_iosched_softc *isc,
2057 cam_iosched_update(&isc->read_stats, sim_latency, bp);
2060 cam_iosched_update(&isc->write_stats, sim_latency, bp);
2063 cam_iosched_update(&isc->trim_stats, sim_latency, bp);
2087 struct cam_iosched_softc *isc;
2093 isc = (struct cam_iosched_softc *)addr;
2094 db_printf("pending_reads: %d\n", isc->read_stats.pending);
2095 db_printf("min_reads: %d\n", isc->read_stats.min);
2096 db_printf("max_reads: %d\n", isc->read_stats.max);
2097 db_printf("reads: %d\n", isc->read_stats.total);
2098 db_printf("in_reads: %d\n", isc->read_stats.in);
2099 db_printf("out_reads: %d\n", isc->read_stats.out);
2100 db_printf("queued_reads: %d\n", isc->read_stats.queued);
2101 db_printf("Read Q len %d\n", biolen(&isc->bio_queue));
2102 db_printf("pending_writes: %d\n", isc->write_stats.pending);
2103 db_printf("min_writes: %d\n", isc->write_stats.min);
2104 db_printf("max_writes: %d\n", isc->write_stats.max);
2105 db_printf("writes: %d\n", isc->write_stats.total);
2106 db_printf("in_writes: %d\n", isc->write_stats.in);
2107 db_printf("out_writes: %d\n", isc->write_stats.out);
2108 db_printf("queued_writes: %d\n", isc->write_stats.queued);
2109 db_printf("Write Q len %d\n", biolen(&isc->write_queue));
2110 db_printf("pending_trims: %d\n", isc->trim_stats.pending);
2111 db_printf("min_trims: %d\n", isc->trim_stats.min);
2112 db_printf("max_trims: %d\n", isc->trim_stats.max);
2113 db_printf("trims: %d\n", isc->trim_stats.total);
2114 db_printf("in_trims: %d\n", isc->trim_stats.in);
2115 db_printf("out_trims: %d\n", isc->trim_stats.out);
2116 db_printf("queued_trims: %d\n", isc->trim_stats.queued);
2117 db_printf("Trim Q len %d\n", biolen(&isc->trim_queue));
2118 db_printf("read_bias: %d\n", isc->read_bias);
2119 db_printf("current_read_bias: %d\n", isc->current_read_bias);
2121 (isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) ? "yes" : "no");