1*f90eedcaSdlg /* $OpenBSD: kern_intrmap.c,v 1.3 2020/06/23 01:40:03 dlg Exp $ */
27bd9aa6bSdlg
37bd9aa6bSdlg /*
47bd9aa6bSdlg * Copyright (c) 1980, 1986, 1993
57bd9aa6bSdlg * The Regents of the University of California. All rights reserved.
67bd9aa6bSdlg *
77bd9aa6bSdlg * Redistribution and use in source and binary forms, with or without
87bd9aa6bSdlg * modification, are permitted provided that the following conditions
97bd9aa6bSdlg * are met:
107bd9aa6bSdlg * 1. Redistributions of source code must retain the above copyright
117bd9aa6bSdlg * notice, this list of conditions and the following disclaimer.
127bd9aa6bSdlg * 2. Redistributions in binary form must reproduce the above copyright
137bd9aa6bSdlg * notice, this list of conditions and the following disclaimer in the
147bd9aa6bSdlg * documentation and/or other materials provided with the distribution.
157bd9aa6bSdlg * 3. Neither the name of the University nor the names of its contributors
167bd9aa6bSdlg * may be used to endorse or promote products derived from this software
177bd9aa6bSdlg * without specific prior written permission.
187bd9aa6bSdlg *
197bd9aa6bSdlg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
207bd9aa6bSdlg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
217bd9aa6bSdlg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
227bd9aa6bSdlg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
237bd9aa6bSdlg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
247bd9aa6bSdlg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
257bd9aa6bSdlg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
267bd9aa6bSdlg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
277bd9aa6bSdlg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
287bd9aa6bSdlg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
297bd9aa6bSdlg * SUCH DAMAGE.
307bd9aa6bSdlg *
317bd9aa6bSdlg * @(#)if.c 8.3 (Berkeley) 1/4/94
327bd9aa6bSdlg * $FreeBSD: src/sys/net/if.c,v 1.185 2004/03/13 02:35:03 brooks Exp $
337bd9aa6bSdlg */
347bd9aa6bSdlg
357bd9aa6bSdlg /*
367bd9aa6bSdlg * This code is adapted from the if_ringmap code in DragonflyBSD,
377bd9aa6bSdlg * but generalised for use by all types of devices, not just network
387bd9aa6bSdlg * cards.
397bd9aa6bSdlg */
407bd9aa6bSdlg
417bd9aa6bSdlg #include <sys/param.h>
427bd9aa6bSdlg #include <sys/systm.h>
437bd9aa6bSdlg #include <sys/device.h>
447bd9aa6bSdlg #include <sys/malloc.h>
457bd9aa6bSdlg #include <sys/rwlock.h>
467bd9aa6bSdlg
477bd9aa6bSdlg #include <sys/intrmap.h>
487bd9aa6bSdlg
497bd9aa6bSdlg struct intrmap_cpus {
507bd9aa6bSdlg struct refcnt ic_refs;
517bd9aa6bSdlg unsigned int ic_count;
52d9f5f731Sdlg struct cpu_info **ic_cpumap;
537bd9aa6bSdlg };
547bd9aa6bSdlg
557bd9aa6bSdlg struct intrmap {
567bd9aa6bSdlg unsigned int im_count;
577bd9aa6bSdlg unsigned int im_grid;
587bd9aa6bSdlg struct intrmap_cpus *
597bd9aa6bSdlg im_cpus;
607bd9aa6bSdlg unsigned int *im_cpumap;
617bd9aa6bSdlg };
627bd9aa6bSdlg
637bd9aa6bSdlg /*
647bd9aa6bSdlg * The CPUs that should be used for interrupts may be a subset of all CPUs.
657bd9aa6bSdlg */
667bd9aa6bSdlg
677bd9aa6bSdlg struct rwlock intrmap_lock = RWLOCK_INITIALIZER("intrcpus");
687bd9aa6bSdlg struct intrmap_cpus *intrmap_cpus = NULL;
697bd9aa6bSdlg int intrmap_ncpu = 0;
707bd9aa6bSdlg
717bd9aa6bSdlg static void
intrmap_cpus_put(struct intrmap_cpus * ic)727bd9aa6bSdlg intrmap_cpus_put(struct intrmap_cpus *ic)
737bd9aa6bSdlg {
747bd9aa6bSdlg if (ic == NULL)
757bd9aa6bSdlg return;
767bd9aa6bSdlg
777bd9aa6bSdlg if (refcnt_rele(&ic->ic_refs)) {
787bd9aa6bSdlg free(ic->ic_cpumap, M_DEVBUF,
797bd9aa6bSdlg ic->ic_count * sizeof(*ic->ic_cpumap));
807bd9aa6bSdlg free(ic, M_DEVBUF, sizeof(*ic));
817bd9aa6bSdlg }
827bd9aa6bSdlg }
837bd9aa6bSdlg
847bd9aa6bSdlg static struct intrmap_cpus *
intrmap_cpus_get(void)857bd9aa6bSdlg intrmap_cpus_get(void)
867bd9aa6bSdlg {
877bd9aa6bSdlg struct intrmap_cpus *oic = NULL;
887bd9aa6bSdlg struct intrmap_cpus *ic;
897bd9aa6bSdlg
907bd9aa6bSdlg rw_enter_write(&intrmap_lock);
917bd9aa6bSdlg if (intrmap_ncpu != ncpus) {
927bd9aa6bSdlg unsigned int icpus = 0;
93d9f5f731Sdlg struct cpu_info **cpumap;
947bd9aa6bSdlg CPU_INFO_ITERATOR cii;
957bd9aa6bSdlg struct cpu_info *ci;
967bd9aa6bSdlg
977bd9aa6bSdlg /*
987bd9aa6bSdlg * there's a new "version" of the set of CPUs available, so
997bd9aa6bSdlg * we need to figure out which ones we can use for interrupts.
1007bd9aa6bSdlg */
1017bd9aa6bSdlg
1027bd9aa6bSdlg cpumap = mallocarray(ncpus, sizeof(*cpumap),
1037bd9aa6bSdlg M_DEVBUF, M_WAITOK);
1047bd9aa6bSdlg
1057bd9aa6bSdlg CPU_INFO_FOREACH(cii, ci) {
1067bd9aa6bSdlg #ifdef __HAVE_CPU_TOPOLOGY
1077bd9aa6bSdlg if (ci->ci_smt_id > 0)
1087bd9aa6bSdlg continue;
1097bd9aa6bSdlg #endif
110d9f5f731Sdlg cpumap[icpus++] = ci;
1117bd9aa6bSdlg }
1127bd9aa6bSdlg
1137bd9aa6bSdlg if (icpus < ncpus) {
1147bd9aa6bSdlg /* this is mostly about free(9) needing a size */
115d9f5f731Sdlg struct cpu_info **icpumap = mallocarray(icpus,
1167bd9aa6bSdlg sizeof(*icpumap), M_DEVBUF, M_WAITOK);
1177bd9aa6bSdlg memcpy(icpumap, cpumap, icpus * sizeof(*icpumap));
1187bd9aa6bSdlg free(cpumap, M_DEVBUF, ncpus * sizeof(*cpumap));
1197bd9aa6bSdlg cpumap = icpumap;
1207bd9aa6bSdlg }
1217bd9aa6bSdlg
1227bd9aa6bSdlg ic = malloc(sizeof(*ic), M_DEVBUF, M_WAITOK);
1237bd9aa6bSdlg refcnt_init(&ic->ic_refs);
1247bd9aa6bSdlg ic->ic_count = icpus;
1257bd9aa6bSdlg ic->ic_cpumap = cpumap;
1267bd9aa6bSdlg
1277bd9aa6bSdlg oic = intrmap_cpus;
1287bd9aa6bSdlg intrmap_cpus = ic; /* give this ref to the global. */
1297bd9aa6bSdlg } else
1307bd9aa6bSdlg ic = intrmap_cpus;
1317bd9aa6bSdlg
1327bd9aa6bSdlg refcnt_take(&ic->ic_refs); /* take a ref for the caller */
1337bd9aa6bSdlg rw_exit_write(&intrmap_lock);
1347bd9aa6bSdlg
1357bd9aa6bSdlg intrmap_cpus_put(oic);
1367bd9aa6bSdlg
1377bd9aa6bSdlg return (ic);
1387bd9aa6bSdlg }
1397bd9aa6bSdlg
1407bd9aa6bSdlg static int
intrmap_nintrs(const struct intrmap_cpus * ic,unsigned int nintrs,unsigned int maxintrs)1417bd9aa6bSdlg intrmap_nintrs(const struct intrmap_cpus *ic, unsigned int nintrs,
1427bd9aa6bSdlg unsigned int maxintrs)
1437bd9aa6bSdlg {
1447bd9aa6bSdlg KASSERTMSG(maxintrs > 0, "invalid maximum interrupt count %u",
1457bd9aa6bSdlg maxintrs);
1467bd9aa6bSdlg
1477bd9aa6bSdlg if (nintrs == 0 || nintrs > maxintrs)
1487bd9aa6bSdlg nintrs = maxintrs;
1497bd9aa6bSdlg if (nintrs > ic->ic_count)
1507bd9aa6bSdlg nintrs = ic->ic_count;
1517bd9aa6bSdlg return (nintrs);
1527bd9aa6bSdlg }
1537bd9aa6bSdlg
1547bd9aa6bSdlg static void
intrmap_set_grid(struct intrmap * im,unsigned int unit,unsigned int grid)1557bd9aa6bSdlg intrmap_set_grid(struct intrmap *im, unsigned int unit, unsigned int grid)
1567bd9aa6bSdlg {
1577bd9aa6bSdlg unsigned int i, offset;
1587bd9aa6bSdlg unsigned int *cpumap = im->im_cpumap;
1597bd9aa6bSdlg const struct intrmap_cpus *ic = im->im_cpus;
1607bd9aa6bSdlg
1617bd9aa6bSdlg KASSERTMSG(grid > 0, "invalid if_ringmap grid %u", grid);
1627bd9aa6bSdlg KASSERTMSG(grid >= im->im_count, "invalid intrmap grid %u, count %u",
1637bd9aa6bSdlg grid, im->im_count);
1647bd9aa6bSdlg im->im_grid = grid;
1657bd9aa6bSdlg
1667bd9aa6bSdlg offset = (grid * unit) % ic->ic_count;
1677bd9aa6bSdlg for (i = 0; i < im->im_count; i++) {
1687bd9aa6bSdlg cpumap[i] = offset + i;
1697bd9aa6bSdlg KASSERTMSG(cpumap[i] < ic->ic_count,
1707bd9aa6bSdlg "invalid cpumap[%u] = %u, offset %u (ncpu %d)", i,
1717bd9aa6bSdlg cpumap[i], offset, ic->ic_count);
1727bd9aa6bSdlg }
1737bd9aa6bSdlg }
1747bd9aa6bSdlg
1757bd9aa6bSdlg struct intrmap *
intrmap_create(const struct device * dv,unsigned int nintrs,unsigned int maxintrs,unsigned int flags)1767bd9aa6bSdlg intrmap_create(const struct device *dv,
1777bd9aa6bSdlg unsigned int nintrs, unsigned int maxintrs, unsigned int flags)
1787bd9aa6bSdlg {
1797bd9aa6bSdlg struct intrmap *im;
1807bd9aa6bSdlg unsigned int unit = dv->dv_unit;
1817bd9aa6bSdlg unsigned int i, grid = 0, prev_grid;
1827bd9aa6bSdlg struct intrmap_cpus *ic;
1837bd9aa6bSdlg
1847bd9aa6bSdlg ic = intrmap_cpus_get();
1857bd9aa6bSdlg
1867bd9aa6bSdlg nintrs = intrmap_nintrs(ic, nintrs, maxintrs);
1877bd9aa6bSdlg if (ISSET(flags, INTRMAP_POWEROF2))
1887bd9aa6bSdlg nintrs = 1 << (fls(nintrs) - 1);
1897bd9aa6bSdlg im = malloc(sizeof(*im), M_DEVBUF, M_WAITOK | M_ZERO);
1907bd9aa6bSdlg im->im_count = nintrs;
1917bd9aa6bSdlg im->im_cpus = ic;
1927bd9aa6bSdlg im->im_cpumap = mallocarray(nintrs, sizeof(*im->im_cpumap), M_DEVBUF,
1937bd9aa6bSdlg M_WAITOK | M_ZERO);
1947bd9aa6bSdlg
1957bd9aa6bSdlg prev_grid = ic->ic_count;
1967bd9aa6bSdlg for (i = 0; i < ic->ic_count; i++) {
1977bd9aa6bSdlg if (ic->ic_count % (i + 1) != 0)
1987bd9aa6bSdlg continue;
1997bd9aa6bSdlg
2007bd9aa6bSdlg grid = ic->ic_count / (i + 1);
2017bd9aa6bSdlg if (nintrs > grid) {
2027bd9aa6bSdlg grid = prev_grid;
2037bd9aa6bSdlg break;
2047bd9aa6bSdlg }
2057bd9aa6bSdlg
2067bd9aa6bSdlg if (nintrs > ic->ic_count / (i + 2))
2077bd9aa6bSdlg break;
2087bd9aa6bSdlg prev_grid = grid;
2097bd9aa6bSdlg }
2107bd9aa6bSdlg intrmap_set_grid(im, unit, grid);
2117bd9aa6bSdlg
2127bd9aa6bSdlg return (im);
2137bd9aa6bSdlg }
2147bd9aa6bSdlg
2157bd9aa6bSdlg void
intrmap_destroy(struct intrmap * im)2167bd9aa6bSdlg intrmap_destroy(struct intrmap *im)
2177bd9aa6bSdlg {
2187bd9aa6bSdlg free(im->im_cpumap, M_DEVBUF, im->im_count * sizeof(*im->im_cpumap));
2197bd9aa6bSdlg intrmap_cpus_put(im->im_cpus);
2207bd9aa6bSdlg free(im, M_DEVBUF, sizeof(*im));
2217bd9aa6bSdlg }
2227bd9aa6bSdlg
2237bd9aa6bSdlg /*
2247bd9aa6bSdlg * Align the two ringmaps.
2257bd9aa6bSdlg *
2267bd9aa6bSdlg * e.g. 8 netisrs, rm0 contains 4 rings, rm1 contains 2 rings.
2277bd9aa6bSdlg *
2287bd9aa6bSdlg * Before:
2297bd9aa6bSdlg *
2307bd9aa6bSdlg * CPU 0 1 2 3 4 5 6 7
2317bd9aa6bSdlg * NIC_RX n0 n1 n2 n3
2327bd9aa6bSdlg * NIC_TX N0 N1
2337bd9aa6bSdlg *
2347bd9aa6bSdlg * After:
2357bd9aa6bSdlg *
2367bd9aa6bSdlg * CPU 0 1 2 3 4 5 6 7
2377bd9aa6bSdlg * NIC_RX n0 n1 n2 n3
2387bd9aa6bSdlg * NIC_TX N0 N1
2397bd9aa6bSdlg */
2407bd9aa6bSdlg void
intrmap_align(const struct device * dv,struct intrmap * im0,struct intrmap * im1)2417bd9aa6bSdlg intrmap_align(const struct device *dv,
2427bd9aa6bSdlg struct intrmap *im0, struct intrmap *im1)
2437bd9aa6bSdlg {
2447bd9aa6bSdlg unsigned int unit = dv->dv_unit;
2457bd9aa6bSdlg
2467bd9aa6bSdlg KASSERT(im0->im_cpus == im1->im_cpus);
2477bd9aa6bSdlg
2487bd9aa6bSdlg if (im0->im_grid > im1->im_grid)
2497bd9aa6bSdlg intrmap_set_grid(im1, unit, im0->im_grid);
2507bd9aa6bSdlg else if (im0->im_grid < im1->im_grid)
2517bd9aa6bSdlg intrmap_set_grid(im0, unit, im1->im_grid);
2527bd9aa6bSdlg }
2537bd9aa6bSdlg
2547bd9aa6bSdlg void
intrmap_match(const struct device * dv,struct intrmap * im0,struct intrmap * im1)2557bd9aa6bSdlg intrmap_match(const struct device *dv,
2567bd9aa6bSdlg struct intrmap *im0, struct intrmap *im1)
2577bd9aa6bSdlg {
2587bd9aa6bSdlg unsigned int unit = dv->dv_unit;
2597bd9aa6bSdlg const struct intrmap_cpus *ic;
2607bd9aa6bSdlg unsigned int subset_grid, cnt, divisor, mod, offset, i;
2617bd9aa6bSdlg struct intrmap *subset_im, *im;
2627bd9aa6bSdlg unsigned int old_im0_grid, old_im1_grid;
2637bd9aa6bSdlg
2647bd9aa6bSdlg KASSERT(im0->im_cpus == im1->im_cpus);
2657bd9aa6bSdlg if (im0->im_grid == im1->im_grid)
2667bd9aa6bSdlg return;
2677bd9aa6bSdlg
2687bd9aa6bSdlg /* Save grid for later use */
2697bd9aa6bSdlg old_im0_grid = im0->im_grid;
2707bd9aa6bSdlg old_im1_grid = im1->im_grid;
2717bd9aa6bSdlg
2727bd9aa6bSdlg intrmap_align(dv, im0, im1);
2737bd9aa6bSdlg
2747bd9aa6bSdlg /*
2757bd9aa6bSdlg * Re-shuffle rings to get more even distribution.
2767bd9aa6bSdlg *
2777bd9aa6bSdlg * e.g. 12 netisrs, rm0 contains 4 rings, rm1 contains 2 rings.
2787bd9aa6bSdlg *
2797bd9aa6bSdlg * CPU 0 1 2 3 4 5 6 7 8 9 10 11
2807bd9aa6bSdlg *
2817bd9aa6bSdlg * NIC_RX a0 a1 a2 a3 b0 b1 b2 b3 c0 c1 c2 c3
2827bd9aa6bSdlg * NIC_TX A0 A1 B0 B1 C0 C1
2837bd9aa6bSdlg *
2847bd9aa6bSdlg * NIC_RX d0 d1 d2 d3 e0 e1 e2 e3 f0 f1 f2 f3
2857bd9aa6bSdlg * NIC_TX D0 D1 E0 E1 F0 F1
2867bd9aa6bSdlg */
2877bd9aa6bSdlg
2887bd9aa6bSdlg if (im0->im_count >= (2 * old_im1_grid)) {
2897bd9aa6bSdlg cnt = im0->im_count;
2907bd9aa6bSdlg subset_grid = old_im1_grid;
2917bd9aa6bSdlg subset_im = im1;
2927bd9aa6bSdlg im = im0;
2937bd9aa6bSdlg } else if (im1->im_count > (2 * old_im0_grid)) {
2947bd9aa6bSdlg cnt = im1->im_count;
2957bd9aa6bSdlg subset_grid = old_im0_grid;
2967bd9aa6bSdlg subset_im = im0;
2977bd9aa6bSdlg im = im1;
2987bd9aa6bSdlg } else {
2997bd9aa6bSdlg /* No space to shuffle. */
3007bd9aa6bSdlg return;
3017bd9aa6bSdlg }
3027bd9aa6bSdlg
3037bd9aa6bSdlg ic = im0->im_cpus;
3047bd9aa6bSdlg
3057bd9aa6bSdlg mod = cnt / subset_grid;
3067bd9aa6bSdlg KASSERT(mod >= 2);
3077bd9aa6bSdlg divisor = ic->ic_count / im->im_grid;
3087bd9aa6bSdlg offset = ((unit / divisor) % mod) * subset_grid;
3097bd9aa6bSdlg
3107bd9aa6bSdlg for (i = 0; i < subset_im->im_count; i++) {
3117bd9aa6bSdlg subset_im->im_cpumap[i] += offset;
3127bd9aa6bSdlg KASSERTMSG(subset_im->im_cpumap[i] < ic->ic_count,
3137bd9aa6bSdlg "match: invalid cpumap[%d] = %d, offset %d",
3147bd9aa6bSdlg i, subset_im->im_cpumap[i], offset);
3157bd9aa6bSdlg }
3167bd9aa6bSdlg #ifdef DIAGNOSTIC
3177bd9aa6bSdlg for (i = 0; i < subset_im->im_count; i++) {
3187bd9aa6bSdlg unsigned int j;
3197bd9aa6bSdlg
3207bd9aa6bSdlg for (j = 0; j < im->im_count; j++) {
3217bd9aa6bSdlg if (im->im_cpumap[j] == subset_im->im_cpumap[i])
3227bd9aa6bSdlg break;
3237bd9aa6bSdlg }
3247bd9aa6bSdlg KASSERTMSG(j < im->im_count,
3257bd9aa6bSdlg "subset cpumap[%u] = %u not found in superset",
3267bd9aa6bSdlg i, subset_im->im_cpumap[i]);
3277bd9aa6bSdlg }
3287bd9aa6bSdlg #endif
3297bd9aa6bSdlg }
3307bd9aa6bSdlg
3317bd9aa6bSdlg unsigned int
intrmap_count(const struct intrmap * im)3327bd9aa6bSdlg intrmap_count(const struct intrmap *im)
3337bd9aa6bSdlg {
3347bd9aa6bSdlg return (im->im_count);
3357bd9aa6bSdlg }
3367bd9aa6bSdlg
337d9f5f731Sdlg struct cpu_info *
intrmap_cpu(const struct intrmap * im,unsigned int ring)3387bd9aa6bSdlg intrmap_cpu(const struct intrmap *im, unsigned int ring)
3397bd9aa6bSdlg {
3407bd9aa6bSdlg const struct intrmap_cpus *ic = im->im_cpus;
3417bd9aa6bSdlg unsigned int icpu;
3427bd9aa6bSdlg KASSERTMSG(ring < im->im_count, "invalid ring %u", ring);
3437bd9aa6bSdlg icpu = im->im_cpumap[ring];
3447bd9aa6bSdlg KASSERTMSG(icpu < ic->ic_count, "invalid interrupt cpu %u for ring %u"
3457bd9aa6bSdlg " (intrmap %p)", icpu, ring, im);
3467bd9aa6bSdlg return (ic->ic_cpumap[icpu]);
3477bd9aa6bSdlg }
348*f90eedcaSdlg
349*f90eedcaSdlg struct cpu_info *
intrmap_one(const struct device * dv)350*f90eedcaSdlg intrmap_one(const struct device *dv)
351*f90eedcaSdlg {
352*f90eedcaSdlg unsigned int unit = dv->dv_unit;
353*f90eedcaSdlg struct intrmap_cpus *ic;
354*f90eedcaSdlg struct cpu_info *ci;
355*f90eedcaSdlg
356*f90eedcaSdlg ic = intrmap_cpus_get();
357*f90eedcaSdlg ci = ic->ic_cpumap[unit % ic->ic_count];
358*f90eedcaSdlg intrmap_cpus_put(ic);
359*f90eedcaSdlg
360*f90eedcaSdlg return (ci);
361*f90eedcaSdlg }
362