Lines Matching defs:zone
41 * The basic ideas stem from similar slab/zone based allocators whose algorithms
115 * This is the zone and keg from which all zones are spawned.
133 * One zone is for slab headers that can represent a larger number of items,
134 * making the slabs themselves more efficient, and the other zone is for
147 * The initial hash tables come out of this zone so they can be allocated
212 * This structure is passed as the zone ctor arg so that I don't have to create
231 uma_zone_t zone;
294 static void bucket_cache_reclaim(uma_zone_t zone, bool, int);
301 static inline void item_dtor(uma_zone_t zone, void *item, int size,
304 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
308 static void zone_timeout(uma_zone_t zone, void *);
316 static int zone_alloc_limit(uma_zone_t zone, int count, int flags);
317 static void zone_free_limit(uma_zone_t zone, int count);
320 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
321 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
325 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
327 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
342 static uint64_t uma_zone_get_allocs(uma_zone_t zone);
348 static uint64_t uma_keg_get_allocs(uma_keg_t zone);
352 static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
353 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
354 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
389 * Select the slab zone for an offpage slab with the given maximum item count.
412 * For each zone, calculate the memory required for each bucket, consisting
432 * Given a desired number of entries for a bucket, return the zone from which
464 bucket_alloc(uma_zone_t zone, void *udata, int flags)
476 * To limit bucket recursion we store the original zone flags
480 * a bucket for a bucket zone so we do not allow infinite bucket
485 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
486 udata = (void *)(uintptr_t)zone->uz_flags;
494 ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size));
495 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
504 zone->uz_bucket_size_max);
506 CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p",
507 zone->uz_name, zone, bucket);
514 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
519 bucket_drain(zone, bucket);
525 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
526 udata = (void *)(uintptr_t)zone->uz_flags;
546 kasan_mark_item_valid(uma_zone_t zone, void *item)
552 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0)
555 sz = zone->uz_size;
557 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) {
568 kasan_mark_item_invalid(uma_zone_t zone, void *item)
574 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0)
577 sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE);
578 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) {
614 kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused)
619 kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused)
636 kmsan_mark_item_uninitialized(uma_zone_t zone, void *item)
642 if ((zone->uz_flags &
651 * zone and thus cannot safely be marked by UMA.
658 if (zone->uz_keg->uk_init != NULL) {
667 sz = zone->uz_size;
668 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) {
683 kmsan_mark_item_uninitialized(uma_zone_t zone __unused, void *item __unused)
692 zone_domain_lock(uma_zone_t zone, int domain)
697 zdom = ZDOM_GET(zone, domain);
703 if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
704 zone->uz_bucket_size++;
713 zone_domain_lowest(uma_zone_t zone, int pref)
722 nitems = ZDOM_GET(zone, i)->uzd_nitems;
741 zone_domain_highest(uma_zone_t zone, int pref)
747 if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX)
753 nitems = ZDOM_GET(zone, i)->uzd_nitems;
788 * zone's caches. If a bucket is found the zone is not locked on return.
791 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim)
804 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 &&
806 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false))
809 dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR;
846 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size,
854 * whether the bucket's contents should be counted as part of the zone's working
858 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata,
866 zdom = zone_domain_lock(zone, domain);
872 if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) {
902 bucket_free(zone, bucket, udata);
1045 * Attempt to fetch a bucket from a zone on behalf of the current cpu cache.
1048 cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain)
1057 zdom = ZDOM_GET(zone, domain);
1063 !smr_poll(zone->uz_smr, seq, false))
1067 * Check the zone's cache of buckets.
1069 zdom = zone_domain_lock(zone, domain);
1070 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL)
1078 zone_log_warning(uma_zone_t zone)
1082 if (!zone_warnings || zone->uz_warning == NULL)
1085 if (ratecheck(&zone->uz_ratecheck, &warninterval))
1086 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
1090 zone_maxaction(uma_zone_t zone)
1093 if (zone->uz_maxaction.ta_func != NULL)
1094 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction);
1119 * Update the working set size estimates for the zone's bucket cache.
1171 zone_timeout(uma_zone_t zone, void *unused)
1176 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0)
1179 keg = zone->uz_keg;
1225 if ((zone->uz_flags & (UMA_ZONE_UNMANAGED | UMA_ZONE_NOTRIM)) == 0) {
1227 if (bucket_cache_reclaim_domain(zone, false, false, i) &&
1228 (zone->uz_flags & UMA_ZFLAG_CACHE) == 0)
1229 keg_drain(zone->uz_keg, i);
1338 * zone The zone to free to, must be unlocked.
1345 bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
1352 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 &&
1354 smr_wait(zone->uz_smr, bucket->ub_seq);
1357 item_dtor(zone, bucket->ub_bucket[i],
1358 zone->uz_size, NULL, SKIP_NONE);
1360 if (zone->uz_fini)
1362 kasan_mark_item_valid(zone, bucket->ub_bucket[i]);
1363 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
1364 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]);
1366 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
1367 if (zone->uz_max_items > 0)
1368 zone_free_limit(zone, bucket->ub_cnt);
1376 * Drains the per cpu caches for a zone.
1378 * NOTE: This may only be called while the zone is being torn down, and not
1383 * zone The zone to drain, must be unlocked.
1389 cache_drain(uma_zone_t zone)
1398 * tearing down the zone anyway. I.e., there will be no further use
1401 * XXX: It would good to be able to assert that the zone is being
1405 if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
1406 seq = smr_advance(zone->uz_smr);
1408 cache = &zone->uz_cpu[cpu];
1411 bucket_free(zone, bucket, NULL);
1415 bucket_free(zone, bucket, NULL);
1420 bucket_free(zone, bucket, NULL);
1423 bucket_cache_reclaim(zone, true, UMA_ANYDOMAIN);
1427 cache_shrink(uma_zone_t zone, void *unused)
1430 if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
1433 ZONE_LOCK(zone);
1434 zone->uz_bucket_size =
1435 (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
1436 ZONE_UNLOCK(zone);
1440 cache_drain_safe_cpu(uma_zone_t zone, void *unused)
1446 if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
1451 cache = &zone->uz_cpu[curcpu];
1456 * Don't flush SMR zone buckets. This leaves the zone without a
1459 if ((zone->uz_flags & UMA_ZONE_SMR) == 0) {
1466 zone_free_bucket(zone, b1, NULL, domain, false);
1468 zone_free_bucket(zone, b2, NULL, domain, false);
1472 zone_free_bucket(zone, b3, NULL, domain, false);
1477 * Safely drain per-CPU caches of a zone(s) to alloc bucket.
1484 pcpu_cache_drain_safe(uma_zone_t zone)
1491 if (zone)
1492 cache_shrink(zone, NULL);
1501 if (zone)
1502 cache_drain_safe_cpu(zone, NULL);
1512 * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller
1517 bucket_cache_reclaim_domain(uma_zone_t zone, bool drain, bool trim, int domain)
1528 zdom = ZDOM_GET(zone, domain);
1529 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) {
1530 ZONE_CROSS_LOCK(zone);
1533 ZONE_CROSS_UNLOCK(zone);
1535 bucket_free(zone, bucket, NULL);
1539 * If we were asked to drain the zone, we are done only once
1541 * excess of the zone's estimated working set size. Multiple
1560 bucket = zone_fetch_bucket(zone, zdom, true);
1563 bucket_free(zone, bucket, NULL);
1572 bucket_cache_reclaim(uma_zone_t zone, bool drain, int domain)
1577 * Shrink the zone bucket size to ensure that the per-CPU caches
1580 if (zone->uz_bucket_size > zone->uz_bucket_size_min)
1581 zone->uz_bucket_size--;
1584 (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) {
1585 bucket_cache_reclaim_domain(zone, drain, true, domain);
1588 bucket_cache_reclaim_domain(zone, drain, true, i);
1711 zone_reclaim(uma_zone_t zone, int domain, int waitok, bool drain)
1715 * zone_dtor(), which removes the zone from global lists before
1718 * The zone may be destroyed while sleeping, so only zone_dtor() should
1721 ZONE_LOCK(zone);
1723 while (zone->uz_reclaimers > 0)
1724 msleep(zone, ZONE_LOCKPTR(zone), PVM, "zonedrain", 1);
1726 zone->uz_reclaimers++;
1727 ZONE_UNLOCK(zone);
1728 bucket_cache_reclaim(zone, drain, domain);
1730 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0)
1731 keg_drain(zone->uz_keg, domain);
1732 ZONE_LOCK(zone);
1733 zone->uz_reclaimers--;
1734 if (zone->uz_reclaimers == 0)
1735 wakeup(zone);
1736 ZONE_UNLOCK(zone);
1753 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
1781 * first time they are added to a zone.
1797 /* zone is passed for legacy reasons. */
1799 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags);
1824 zone, slab);
1877 startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
1943 page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
1955 pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
2020 noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
2031 keg = zone->uz_keg;
2074 contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
2085 uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
2132 * Frees pcpu zone allocations
2293 * keg The zone we should initialize
2474 uma_zone_t zone;
2494 * The primary zone is passed to us at keg-creation time.
2496 zone = arg->zone;
2497 keg->uk_name = zone->uz_name;
2576 ("zone %s ipers %d rsize %d size %d slab won't fit",
2577 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size));
2583 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone);
2585 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
2594 zone_kva_available(uma_zone_t zone, void *unused)
2598 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
2600 KEG_GET(zone, keg);
2615 zone_alloc_counters(uma_zone_t zone, void *unused)
2618 zone->uz_allocs = counter_u64_alloc(M_WAITOK);
2619 zone->uz_frees = counter_u64_alloc(M_WAITOK);
2620 zone->uz_fails = counter_u64_alloc(M_WAITOK);
2621 zone->uz_xdomain = counter_u64_alloc(M_WAITOK);
2625 zone_alloc_sysctl(uma_zone_t zone, void *unused)
2632 static const char *nokeg = "cache zone";
2636 * Make a sysctl safe copy of the zone name by removing
2640 if (zone->uz_namecnt != 0) {
2642 for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++)
2644 zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1,
2646 sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name,
2647 zone->uz_namecnt);
2649 zone->uz_ctlname = strdup(zone->uz_name, M_UMA);
2650 for (c = zone->uz_ctlname; *c != '\0'; c++)
2657 zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma),
2658 OID_AUTO, zone->uz_ctlname, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2659 oid = zone->uz_oid;
2661 "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size");
2664 zone, 0, sysctl_handle_uma_zone_flags, "A",
2667 "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0,
2670 "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0,
2676 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0)
2680 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
2682 keg = zone->uz_keg;
2683 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) {
2727 * Information about zone limits.
2729 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
2733 zone, 0, sysctl_handle_uma_zone_items, "QU",
2736 "max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
2739 "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
2742 "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
2743 "Total zone limit sleeps");
2745 "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0,
2749 * Per-domain zone information.
2751 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
2754 zdom = ZDOM_GET(zone, i);
2784 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
2788 zone, 1, sysctl_handle_uma_zone_cur, "I",
2792 zone, 0, sysctl_handle_uma_zone_allocs, "QU",
2796 zone, 0, sysctl_handle_uma_zone_frees, "QU",
2799 "fails", CTLFLAG_RD, &zone->uz_fails,
2802 "xdomain", CTLFLAG_RD, &zone->uz_xdomain,
2812 zone_count(uma_zone_t zone, void *arg)
2822 if (strcmp(zone->uz_name, cnt->name) == 0)
2824 zone->uz_namecnt + 1);
2828 zone_update_caches(uma_zone_t zone)
2833 cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size);
2834 cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags);
2850 uma_zone_t zone = mem;
2855 bzero(zone, size);
2856 zone->uz_name = arg->name;
2857 zone->uz_ctor = arg->ctor;
2858 zone->uz_dtor = arg->dtor;
2859 zone->uz_init = NULL;
2860 zone->uz_fini = NULL;
2861 zone->uz_sleeps = 0;
2862 zone->uz_bucket_size = 0;
2863 zone->uz_bucket_size_min = 0;
2864 zone->uz_bucket_size_max = BUCKET_MAX;
2865 zone->uz_flags = (arg->flags & UMA_ZONE_SMR);
2866 zone->uz_warning = NULL;
2868 zone->uz_bucket_max = ULONG_MAX;
2869 timevalclear(&zone->uz_ratecheck);
2875 zone->uz_namecnt = cnt.count;
2876 ZONE_CROSS_LOCK_INIT(zone);
2879 zdom = ZDOM_GET(zone, i);
2880 ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS));
2886 zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR;
2893 * This is a pure cache zone, no kegs.
2897 ("zone_ctor: Import specified for non-cache zone."));
2898 zone->uz_flags = arg->flags;
2899 zone->uz_size = arg->size;
2900 zone->uz_import = arg->import;
2901 zone->uz_release = arg->release;
2902 zone->uz_arg = arg->arg;
2909 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0)
2910 zone->uz_flags |= UMA_ZONE_ROUNDROBIN;
2913 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
2919 * Use the regular zone/keg/slab allocator.
2921 zone->uz_import = zone_import;
2922 zone->uz_release = zone_release;
2923 zone->uz_arg = zone;
2927 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
2928 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
2929 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
2930 zone->uz_init = arg->uminit;
2931 zone->uz_fini = arg->fini;
2932 zone->uz_flags |= UMA_ZONE_SECONDARY;
2934 ZONE_LOCK(zone);
2937 LIST_INSERT_AFTER(z, zone, uz_link);
2941 ZONE_UNLOCK(zone);
2944 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
2957 karg.zone = zone;
2965 zone->uz_keg = keg;
2966 zone->uz_size = keg->uk_size;
2967 zone->uz_flags |= (keg->uk_flags &
2972 zone_alloc_counters(zone, NULL);
2974 zone_alloc_sysctl(zone, NULL);
2976 zone->uz_allocs = EARLY_COUNTER;
2977 zone->uz_frees = EARLY_COUNTER;
2978 zone->uz_fails = EARLY_COUNTER;
2982 if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
2983 zone->uz_smr = smr_create(zone->uz_name, 0, 0);
2987 ("Invalid zone flag combination"));
2989 zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
2991 zone->uz_bucket_size = BUCKET_MAX;
2993 zone->uz_bucket_size = 0;
2995 zone->uz_bucket_size = bucket_select(zone->uz_size);
2996 zone->uz_bucket_size_min = zone->uz_bucket_size;
2997 if (zone->uz_dtor != NULL || zone->uz_ctor != NULL)
2998 zone->uz_flags |= UMA_ZFLAG_CTORDTOR;
2999 zone_update_caches(zone);
3043 uma_zone_t zone;
3047 zone = (uma_zone_t)arg;
3049 sysctl_remove_oid(zone->uz_oid, 1, 1);
3051 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
3052 cache_drain(zone);
3055 LIST_REMOVE(zone, uz_link);
3057 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) {
3058 keg = zone->uz_keg;
3061 zone_reclaim(zone, UMA_ANYDOMAIN, M_WAITOK, true);
3066 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) {
3067 keg = zone->uz_keg;
3073 counter_u64_free(zone->uz_allocs);
3074 counter_u64_free(zone->uz_frees);
3075 counter_u64_free(zone->uz_fails);
3076 counter_u64_free(zone->uz_xdomain);
3077 free(zone->uz_ctlname, M_UMA);
3079 ZDOM_LOCK_FINI(ZDOM_GET(zone, i));
3080 ZONE_CROSS_LOCK_FINI(zone);
3087 uma_zone_t zone;
3090 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
3091 zfunc(zone, arg);
3093 LIST_FOREACH(zone, &uma_cachezones, uz_link)
3094 zfunc(zone, arg);
3098 * Traverses every zone in the system and calls a callback
3101 * zfunc A pointer to a function which accepts a zone
3143 /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */
3157 /* "manually" create the initial zone */
3216 /* Set up radix zone to use noobj_alloc. */
3276 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
3286 args.zone = zone;
3335 /* This stuff is essential for the zone ctor */
3423 uma_zdestroy(uma_zone_t zone)
3431 zone->uz_fini == NULL && zone->uz_release == zone_release)
3434 zone_free_item(zones, zone, NULL, SKIP_NONE);
3439 uma_zwait(uma_zone_t zone)
3442 if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
3443 uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK));
3444 else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0)
3445 uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK));
3447 uma_zfree(zone, uma_zalloc(zone, M_WAITOK));
3451 uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags)
3457 MPASS(zone->uz_flags & UMA_ZONE_PCPU);
3459 item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO);
3466 bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size);
3468 bzero(item, zone->uz_size);
3478 uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata)
3483 MPASS(zone->uz_flags & UMA_ZONE_PCPU);
3491 uma_zfree_arg(zone, item, udata);
3495 item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags,
3502 kasan_mark_item_valid(zone, item);
3503 kmsan_mark_item_uninitialized(zone, item);
3506 skipdbg = uma_dbg_zskip(zone, item);
3508 zone->uz_ctor != trash_ctor)
3509 trash_ctor(item, size, zone, flags);
3514 __predict_false(zone->uz_ctor != NULL) &&
3515 zone->uz_ctor(item, size, udata, flags) != 0) {
3516 counter_u64_add(zone->uz_fails, 1);
3517 zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
3522 uma_dbg_alloc(zone, NULL, item);
3531 item_dtor(uma_zone_t zone, void *item, int size, void *udata,
3537 skipdbg = uma_dbg_zskip(zone, item);
3539 if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0)
3540 uma_dbg_free(zone, udata, item);
3542 uma_dbg_free(zone, NULL, item);
3546 if (zone->uz_dtor != NULL)
3547 zone->uz_dtor(item, size, udata);
3549 if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 &&
3550 zone->uz_dtor != trash_dtor)
3551 trash_dtor(item, size, zone);
3554 kasan_mark_item_invalid(zone, item);
3576 uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags)
3584 "uma_zalloc_debug: zone \"%s\"", zone->uz_name);
3593 KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0,
3594 ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO"));
3623 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 &&
3624 memguard_cmp_zone(zone)) {
3626 item = memguard_alloc(zone->uz_size, flags);
3629 if (zone->uz_init != NULL &&
3630 zone->uz_init(item, zone->uz_size, flags) != 0) {
3634 if (zone->uz_ctor != NULL &&
3635 zone->uz_ctor(item, zone->uz_size, udata,
3637 counter_u64_add(zone->uz_fails, 1);
3638 if (zone->uz_fini != NULL)
3639 zone->uz_fini(item, zone->uz_size);
3653 uma_zfree_debug(uma_zone_t zone, void *item, void *udata)
3659 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 &&
3661 if (zone->uz_dtor != NULL)
3662 zone->uz_dtor(item, zone->uz_size, udata);
3663 if (zone->uz_fini != NULL)
3664 zone->uz_fini(item, zone->uz_size);
3674 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket,
3684 return (item_ctor(zone, uz_flags, size, udata, flags, item));
3688 cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
3693 while (cache_alloc(zone, cache, udata, flags)) {
3694 cache = &zone->uz_cpu[curcpu];
3698 return (cache_alloc_item(zone, cache, bucket, udata, flags));
3705 if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH)
3709 return (zone_alloc_item(zone, udata, domain, flags));
3714 uma_zalloc_smr(uma_zone_t zone, int flags)
3719 CTR3(KTR_UMA, "uma_zalloc_smr zone %s(%p) flags %d", zone->uz_name,
3720 zone, flags);
3725 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
3726 ("uma_zalloc_arg: called with non-SMR zone."));
3727 if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN)
3732 cache = &zone->uz_cpu[curcpu];
3735 return (cache_alloc_retry(zone, cache, NULL, flags));
3736 return (cache_alloc_item(zone, cache, bucket, NULL, flags));
3741 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
3747 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
3750 CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name,
3751 zone, flags);
3756 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
3757 ("uma_zalloc_arg: called with SMR zone."));
3758 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN)
3769 * order to acquire the zone mutex if we are unable to allocate from
3774 cache = &zone->uz_cpu[curcpu];
3777 return (cache_alloc_retry(zone, cache, udata, flags));
3778 return (cache_alloc_item(zone, cache, bucket, udata, flags));
3789 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
3820 bucket_free(zone, bucket, udata);
3825 * we must go back to the zone. This requires the zdom lock, so we
3835 domain = zone_domain_highest(zone, domain);
3836 bucket = cache_fetch_bucket(zone, cache, domain);
3837 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) {
3838 bucket = zone_alloc_bucket(zone, udata, domain, flags);
3844 CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p",
3845 zone->uz_name, zone, bucket);
3857 cache = &zone->uz_cpu[curcpu];
3863 atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax,
3873 zone_put_bucket(zone, domain, bucket, udata, !new);
3880 uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags)
3889 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
3892 CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d",
3893 zone->uz_name, zone, domain, flags);
3895 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
3896 ("uma_zalloc_domain: called with SMR zone."));
3898 KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0,
3899 ("uma_zalloc_domain: called with non-FIRSTTOUCH zone."));
3902 return (uma_zalloc_arg(zone, udata, flags));
3905 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN)
3916 zdom = zone_domain_lock(zone, domain);
3917 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) {
3923 zone_put_bucket(zone, domain, bucket, udata, true);
3924 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata,
3930 counter_u64_add(zone->uz_allocs, 1);
3935 return (zone_alloc_item(zone, udata, domain, flags));
3937 return (uma_zalloc_arg(zone, udata, flags));
4003 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
4042 slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
4113 uma_zone_t zone;
4121 zone = arg;
4123 keg = zone->uz_keg;
4126 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)
4146 * If the zone is striped we pick a new slab for every
4153 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 &&
4169 zone_alloc_limit_hard(uma_zone_t zone, int count, int flags)
4181 zone_free_limit(zone, count);
4182 zone_log_warning(zone);
4183 zone_maxaction(zone);
4192 sleepq_lock(&zone->uz_max_items);
4193 old = zone->uz_items;
4197 max = zone->uz_max_items;
4203 } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0);
4207 sleepq_release(&zone->uz_max_items);
4217 atomic_add_32(&zone->uz_sleepers, 1);
4218 atomic_add_64(&zone->uz_sleeps, 1);
4224 sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0);
4225 sleepq_wait(&zone->uz_max_items, PVM);
4234 atomic_subtract_32(&zone->uz_sleepers, 1);
4235 old = atomic_fetchadd_64(&zone->uz_items,
4246 max = zone->uz_max_items;
4251 zone_free_limit(zone, total + count - max);
4254 wakeup_one(&zone->uz_max_items);
4266 zone_alloc_limit(uma_zone_t zone, int count, int flags)
4271 max = zone->uz_max_items;
4278 old = atomic_fetchadd_64(&zone->uz_items, count);
4289 zone_free_limit(zone, (old + count) - max);
4292 return (zone_alloc_limit_hard(zone, count, flags));
4299 zone_free_limit(uma_zone_t zone, int count)
4309 old = atomic_fetchadd_64(&zone->uz_items, -count);
4311 UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items))
4318 wakeup_one(&zone->uz_max_items);
4322 zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags)
4327 CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name,
4328 zone, domain);
4333 else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
4336 if (zone->uz_max_items > 0)
4337 maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size,
4340 maxbucket = zone->uz_bucket_size;
4345 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
4351 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
4357 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
4361 kasan_mark_item_valid(zone, bucket->ub_bucket[i]);
4362 error = zone->uz_init(bucket->ub_bucket[i],
4363 zone->uz_size, flags);
4364 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]);
4374 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
4386 bucket_free(zone, bucket, udata);
4387 counter_u64_add(zone->uz_fails, 1);
4391 if (zone->uz_max_items > 0 && cnt < maxbucket)
4392 zone_free_limit(zone, maxbucket - cnt);
4398 * Allocates a single item from a zone.
4401 * zone The zone to alloc for.
4412 zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags)
4416 if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) {
4417 counter_u64_add(zone->uz_fails, 1);
4425 if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1)
4429 * We have to call both the zone's init (not the keg's init)
4430 * and the zone's ctor. This is because the item is going from
4432 * to be both zone-init'd as well as zone-ctor'd.
4434 if (zone->uz_init != NULL) {
4437 kasan_mark_item_valid(zone, item);
4438 error = zone->uz_init(item, zone->uz_size, flags);
4439 kasan_mark_item_invalid(zone, item);
4441 zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT);
4445 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags,
4450 counter_u64_add(zone->uz_allocs, 1);
4452 zone->uz_name, zone);
4457 counter_u64_add(zone->uz_fails, 1);
4459 if (zone->uz_max_items > 0)
4460 zone_free_limit(zone, 1);
4462 zone->uz_name, zone);
4469 uma_zfree_smr(uma_zone_t zone, void *item)
4478 CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p",
4479 zone->uz_name, zone, item);
4482 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
4483 ("uma_zfree_smr: called with non-SMR zone."));
4485 SMR_ASSERT_NOT_ENTERED(zone->uz_smr);
4486 if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN)
4489 cache = &zone->uz_cpu[curcpu];
4498 cache = &zone->uz_cpu[curcpu];
4512 } while (cache_free(zone, cache, NULL, itemdomain));
4518 zone_free_item(zone, item, NULL, SKIP_NONE);
4523 uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
4530 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
4532 CTR3(KTR_UMA, "uma_zfree_arg zone %s(%p) item %p",
4533 zone->uz_name, zone, item);
4536 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
4537 ("uma_zfree_arg: called with SMR zone."));
4538 if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN)
4550 cache = &zone->uz_cpu[curcpu];
4554 item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE);
4561 if (atomic_load_32(&zone->uz_sleepers) > 0)
4572 * order to acquire the zone mutex if we are unable to free to the
4583 cache = &zone->uz_cpu[curcpu];
4607 } while (cache_free(zone, cache, udata, itemdomain));
4614 zone_free_item(zone, item, udata, SKIP_DTOR);
4623 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata)
4633 "uma_zfree: zone %s(%p) draining cross bucket %p",
4634 zone->uz_name, zone, bucket);
4641 if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
4642 seq = smr_advance(zone->uz_smr);
4651 ZONE_CROSS_LOCK(zone);
4655 zdom = ZDOM_GET(zone, domain);
4667 ZONE_CROSS_UNLOCK(zone);
4668 b = bucket_alloc(zone, udata, M_NOWAIT);
4671 ZONE_CROSS_LOCK(zone);
4690 ZONE_CROSS_UNLOCK(zone);
4694 bucket_free(zone, bucket, udata);
4698 bucket_free(zone, b, udata);
4703 zone_put_bucket(zone, domain, b, udata, true);
4709 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
4720 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
4722 zone_free_cross(zone, bucket, udata);
4728 * Attempt to save the bucket in the zone's domain bucket cache.
4731 "uma_zfree: zone %s(%p) putting bucket %p on free list",
4732 zone->uz_name, zone, bucket);
4734 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
4735 itemdomain = zone_domain_lowest(zone, itemdomain);
4736 zone_put_bucket(zone, itemdomain, bucket, udata, ws);
4741 * existing full bucket either to the zone cache or back to the slab layer.
4748 cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, int itemdomain)
4755 if (zone->uz_bucket_size == 0)
4758 cache = &zone->uz_cpu[curcpu];
4772 counter_u64_add(zone->uz_xdomain,
4790 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) {
4792 bucket->ub_seq = smr_advance(zone->uz_smr);
4793 newbucket = bucket_alloc(zone, udata, M_NOWAIT);
4795 bucket_drain(zone, bucket);
4800 newbucket = bucket_alloc(zone, udata, M_NOWAIT);
4803 zone_free_bucket(zone, bucket, udata, itemdomain, true);
4808 cache = &zone->uz_cpu[curcpu];
4828 bucket_free(zone, bucket, udata);
4837 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)
4843 keg = zone->uz_keg;
4870 uma_zone_t zone;
4877 zone = arg;
4878 keg = zone->uz_keg;
4880 if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0))
4884 if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) {
4888 if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0)
4898 slab_free_item(zone, slab, item);
4905 * Frees a single item to any zone.
4908 * zone The zone to free to
4914 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
4918 * If a free is sent directly to an SMR zone we have to
4923 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE)
4924 smr_synchronize(zone->uz_smr);
4926 item_dtor(zone, item, zone->uz_size, udata, skip);
4928 if (skip < SKIP_FINI && zone->uz_fini) {
4929 kasan_mark_item_valid(zone, item);
4930 zone->uz_fini(item, zone->uz_size);
4931 kasan_mark_item_invalid(zone, item);
4934 zone->uz_release(zone->uz_arg, &item, 1);
4939 counter_u64_add(zone->uz_frees, 1);
4941 if (zone->uz_max_items > 0)
4942 zone_free_limit(zone, 1);
4947 uma_zone_set_max(uma_zone_t zone, int nitems)
4954 uma_zone_set_maxcache(zone, nitems);
4957 * XXX This can misbehave if the zone has any allocations with
4961 ZONE_LOCK(zone);
4962 if (zone->uz_max_items == 0)
4963 ZONE_ASSERT_COLD(zone);
4964 zone->uz_max_items = nitems;
4965 zone->uz_flags |= UMA_ZFLAG_LIMIT;
4966 zone_update_caches(zone);
4968 wakeup(&zone->uz_max_items);
4969 ZONE_UNLOCK(zone);
4976 uma_zone_set_maxcache(uma_zone_t zone, int nitems)
4980 ZONE_LOCK(zone);
4984 * the zone. Each CPU gets at least two buckets, and for cross-domain
4992 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) {
5003 zone->uz_bucket_size_max = zone->uz_bucket_size = bsize;
5004 if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
5005 zone->uz_bucket_size_min = zone->uz_bucket_size_max;
5006 zone->uz_bucket_max = nitems - nb * bsize;
5007 ZONE_UNLOCK(zone);
5012 uma_zone_get_max(uma_zone_t zone)
5016 nitems = atomic_load_64(&zone->uz_max_items);
5023 uma_zone_set_warning(uma_zone_t zone, const char *warning)
5026 ZONE_ASSERT_COLD(zone);
5027 zone->uz_warning = warning;
5032 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
5035 ZONE_ASSERT_COLD(zone);
5036 TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone);
5041 uma_zone_get_cur(uma_zone_t zone)
5047 if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER)
5048 nitems = counter_u64_fetch(zone->uz_allocs) -
5049 counter_u64_fetch(zone->uz_frees);
5051 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) -
5052 atomic_load_64(&zone->uz_cpu[i].uc_frees);
5058 uma_zone_get_allocs(uma_zone_t zone)
5064 if (zone->uz_allocs != EARLY_COUNTER)
5065 nitems = counter_u64_fetch(zone->uz_allocs);
5067 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs);
5073 uma_zone_get_frees(uma_zone_t zone)
5079 if (zone->uz_frees != EARLY_COUNTER)
5080 nitems = counter_u64_fetch(zone->uz_frees);
5082 nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees);
5105 uma_zone_set_init(uma_zone_t zone, uma_init uminit)
5109 KEG_GET(zone, keg);
5116 uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
5120 KEG_GET(zone, keg);
5127 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
5130 ZONE_ASSERT_COLD(zone);
5131 zone->uz_init = zinit;
5136 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
5139 ZONE_ASSERT_COLD(zone);
5140 zone->uz_fini = zfini;
5145 uma_zone_set_freef(uma_zone_t zone, uma_free freef)
5149 KEG_GET(zone, keg);
5156 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
5160 KEG_GET(zone, keg);
5167 uma_zone_set_smr(uma_zone_t zone, smr_t smr)
5170 ZONE_ASSERT_COLD(zone);
5173 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
5174 ("zone %p (%s) already uses SMR", zone, zone->uz_name));
5175 zone->uz_flags |= UMA_ZONE_SMR;
5176 zone->uz_smr = smr;
5177 zone_update_caches(zone);
5181 uma_zone_get_smr(uma_zone_t zone)
5184 return (zone->uz_smr);
5189 uma_zone_reserve(uma_zone_t zone, int items)
5193 KEG_GET(zone, keg);
5200 uma_zone_reserve_kva(uma_zone_t zone, int count)
5206 KEG_GET(zone, keg);
5208 ZONE_ASSERT_COLD(zone);
5226 zone->uz_max_items = pages * keg->uk_ipers;
5233 zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5234 zone_update_caches(zone);
5241 uma_prealloc(uma_zone_t zone, int items)
5249 KEG_GET(zone, keg);
5256 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5281 uma_zone_memory(uma_zone_t zone)
5287 if (zone->uz_flags & UMA_ZFLAG_CACHE) {
5289 sz += ZDOM_GET(zone, i)->uzd_nitems;
5290 return (sz * zone->uz_size);
5293 sz += zone->uz_keg->uk_domain[i].ud_pages;
5304 uma_reclaim_domain_cb(uma_zone_t zone, void *arg)
5309 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) != 0)
5312 (zone->uz_flags & UMA_ZONE_NOTRIM) !=0)
5315 uma_zone_reclaim_domain(zone, args->req, args->domain);
5351 * Some slabs may have been freed but this zone will be visited early
5391 uma_zone_reclaim(uma_zone_t zone, int req)
5393 uma_zone_reclaim_domain(zone, req, UMA_ANYDOMAIN);
5397 uma_zone_reclaim_domain(uma_zone_t zone, int req, int domain)
5401 zone_reclaim(zone, domain, M_NOWAIT, false);
5404 zone_reclaim(zone, domain, M_NOWAIT, true);
5407 pcpu_cache_drain_safe(zone);
5408 zone_reclaim(zone, domain, M_NOWAIT, true);
5417 uma_zone_exhausted(uma_zone_t zone)
5420 return (atomic_load_32(&zone->uz_sleepers) > 0);
5453 * Generate statistics across both the zone and its per-cpu cache's. Return
5456 * Note: does not update the zone statistics, as it can't safely clear the
5610 * A zone is secondary is it is not the first entry
5611 * on the keg's zone list.
5643 uma_zone_t zone = *(uma_zone_t *)arg1;
5646 max = uma_zone_get_max(zone);
5651 uma_zone_set_max(zone, max);
5659 uma_zone_t zone;
5667 zone = *(uma_zone_t *)arg1;
5669 zone = arg1;
5670 cur = uma_zone_get_cur(zone);
5677 uma_zone_t zone = arg1;
5680 cur = uma_zone_get_allocs(zone);
5687 uma_zone_t zone = arg1;
5690 cur = uma_zone_get_frees(zone);
5698 uma_zone_t zone = arg1;
5702 if (zone->uz_flags != 0)
5703 sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS);
5736 uma_zone_t zone = arg1;
5739 cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items));
5745 uma_dbg_getslab(uma_zone_t zone, void *item)
5753 * zone is unlocked because the item's allocation state
5757 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5759 if (zone->uz_flags & UMA_ZFLAG_VTOSLAB)
5761 keg = zone->uz_keg;
5772 uma_dbg_zskip(uma_zone_t zone, void *mem)
5775 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5778 return (uma_dbg_kskip(zone->uz_keg, mem));
5812 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
5818 slab = uma_dbg_getslab(zone, item);
5820 panic("uma: item %p did not belong to zone %s",
5821 item, zone->uz_name);
5823 keg = zone->uz_keg;
5828 panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)",
5829 item, zone, zone->uz_name, slab, freei);
5838 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
5844 slab = uma_dbg_getslab(zone, item);
5846 panic("uma: Freed item %p did not belong to zone %s",
5847 item, zone->uz_name);
5849 keg = zone->uz_keg;
5853 panic("Invalid free of %p from zone %p(%s) slab %p(%d)",
5854 item, zone, zone->uz_name, slab, freei);
5857 panic("Unaligned free of %p from zone %p(%s) slab %p(%d)",
5858 item, zone, zone->uz_name, slab, freei);
5862 panic("Duplicate free of %p from zone %p(%s) slab %p(%d)",
5863 item, zone, zone->uz_name, slab, freei);
5932 * zone, we have already printed all preceding