10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 53866Sraf * Common Development and Distribution License (the "License"). 63866Sraf * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 213866Sraf 220Sstevel@tonic-gate /* 236812Sraf * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 240Sstevel@tonic-gate * Use is subject to license terms. 250Sstevel@tonic-gate */ 260Sstevel@tonic-gate 270Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 280Sstevel@tonic-gate 290Sstevel@tonic-gate #include <mtmalloc.h> 300Sstevel@tonic-gate #include "mtmalloc_impl.h" 310Sstevel@tonic-gate #include <unistd.h> 320Sstevel@tonic-gate #include <synch.h> 330Sstevel@tonic-gate #include <thread.h> 343866Sraf #include <pthread.h> 350Sstevel@tonic-gate #include <stdio.h> 360Sstevel@tonic-gate #include <limits.h> 370Sstevel@tonic-gate #include <errno.h> 380Sstevel@tonic-gate #include <string.h> 390Sstevel@tonic-gate #include <strings.h> 400Sstevel@tonic-gate #include <sys/param.h> 410Sstevel@tonic-gate #include <sys/sysmacros.h> 420Sstevel@tonic-gate 430Sstevel@tonic-gate /* 440Sstevel@tonic-gate * To turn on the asserts just compile -DDEBUG 450Sstevel@tonic-gate */ 460Sstevel@tonic-gate 470Sstevel@tonic-gate #ifndef DEBUG 480Sstevel@tonic-gate #define NDEBUG 490Sstevel@tonic-gate #endif 500Sstevel@tonic-gate 510Sstevel@tonic-gate #include <assert.h> 520Sstevel@tonic-gate 530Sstevel@tonic-gate /* 540Sstevel@tonic-gate * The MT hot malloc implementation contained herein is designed to be 550Sstevel@tonic-gate * plug-compatible with the libc version of malloc. It is not intended 560Sstevel@tonic-gate * to replace that implementation until we decide that it is ok to break 570Sstevel@tonic-gate * customer apps (Solaris 3.0). 580Sstevel@tonic-gate * 590Sstevel@tonic-gate * For requests up to 2^^16, the allocator initializes itself into NCPUS 600Sstevel@tonic-gate * worth of chains of caches. When a memory request is made, the calling thread 610Sstevel@tonic-gate * is vectored into one of NCPUS worth of caches. The LWP id gives us a cheap, 620Sstevel@tonic-gate * contention-reducing index to use, eventually, this should be replaced with 630Sstevel@tonic-gate * the actual CPU sequence number, when an interface to get it is available. 640Sstevel@tonic-gate * 650Sstevel@tonic-gate * Once the thread is vectored into one of the list of caches the real 660Sstevel@tonic-gate * allocation of the memory begins. The size is determined to figure out which 670Sstevel@tonic-gate * bucket the allocation should be satisfied from. The management of free 680Sstevel@tonic-gate * buckets is done via a bitmask. A free bucket is represented by a 1. The 690Sstevel@tonic-gate * first free bit represents the first free bucket. The position of the bit, 700Sstevel@tonic-gate * represents the position of the bucket in the arena. 710Sstevel@tonic-gate * 720Sstevel@tonic-gate * When the memory from the arena is handed out, the address of the cache 730Sstevel@tonic-gate * control structure is written in the word preceeding the returned memory. 740Sstevel@tonic-gate * This cache control address is used during free() to mark the buffer free 750Sstevel@tonic-gate * in the cache control structure. 760Sstevel@tonic-gate * 770Sstevel@tonic-gate * When all available memory in a cache has been depleted, a new chunk of memory 780Sstevel@tonic-gate * is allocated via sbrk(). The new cache is allocated from this chunk of memory 790Sstevel@tonic-gate * and initialized in the function create_cache(). New caches are installed at 800Sstevel@tonic-gate * the front of a singly linked list of the same size memory pools. This helps 810Sstevel@tonic-gate * to ensure that there will tend to be available memory in the beginning of the 820Sstevel@tonic-gate * list. 830Sstevel@tonic-gate * 840Sstevel@tonic-gate * Long linked lists hurt performance. To decrease this effect, there is a 850Sstevel@tonic-gate * tunable, requestsize, that bumps up the sbrk allocation size and thus 860Sstevel@tonic-gate * increases the number of available blocks within an arena. We also keep 870Sstevel@tonic-gate * a "hint" for each cache list, which is the last cache in the list allocated 880Sstevel@tonic-gate * from. This lowers the cost of searching if there are a lot of fully 890Sstevel@tonic-gate * allocated blocks at the front of the list. 900Sstevel@tonic-gate * 910Sstevel@tonic-gate * For requests greater than 2^^16 (oversize allocations), there are two pieces 920Sstevel@tonic-gate * of overhead. There is the OVERHEAD used to hold the cache addr 930Sstevel@tonic-gate * (&oversize_list), plus an oversize_t structure to further describe the block. 940Sstevel@tonic-gate * 950Sstevel@tonic-gate * The oversize list is kept as defragmented as possible by coalescing 960Sstevel@tonic-gate * freed oversized allocations with adjacent neighbors. 970Sstevel@tonic-gate * 980Sstevel@tonic-gate * Addresses handed out are stored in a hash table, and are aligned on 990Sstevel@tonic-gate * MTMALLOC_MIN_ALIGN-byte boundaries at both ends. Request sizes are rounded-up 1000Sstevel@tonic-gate * where necessary in order to achieve this. This eases the implementation of 1010Sstevel@tonic-gate * MTDEBUGPATTERN and MTINITPATTERN, particularly where coalescing occurs. 1020Sstevel@tonic-gate * 1030Sstevel@tonic-gate * A memalign allocation takes memalign header overhead. There's two 1040Sstevel@tonic-gate * types of memalign headers distinguished by MTMALLOC_MEMALIGN_MAGIC 1050Sstevel@tonic-gate * and MTMALLOC_MEMALIGN_MIN_MAGIC. When the size of memory taken to 1060Sstevel@tonic-gate * get to the aligned address from malloc'ed address is the minimum size 1070Sstevel@tonic-gate * OVERHEAD, we create a header taking only one OVERHEAD space with magic 1080Sstevel@tonic-gate * number MTMALLOC_MEMALIGN_MIN_MAGIC, and we know by subtracting OVERHEAD 1090Sstevel@tonic-gate * from memaligned address, we can get to the malloc'ed address. Otherwise, 1100Sstevel@tonic-gate * we create a memalign header taking two OVERHEAD space, one stores 1110Sstevel@tonic-gate * MTMALLOC_MEMALIGN_MAGIC magic number, the other one points back to the 1120Sstevel@tonic-gate * malloc'ed address. 1130Sstevel@tonic-gate */ 1140Sstevel@tonic-gate 1150Sstevel@tonic-gate #if defined(__i386) || defined(__amd64) 1160Sstevel@tonic-gate #include <arpa/inet.h> /* for htonl() */ 1170Sstevel@tonic-gate #endif 1180Sstevel@tonic-gate 1190Sstevel@tonic-gate static void * morecore(size_t); 1200Sstevel@tonic-gate static void create_cache(cache_t *, size_t bufsize, uint_t hunks); 1210Sstevel@tonic-gate static void * malloc_internal(size_t, percpu_t *); 1220Sstevel@tonic-gate static void * oversize(size_t); 1230Sstevel@tonic-gate static oversize_t *find_oversize(size_t); 1240Sstevel@tonic-gate static void add_oversize(oversize_t *); 1250Sstevel@tonic-gate static void copy_pattern(uint32_t, void *, size_t); 1260Sstevel@tonic-gate static void * verify_pattern(uint32_t, void *, size_t); 1270Sstevel@tonic-gate static void reinit_cpu_list(void); 1280Sstevel@tonic-gate static void reinit_cache(cache_t *); 1290Sstevel@tonic-gate static void free_oversize(oversize_t *); 1300Sstevel@tonic-gate static oversize_t *oversize_header_alloc(uintptr_t, size_t); 1310Sstevel@tonic-gate 1320Sstevel@tonic-gate /* 1330Sstevel@tonic-gate * oversize hash table stuff 1340Sstevel@tonic-gate */ 1350Sstevel@tonic-gate #define NUM_BUCKETS 67 /* must be prime */ 1360Sstevel@tonic-gate #define HASH_OVERSIZE(caddr) ((uintptr_t)(caddr) % NUM_BUCKETS) 1370Sstevel@tonic-gate oversize_t *ovsz_hashtab[NUM_BUCKETS]; 1380Sstevel@tonic-gate 1390Sstevel@tonic-gate #define ALIGN(x, a) ((((uintptr_t)(x) + ((uintptr_t)(a) - 1)) \ 1400Sstevel@tonic-gate & ~((uintptr_t)(a) - 1))) 1410Sstevel@tonic-gate 1420Sstevel@tonic-gate /* need this to deal with little endianess of x86 */ 1430Sstevel@tonic-gate #if defined(__i386) || defined(__amd64) 1440Sstevel@tonic-gate #define FLIP_EM(x) htonl((x)) 1450Sstevel@tonic-gate #else 1460Sstevel@tonic-gate #define FLIP_EM(x) (x) 1470Sstevel@tonic-gate #endif 1480Sstevel@tonic-gate 1490Sstevel@tonic-gate #define INSERT_ONLY 0 1500Sstevel@tonic-gate #define COALESCE_LEFT 0x00000001 1510Sstevel@tonic-gate #define COALESCE_RIGHT 0x00000002 1520Sstevel@tonic-gate #define COALESCE_WITH_BOTH_SIDES (COALESCE_LEFT | COALESCE_RIGHT) 1530Sstevel@tonic-gate 1540Sstevel@tonic-gate #define OVERHEAD 8 /* size needed to write cache addr */ 1550Sstevel@tonic-gate #define HUNKSIZE 8192 /* just a multiplier */ 1560Sstevel@tonic-gate 1570Sstevel@tonic-gate #define MAX_CACHED_SHIFT 16 /* 64K is the max cached size */ 1580Sstevel@tonic-gate #define MAX_CACHED (1 << MAX_CACHED_SHIFT) 1590Sstevel@tonic-gate #define MIN_CACHED_SHIFT 4 /* smaller requests rounded up */ 1600Sstevel@tonic-gate #define MTMALLOC_MIN_ALIGN 8 /* min guaranteed alignment */ 1610Sstevel@tonic-gate 1621412Srm88369 /* maximum size before overflow */ 1631412Srm88369 #define MAX_MTMALLOC (SIZE_MAX - (SIZE_MAX % MTMALLOC_MIN_ALIGN) \ 1641412Srm88369 - OVSZ_HEADER_SIZE) 1651412Srm88369 1660Sstevel@tonic-gate #define NUM_CACHES (MAX_CACHED_SHIFT - MIN_CACHED_SHIFT + 1) 1670Sstevel@tonic-gate #define CACHELIST_SIZE ALIGN(NUM_CACHES * sizeof (cache_head_t), \ 1680Sstevel@tonic-gate CACHE_COHERENCY_UNIT) 1690Sstevel@tonic-gate 1700Sstevel@tonic-gate #define MINSIZE 9 /* for requestsize, tunable */ 1710Sstevel@tonic-gate #define MAXSIZE 256 /* arbitrary, big enough, for requestsize */ 1720Sstevel@tonic-gate 1730Sstevel@tonic-gate #define FREEPATTERN 0xdeadbeef /* debug fill pattern for free buf */ 1740Sstevel@tonic-gate #define INITPATTERN 0xbaddcafe /* debug fill pattern for new buf */ 1750Sstevel@tonic-gate 1760Sstevel@tonic-gate #define misaligned(p) ((unsigned)(p) & (sizeof (int) - 1)) 1770Sstevel@tonic-gate #define IS_OVERSIZE(x, y) (((x) < (y)) && (((x) > MAX_CACHED)? 1 : 0)) 1780Sstevel@tonic-gate 1790Sstevel@tonic-gate static long requestsize = MINSIZE; /* 9 pages per cache; tunable; 9 is min */ 1800Sstevel@tonic-gate 1810Sstevel@tonic-gate static uint_t cpu_mask; 1820Sstevel@tonic-gate static curcpu_func curcpu; 1830Sstevel@tonic-gate 1840Sstevel@tonic-gate static int32_t debugopt; 1850Sstevel@tonic-gate static int32_t reinit; 1860Sstevel@tonic-gate 1870Sstevel@tonic-gate static percpu_t *cpu_list; 1880Sstevel@tonic-gate static oversize_t oversize_list; 1893866Sraf static mutex_t oversize_lock = DEFAULTMUTEX; 1900Sstevel@tonic-gate 1913866Sraf static int ncpus = 0; 1920Sstevel@tonic-gate 1930Sstevel@tonic-gate #define MTMALLOC_OVERSIZE_MAGIC ((uintptr_t)&oversize_list) 1940Sstevel@tonic-gate #define MTMALLOC_MEMALIGN_MAGIC ((uintptr_t)&oversize_list + 1) 1950Sstevel@tonic-gate #define MTMALLOC_MEMALIGN_MIN_MAGIC ((uintptr_t)&oversize_list + 2) 1960Sstevel@tonic-gate 1970Sstevel@tonic-gate /* 1980Sstevel@tonic-gate * We require allocations handed out to be aligned on MTMALLOC_MIN_ALIGN-byte 1990Sstevel@tonic-gate * boundaries. We round up sizeof (oversize_t) (when necessary) to ensure that 2000Sstevel@tonic-gate * this is achieved. 2010Sstevel@tonic-gate */ 2020Sstevel@tonic-gate #define OVSZ_SIZE (ALIGN(sizeof (oversize_t), MTMALLOC_MIN_ALIGN)) 2030Sstevel@tonic-gate #define OVSZ_HEADER_SIZE (OVSZ_SIZE + OVERHEAD) 2040Sstevel@tonic-gate 2050Sstevel@tonic-gate /* 2060Sstevel@tonic-gate * memalign header takes 2 OVERHEAD space. One for memalign magic, and the 2070Sstevel@tonic-gate * other one points back to the start address of originally allocated space. 2080Sstevel@tonic-gate */ 2090Sstevel@tonic-gate #define MEMALIGN_HEADER_SIZE 2 * OVERHEAD 2100Sstevel@tonic-gate #define MEMALIGN_HEADER_ALLOC(x, shift, malloc_addr)\ 2110Sstevel@tonic-gate if (shift == OVERHEAD)\ 2120Sstevel@tonic-gate *((uintptr_t *)((caddr_t)x - OVERHEAD)) = \ 2130Sstevel@tonic-gate MTMALLOC_MEMALIGN_MIN_MAGIC; \ 2140Sstevel@tonic-gate else {\ 2150Sstevel@tonic-gate *((uintptr_t *)((caddr_t)x - OVERHEAD)) = \ 2160Sstevel@tonic-gate MTMALLOC_MEMALIGN_MAGIC; \ 2170Sstevel@tonic-gate *((uintptr_t *)((caddr_t)x - 2 * OVERHEAD)) = \ 2180Sstevel@tonic-gate (uintptr_t)malloc_addr; \ 2190Sstevel@tonic-gate } 2200Sstevel@tonic-gate 221*7166Sraf /* 222*7166Sraf * Add big to the oversize hash table at the head of the relevant bucket. 223*7166Sraf */ 224*7166Sraf static void 225*7166Sraf insert_hash(oversize_t *big) 226*7166Sraf { 227*7166Sraf caddr_t ret = big->addr; 228*7166Sraf int bucket = HASH_OVERSIZE(ret); 229*7166Sraf 230*7166Sraf assert(MUTEX_HELD(&oversize_lock)); 231*7166Sraf big->hash_next = ovsz_hashtab[bucket]; 232*7166Sraf ovsz_hashtab[bucket] = big; 233*7166Sraf } 234*7166Sraf 2350Sstevel@tonic-gate void * 2360Sstevel@tonic-gate malloc(size_t bytes) 2370Sstevel@tonic-gate { 2380Sstevel@tonic-gate percpu_t *list_rotor; 2390Sstevel@tonic-gate uint_t list_index; 2400Sstevel@tonic-gate 2410Sstevel@tonic-gate if (bytes > MAX_CACHED) 2420Sstevel@tonic-gate return (oversize(bytes)); 2430Sstevel@tonic-gate 2440Sstevel@tonic-gate list_index = (curcpu() & cpu_mask); 2450Sstevel@tonic-gate 2460Sstevel@tonic-gate list_rotor = &cpu_list[list_index]; 2470Sstevel@tonic-gate 2480Sstevel@tonic-gate return (malloc_internal(bytes, list_rotor)); 2490Sstevel@tonic-gate } 2500Sstevel@tonic-gate 2510Sstevel@tonic-gate void * 2520Sstevel@tonic-gate realloc(void * ptr, size_t bytes) 2530Sstevel@tonic-gate { 2540Sstevel@tonic-gate void *new, *data_ptr; 2550Sstevel@tonic-gate cache_t *cacheptr; 2560Sstevel@tonic-gate caddr_t mem; 2570Sstevel@tonic-gate size_t shift = 0; 2580Sstevel@tonic-gate 2590Sstevel@tonic-gate if (ptr == NULL) 2600Sstevel@tonic-gate return (malloc(bytes)); 2610Sstevel@tonic-gate 2620Sstevel@tonic-gate if (bytes == 0) { 2630Sstevel@tonic-gate free(ptr); 2640Sstevel@tonic-gate return (NULL); 2650Sstevel@tonic-gate } 2660Sstevel@tonic-gate 2670Sstevel@tonic-gate data_ptr = ptr; 2680Sstevel@tonic-gate mem = (caddr_t)ptr - OVERHEAD; 2690Sstevel@tonic-gate 2700Sstevel@tonic-gate new = malloc(bytes); 2710Sstevel@tonic-gate 2720Sstevel@tonic-gate if (new == NULL) 2730Sstevel@tonic-gate return (NULL); 2740Sstevel@tonic-gate 2750Sstevel@tonic-gate /* 2760Sstevel@tonic-gate * If new == ptr, ptr has previously been freed. Passing a freed pointer 2770Sstevel@tonic-gate * to realloc() is not allowed - unless the caller specifically states 2780Sstevel@tonic-gate * otherwise, in which case we must avoid freeing ptr (ie new) before we 2790Sstevel@tonic-gate * return new. There is (obviously) no requirement to memcpy() ptr to 2800Sstevel@tonic-gate * new before we return. 2810Sstevel@tonic-gate */ 2820Sstevel@tonic-gate if (new == ptr) { 2830Sstevel@tonic-gate if (!(debugopt & MTDOUBLEFREE)) 2840Sstevel@tonic-gate abort(); 2850Sstevel@tonic-gate return (new); 2860Sstevel@tonic-gate } 2870Sstevel@tonic-gate 2880Sstevel@tonic-gate if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) { 2890Sstevel@tonic-gate mem -= OVERHEAD; 2900Sstevel@tonic-gate ptr = (void *)*(uintptr_t *)mem; 2910Sstevel@tonic-gate mem = (caddr_t)ptr - OVERHEAD; 2920Sstevel@tonic-gate shift = (size_t)((uintptr_t)data_ptr - (uintptr_t)ptr); 2930Sstevel@tonic-gate } else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) { 2940Sstevel@tonic-gate ptr = (void *) mem; 2950Sstevel@tonic-gate mem -= OVERHEAD; 2960Sstevel@tonic-gate shift = OVERHEAD; 2970Sstevel@tonic-gate } 2980Sstevel@tonic-gate 2990Sstevel@tonic-gate if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) { 3000Sstevel@tonic-gate oversize_t *old; 3010Sstevel@tonic-gate 3020Sstevel@tonic-gate old = (oversize_t *)(mem - OVSZ_SIZE); 3030Sstevel@tonic-gate (void) memcpy(new, data_ptr, MIN(bytes, old->size - shift)); 3040Sstevel@tonic-gate free(ptr); 3050Sstevel@tonic-gate return (new); 3060Sstevel@tonic-gate } 3070Sstevel@tonic-gate 3080Sstevel@tonic-gate cacheptr = (cache_t *)*(uintptr_t *)mem; 3090Sstevel@tonic-gate 3100Sstevel@tonic-gate (void) memcpy(new, data_ptr, 3110Sstevel@tonic-gate MIN(cacheptr->mt_size - OVERHEAD - shift, bytes)); 3120Sstevel@tonic-gate free(ptr); 3130Sstevel@tonic-gate 3140Sstevel@tonic-gate return (new); 3150Sstevel@tonic-gate } 3160Sstevel@tonic-gate 3170Sstevel@tonic-gate void * 3180Sstevel@tonic-gate calloc(size_t nelem, size_t bytes) 3190Sstevel@tonic-gate { 3200Sstevel@tonic-gate void * ptr; 3210Sstevel@tonic-gate size_t size = nelem * bytes; 3220Sstevel@tonic-gate 3230Sstevel@tonic-gate ptr = malloc(size); 3240Sstevel@tonic-gate if (ptr == NULL) 3250Sstevel@tonic-gate return (NULL); 3263866Sraf (void) memset(ptr, 0, size); 3270Sstevel@tonic-gate 3280Sstevel@tonic-gate return (ptr); 3290Sstevel@tonic-gate } 3300Sstevel@tonic-gate 3310Sstevel@tonic-gate void 3320Sstevel@tonic-gate free(void * ptr) 3330Sstevel@tonic-gate { 3340Sstevel@tonic-gate cache_t *cacheptr; 3350Sstevel@tonic-gate caddr_t mem; 3360Sstevel@tonic-gate int32_t i; 3370Sstevel@tonic-gate caddr_t freeblocks; 3380Sstevel@tonic-gate uintptr_t offset; 3390Sstevel@tonic-gate uchar_t mask; 3400Sstevel@tonic-gate int32_t which_bit, num_bytes; 3410Sstevel@tonic-gate 3420Sstevel@tonic-gate if (ptr == NULL) 3430Sstevel@tonic-gate return; 3440Sstevel@tonic-gate 3450Sstevel@tonic-gate mem = (caddr_t)ptr - OVERHEAD; 3460Sstevel@tonic-gate 3470Sstevel@tonic-gate if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) { 3480Sstevel@tonic-gate mem -= OVERHEAD; 3490Sstevel@tonic-gate ptr = (void *)*(uintptr_t *)mem; 3500Sstevel@tonic-gate mem = (caddr_t)ptr - OVERHEAD; 3510Sstevel@tonic-gate } else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) { 3520Sstevel@tonic-gate ptr = (void *) mem; 3530Sstevel@tonic-gate mem -= OVERHEAD; 3540Sstevel@tonic-gate } 3550Sstevel@tonic-gate 3560Sstevel@tonic-gate if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) { 3570Sstevel@tonic-gate oversize_t *big, **opp; 3580Sstevel@tonic-gate int bucket; 3590Sstevel@tonic-gate 3600Sstevel@tonic-gate big = (oversize_t *)(mem - OVSZ_SIZE); 3610Sstevel@tonic-gate (void) mutex_lock(&oversize_lock); 3620Sstevel@tonic-gate 3630Sstevel@tonic-gate bucket = HASH_OVERSIZE(big->addr); 3640Sstevel@tonic-gate for (opp = &ovsz_hashtab[bucket]; *opp != NULL; 3650Sstevel@tonic-gate opp = &(*opp)->hash_next) 3660Sstevel@tonic-gate if (*opp == big) 3670Sstevel@tonic-gate break; 3680Sstevel@tonic-gate 3690Sstevel@tonic-gate if (*opp == NULL) { 3700Sstevel@tonic-gate if (!(debugopt & MTDOUBLEFREE)) 3710Sstevel@tonic-gate abort(); 3720Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 3730Sstevel@tonic-gate return; 3740Sstevel@tonic-gate } 3750Sstevel@tonic-gate 3760Sstevel@tonic-gate *opp = big->hash_next; /* remove big from the hash table */ 3770Sstevel@tonic-gate big->hash_next = NULL; 3780Sstevel@tonic-gate 3790Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 3800Sstevel@tonic-gate copy_pattern(FREEPATTERN, ptr, big->size); 3810Sstevel@tonic-gate add_oversize(big); 3820Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 3830Sstevel@tonic-gate return; 3840Sstevel@tonic-gate } 3850Sstevel@tonic-gate 3860Sstevel@tonic-gate cacheptr = (cache_t *)*(uintptr_t *)mem; 3870Sstevel@tonic-gate freeblocks = cacheptr->mt_freelist; 3880Sstevel@tonic-gate 3890Sstevel@tonic-gate /* 3900Sstevel@tonic-gate * This is the distance measured in bits into the arena. 3910Sstevel@tonic-gate * The value of offset is in bytes but there is a 1-1 correlation 3920Sstevel@tonic-gate * between distance into the arena and distance into the 3930Sstevel@tonic-gate * freelist bitmask. 3940Sstevel@tonic-gate */ 3950Sstevel@tonic-gate offset = mem - cacheptr->mt_arena; 3960Sstevel@tonic-gate 3970Sstevel@tonic-gate /* 3980Sstevel@tonic-gate * i is total number of bits to offset into freelist bitmask. 3990Sstevel@tonic-gate */ 4000Sstevel@tonic-gate 4010Sstevel@tonic-gate i = offset / cacheptr->mt_size; 4020Sstevel@tonic-gate 4030Sstevel@tonic-gate num_bytes = i >> 3; 4040Sstevel@tonic-gate 4050Sstevel@tonic-gate /* 4060Sstevel@tonic-gate * which_bit is the bit offset into the byte in the freelist. 4070Sstevel@tonic-gate * if our freelist bitmask looks like 0xf3 and we are freeing 4080Sstevel@tonic-gate * block 5 (ie: the 6th block) our mask will be 0xf7 after 4090Sstevel@tonic-gate * the free. Things go left to right that's why the mask is 0x80 4100Sstevel@tonic-gate * and not 0x01. 4110Sstevel@tonic-gate */ 4120Sstevel@tonic-gate which_bit = i - (num_bytes << 3); 4130Sstevel@tonic-gate 4140Sstevel@tonic-gate mask = 0x80 >> which_bit; 4150Sstevel@tonic-gate 4160Sstevel@tonic-gate freeblocks += num_bytes; 4170Sstevel@tonic-gate 4180Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 4190Sstevel@tonic-gate copy_pattern(FREEPATTERN, ptr, cacheptr->mt_size - OVERHEAD); 4200Sstevel@tonic-gate 4210Sstevel@tonic-gate (void) mutex_lock(&cacheptr->mt_cache_lock); 4220Sstevel@tonic-gate 4230Sstevel@tonic-gate if (*freeblocks & mask) { 4240Sstevel@tonic-gate if (!(debugopt & MTDOUBLEFREE)) 4250Sstevel@tonic-gate abort(); 4260Sstevel@tonic-gate } else { 4270Sstevel@tonic-gate *freeblocks |= mask; 4280Sstevel@tonic-gate cacheptr->mt_nfree++; 4290Sstevel@tonic-gate } 4300Sstevel@tonic-gate 4310Sstevel@tonic-gate (void) mutex_unlock(&cacheptr->mt_cache_lock); 4320Sstevel@tonic-gate } 4330Sstevel@tonic-gate 4340Sstevel@tonic-gate void * 4350Sstevel@tonic-gate memalign(size_t alignment, size_t size) 4360Sstevel@tonic-gate { 4370Sstevel@tonic-gate size_t alloc_size; 4380Sstevel@tonic-gate uintptr_t offset; 4390Sstevel@tonic-gate void *alloc_buf; 4400Sstevel@tonic-gate void *ret_buf; 4410Sstevel@tonic-gate 4420Sstevel@tonic-gate if (size == 0 || alignment == 0 || 4430Sstevel@tonic-gate misaligned(alignment) || 4440Sstevel@tonic-gate (alignment & (alignment - 1)) != 0) { 4450Sstevel@tonic-gate errno = EINVAL; 4460Sstevel@tonic-gate return (NULL); 4470Sstevel@tonic-gate } 4480Sstevel@tonic-gate 4490Sstevel@tonic-gate /* <= MTMALLOC_MIN_ALIGN, malloc can provide directly */ 4500Sstevel@tonic-gate if (alignment <= MTMALLOC_MIN_ALIGN) 4510Sstevel@tonic-gate return (malloc(size)); 4520Sstevel@tonic-gate 4530Sstevel@tonic-gate alloc_size = size + alignment - MTMALLOC_MIN_ALIGN; 4540Sstevel@tonic-gate 4550Sstevel@tonic-gate if (alloc_size < size) { /* overflow */ 4560Sstevel@tonic-gate errno = ENOMEM; 4570Sstevel@tonic-gate return (NULL); 4580Sstevel@tonic-gate } 4590Sstevel@tonic-gate 4600Sstevel@tonic-gate alloc_buf = malloc(alloc_size); 4610Sstevel@tonic-gate 4620Sstevel@tonic-gate if (alloc_buf == NULL) 4630Sstevel@tonic-gate /* malloc sets errno */ 4640Sstevel@tonic-gate return (NULL); 4650Sstevel@tonic-gate 4660Sstevel@tonic-gate /* 4670Sstevel@tonic-gate * If alloc_size > MAX_CACHED, malloc() will have returned a multiple of 4680Sstevel@tonic-gate * MTMALLOC_MIN_ALIGN, having rounded-up alloc_size if necessary. Since 4690Sstevel@tonic-gate * we will use alloc_size to return the excess fragments to the free 4700Sstevel@tonic-gate * list, we also round-up alloc_size if necessary. 4710Sstevel@tonic-gate */ 4720Sstevel@tonic-gate if ((alloc_size > MAX_CACHED) && 4730Sstevel@tonic-gate (alloc_size & (MTMALLOC_MIN_ALIGN - 1))) 4740Sstevel@tonic-gate alloc_size = ALIGN(alloc_size, MTMALLOC_MIN_ALIGN); 4750Sstevel@tonic-gate 4760Sstevel@tonic-gate if ((offset = (uintptr_t)alloc_buf & (alignment - 1)) == 0) { 4770Sstevel@tonic-gate /* aligned correctly */ 4780Sstevel@tonic-gate 4790Sstevel@tonic-gate size_t frag_size = alloc_size - 4800Sstevel@tonic-gate (size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE); 4810Sstevel@tonic-gate 4820Sstevel@tonic-gate /* 4830Sstevel@tonic-gate * If the leftover piece of the memory > MAX_CACHED, 4840Sstevel@tonic-gate * split off the piece and return it back to the freelist. 4850Sstevel@tonic-gate */ 4860Sstevel@tonic-gate if (IS_OVERSIZE(frag_size, alloc_size)) { 4870Sstevel@tonic-gate oversize_t *orig, *tail; 4880Sstevel@tonic-gate uintptr_t taddr; 4890Sstevel@tonic-gate size_t data_size; 4900Sstevel@tonic-gate taddr = ALIGN((uintptr_t)alloc_buf + size, 4910Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 4920Sstevel@tonic-gate data_size = taddr - (uintptr_t)alloc_buf; 4930Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 4940Sstevel@tonic-gate OVSZ_HEADER_SIZE); 4950Sstevel@tonic-gate frag_size = orig->size - data_size - 4960Sstevel@tonic-gate OVSZ_HEADER_SIZE; 4970Sstevel@tonic-gate orig->size = data_size; 4980Sstevel@tonic-gate tail = oversize_header_alloc(taddr, frag_size); 4990Sstevel@tonic-gate free_oversize(tail); 5000Sstevel@tonic-gate } 5010Sstevel@tonic-gate ret_buf = alloc_buf; 5020Sstevel@tonic-gate } else { 5030Sstevel@tonic-gate uchar_t oversize_bits = 0; 5040Sstevel@tonic-gate size_t head_sz, data_sz, tail_sz; 5050Sstevel@tonic-gate uintptr_t ret_addr, taddr, shift, tshift; 506*7166Sraf oversize_t *orig, *tail, *big; 5070Sstevel@tonic-gate size_t tsize; 5080Sstevel@tonic-gate 5090Sstevel@tonic-gate /* needs to be aligned */ 5100Sstevel@tonic-gate shift = alignment - offset; 5110Sstevel@tonic-gate 5120Sstevel@tonic-gate assert(shift >= MTMALLOC_MIN_ALIGN); 5130Sstevel@tonic-gate 5140Sstevel@tonic-gate ret_addr = ((uintptr_t)alloc_buf + shift); 5150Sstevel@tonic-gate ret_buf = (void *)ret_addr; 5160Sstevel@tonic-gate 5170Sstevel@tonic-gate if (alloc_size <= MAX_CACHED) { 5180Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_addr, shift, alloc_buf); 5190Sstevel@tonic-gate return (ret_buf); 5200Sstevel@tonic-gate } 5210Sstevel@tonic-gate 5220Sstevel@tonic-gate /* 5230Sstevel@tonic-gate * Only check for the fragments when the memory is allocted 5240Sstevel@tonic-gate * from oversize_list. Split off a fragment and return it 5250Sstevel@tonic-gate * to the oversize freelist when it's > MAX_CACHED. 5260Sstevel@tonic-gate */ 5270Sstevel@tonic-gate 5280Sstevel@tonic-gate head_sz = shift - MAX(MEMALIGN_HEADER_SIZE, OVSZ_HEADER_SIZE); 5290Sstevel@tonic-gate 5300Sstevel@tonic-gate tail_sz = alloc_size - 5310Sstevel@tonic-gate (shift + size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE); 5320Sstevel@tonic-gate 5330Sstevel@tonic-gate oversize_bits |= IS_OVERSIZE(head_sz, alloc_size) | 5340Sstevel@tonic-gate IS_OVERSIZE(size, alloc_size) << DATA_SHIFT | 5350Sstevel@tonic-gate IS_OVERSIZE(tail_sz, alloc_size) << TAIL_SHIFT; 5360Sstevel@tonic-gate 5370Sstevel@tonic-gate switch (oversize_bits) { 5380Sstevel@tonic-gate case NONE_OVERSIZE: 5390Sstevel@tonic-gate case DATA_OVERSIZE: 5400Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_addr, shift, 5410Sstevel@tonic-gate alloc_buf); 5420Sstevel@tonic-gate break; 5430Sstevel@tonic-gate case HEAD_OVERSIZE: 5440Sstevel@tonic-gate /* 5450Sstevel@tonic-gate * If we can extend data > MAX_CACHED and have 5460Sstevel@tonic-gate * head still > MAX_CACHED, we split head-end 5470Sstevel@tonic-gate * as the case of head-end and data oversized, 5480Sstevel@tonic-gate * otherwise just create memalign header. 5490Sstevel@tonic-gate */ 5500Sstevel@tonic-gate tsize = (shift + size) - (MAX_CACHED + 8 + 5510Sstevel@tonic-gate MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE); 5520Sstevel@tonic-gate 5530Sstevel@tonic-gate if (!IS_OVERSIZE(tsize, alloc_size)) { 5540Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_addr, shift, 5550Sstevel@tonic-gate alloc_buf); 5560Sstevel@tonic-gate break; 5570Sstevel@tonic-gate } else { 5580Sstevel@tonic-gate tsize += OVSZ_HEADER_SIZE; 5590Sstevel@tonic-gate taddr = ALIGN((uintptr_t)alloc_buf + 5600Sstevel@tonic-gate tsize, MTMALLOC_MIN_ALIGN); 5610Sstevel@tonic-gate tshift = ret_addr - taddr; 5620Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_addr, tshift, 5630Sstevel@tonic-gate taddr); 5640Sstevel@tonic-gate ret_addr = taddr; 5650Sstevel@tonic-gate shift = ret_addr - (uintptr_t)alloc_buf; 5660Sstevel@tonic-gate } 5670Sstevel@tonic-gate /* FALLTHROUGH */ 5680Sstevel@tonic-gate case HEAD_AND_DATA_OVERSIZE: 5690Sstevel@tonic-gate /* 5700Sstevel@tonic-gate * Split off the head fragment and 5710Sstevel@tonic-gate * return it back to oversize freelist. 5720Sstevel@tonic-gate * Create oversize header for the piece 5730Sstevel@tonic-gate * of (data + tail fragment). 5740Sstevel@tonic-gate */ 5750Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 5760Sstevel@tonic-gate OVSZ_HEADER_SIZE); 577*7166Sraf big = oversize_header_alloc(ret_addr - 5780Sstevel@tonic-gate OVSZ_HEADER_SIZE, 5790Sstevel@tonic-gate (orig->size - shift)); 580*7166Sraf (void) mutex_lock(&oversize_lock); 581*7166Sraf insert_hash(big); 582*7166Sraf (void) mutex_unlock(&oversize_lock); 5830Sstevel@tonic-gate orig->size = shift - OVSZ_HEADER_SIZE; 5840Sstevel@tonic-gate 5850Sstevel@tonic-gate /* free up the head fragment */ 5860Sstevel@tonic-gate free_oversize(orig); 5870Sstevel@tonic-gate break; 5880Sstevel@tonic-gate case TAIL_OVERSIZE: 5890Sstevel@tonic-gate /* 5900Sstevel@tonic-gate * If we can extend data > MAX_CACHED and have 5910Sstevel@tonic-gate * tail-end still > MAX_CACHED, we split tail 5920Sstevel@tonic-gate * end, otherwise just create memalign header. 5930Sstevel@tonic-gate */ 5940Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 5950Sstevel@tonic-gate OVSZ_HEADER_SIZE); 5960Sstevel@tonic-gate tsize = orig->size - (MAX_CACHED + 8 + 5970Sstevel@tonic-gate shift + OVSZ_HEADER_SIZE + 5980Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 5990Sstevel@tonic-gate if (!IS_OVERSIZE(tsize, alloc_size)) { 6000Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_addr, shift, 6010Sstevel@tonic-gate alloc_buf); 6020Sstevel@tonic-gate break; 6030Sstevel@tonic-gate } else { 6040Sstevel@tonic-gate size = MAX_CACHED + 8; 6050Sstevel@tonic-gate } 6060Sstevel@tonic-gate /* FALLTHROUGH */ 6070Sstevel@tonic-gate case DATA_AND_TAIL_OVERSIZE: 6080Sstevel@tonic-gate /* 6090Sstevel@tonic-gate * Split off the tail fragment and 6100Sstevel@tonic-gate * return it back to oversize freelist. 6110Sstevel@tonic-gate * Create memalign header and adjust 6120Sstevel@tonic-gate * the size for the piece of 6130Sstevel@tonic-gate * (head fragment + data). 6140Sstevel@tonic-gate */ 6150Sstevel@tonic-gate taddr = ALIGN(ret_addr + size, 6160Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 6170Sstevel@tonic-gate data_sz = (size_t)(taddr - 6180Sstevel@tonic-gate (uintptr_t)alloc_buf); 6190Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 6200Sstevel@tonic-gate OVSZ_HEADER_SIZE); 6210Sstevel@tonic-gate tsize = orig->size - data_sz; 6220Sstevel@tonic-gate orig->size = data_sz; 6230Sstevel@tonic-gate MEMALIGN_HEADER_ALLOC(ret_buf, shift, 6240Sstevel@tonic-gate alloc_buf); 6250Sstevel@tonic-gate tsize -= OVSZ_HEADER_SIZE; 6260Sstevel@tonic-gate tail = oversize_header_alloc(taddr, tsize); 6270Sstevel@tonic-gate free_oversize(tail); 6280Sstevel@tonic-gate break; 6290Sstevel@tonic-gate case HEAD_AND_TAIL_OVERSIZE: 6300Sstevel@tonic-gate /* 6310Sstevel@tonic-gate * Split off the head fragment. 6320Sstevel@tonic-gate * We try to free up tail-end when we can 6330Sstevel@tonic-gate * extend data size to (MAX_CACHED + 8) 6340Sstevel@tonic-gate * and remain tail-end oversized. 6350Sstevel@tonic-gate * The bottom line is all split pieces 6360Sstevel@tonic-gate * should be oversize in size. 6370Sstevel@tonic-gate */ 6380Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 6390Sstevel@tonic-gate OVSZ_HEADER_SIZE); 6400Sstevel@tonic-gate tsize = orig->size - (MAX_CACHED + 8 + 6410Sstevel@tonic-gate OVSZ_HEADER_SIZE + shift + 6420Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 6430Sstevel@tonic-gate 6440Sstevel@tonic-gate if (!IS_OVERSIZE(tsize, alloc_size)) { 6450Sstevel@tonic-gate /* 6460Sstevel@tonic-gate * If the chunk is not big enough 6470Sstevel@tonic-gate * to make both data and tail oversize 6480Sstevel@tonic-gate * we just keep them as one piece. 6490Sstevel@tonic-gate */ 650*7166Sraf big = oversize_header_alloc(ret_addr - 6510Sstevel@tonic-gate OVSZ_HEADER_SIZE, 6520Sstevel@tonic-gate orig->size - shift); 653*7166Sraf (void) mutex_lock(&oversize_lock); 654*7166Sraf insert_hash(big); 655*7166Sraf (void) mutex_unlock(&oversize_lock); 6560Sstevel@tonic-gate orig->size = shift - 6570Sstevel@tonic-gate OVSZ_HEADER_SIZE; 6580Sstevel@tonic-gate free_oversize(orig); 6590Sstevel@tonic-gate break; 6600Sstevel@tonic-gate } else { 6610Sstevel@tonic-gate /* 6620Sstevel@tonic-gate * extend data size > MAX_CACHED 6630Sstevel@tonic-gate * and handle it as head, data, tail 6640Sstevel@tonic-gate * are all oversized. 6650Sstevel@tonic-gate */ 6660Sstevel@tonic-gate size = MAX_CACHED + 8; 6670Sstevel@tonic-gate } 6680Sstevel@tonic-gate /* FALLTHROUGH */ 6690Sstevel@tonic-gate case ALL_OVERSIZE: 6700Sstevel@tonic-gate /* 6710Sstevel@tonic-gate * split off the head and tail fragments, 6720Sstevel@tonic-gate * return them back to the oversize freelist. 6730Sstevel@tonic-gate * Alloc oversize header for data seg. 6740Sstevel@tonic-gate */ 6750Sstevel@tonic-gate orig = (oversize_t *)((uintptr_t)alloc_buf - 6760Sstevel@tonic-gate OVSZ_HEADER_SIZE); 6770Sstevel@tonic-gate tsize = orig->size; 6780Sstevel@tonic-gate orig->size = shift - OVSZ_HEADER_SIZE; 6790Sstevel@tonic-gate free_oversize(orig); 6800Sstevel@tonic-gate 6810Sstevel@tonic-gate taddr = ALIGN(ret_addr + size, 6820Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 6830Sstevel@tonic-gate data_sz = taddr - ret_addr; 6840Sstevel@tonic-gate assert(tsize > (shift + data_sz + 6850Sstevel@tonic-gate OVSZ_HEADER_SIZE)); 6860Sstevel@tonic-gate tail_sz = tsize - 6870Sstevel@tonic-gate (shift + data_sz + OVSZ_HEADER_SIZE); 6880Sstevel@tonic-gate 6890Sstevel@tonic-gate /* create oversize header for data seg */ 690*7166Sraf big = oversize_header_alloc(ret_addr - 6910Sstevel@tonic-gate OVSZ_HEADER_SIZE, data_sz); 692*7166Sraf (void) mutex_lock(&oversize_lock); 693*7166Sraf insert_hash(big); 694*7166Sraf (void) mutex_unlock(&oversize_lock); 6950Sstevel@tonic-gate 6960Sstevel@tonic-gate /* create oversize header for tail fragment */ 6970Sstevel@tonic-gate tail = oversize_header_alloc(taddr, tail_sz); 6980Sstevel@tonic-gate free_oversize(tail); 6990Sstevel@tonic-gate break; 7000Sstevel@tonic-gate default: 7010Sstevel@tonic-gate /* should not reach here */ 7020Sstevel@tonic-gate assert(0); 7030Sstevel@tonic-gate } 7040Sstevel@tonic-gate } 7050Sstevel@tonic-gate return (ret_buf); 7060Sstevel@tonic-gate } 7070Sstevel@tonic-gate 7080Sstevel@tonic-gate 7090Sstevel@tonic-gate void * 7100Sstevel@tonic-gate valloc(size_t size) 7110Sstevel@tonic-gate { 7120Sstevel@tonic-gate static unsigned pagesize; 7130Sstevel@tonic-gate 7140Sstevel@tonic-gate if (size == 0) 7150Sstevel@tonic-gate return (NULL); 7160Sstevel@tonic-gate 7170Sstevel@tonic-gate if (!pagesize) 7180Sstevel@tonic-gate pagesize = sysconf(_SC_PAGESIZE); 7190Sstevel@tonic-gate 7200Sstevel@tonic-gate return (memalign(pagesize, size)); 7210Sstevel@tonic-gate } 7220Sstevel@tonic-gate 7230Sstevel@tonic-gate void 7240Sstevel@tonic-gate mallocctl(int cmd, long value) 7250Sstevel@tonic-gate { 7260Sstevel@tonic-gate switch (cmd) { 7270Sstevel@tonic-gate 7280Sstevel@tonic-gate case MTDEBUGPATTERN: 7290Sstevel@tonic-gate /* 7300Sstevel@tonic-gate * Reinitialize free blocks in case malloc() is called prior 7310Sstevel@tonic-gate * to mallocctl(). 7320Sstevel@tonic-gate */ 7330Sstevel@tonic-gate if (value && !(debugopt & cmd)) { 7340Sstevel@tonic-gate reinit++; 7350Sstevel@tonic-gate debugopt |= cmd; 7360Sstevel@tonic-gate reinit_cpu_list(); 7370Sstevel@tonic-gate } 7380Sstevel@tonic-gate /*FALLTHRU*/ 7390Sstevel@tonic-gate case MTDOUBLEFREE: 7400Sstevel@tonic-gate case MTINITBUFFER: 7410Sstevel@tonic-gate if (value) 7420Sstevel@tonic-gate debugopt |= cmd; 7430Sstevel@tonic-gate else 7440Sstevel@tonic-gate debugopt &= ~cmd; 7450Sstevel@tonic-gate break; 7460Sstevel@tonic-gate case MTCHUNKSIZE: 7470Sstevel@tonic-gate if (value >= MINSIZE && value <= MAXSIZE) 7480Sstevel@tonic-gate requestsize = value; 7490Sstevel@tonic-gate break; 7500Sstevel@tonic-gate default: 7510Sstevel@tonic-gate break; 7520Sstevel@tonic-gate } 7530Sstevel@tonic-gate } 7540Sstevel@tonic-gate 7550Sstevel@tonic-gate /* 7563866Sraf * Initialization function, called from the init section of the library. 7573866Sraf * No locking is required here because we are single-threaded during 7583866Sraf * library initialization. 7590Sstevel@tonic-gate */ 7603866Sraf static void 7610Sstevel@tonic-gate setup_caches(void) 7620Sstevel@tonic-gate { 7630Sstevel@tonic-gate uintptr_t oldbrk; 7640Sstevel@tonic-gate uintptr_t newbrk; 7650Sstevel@tonic-gate 7660Sstevel@tonic-gate size_t cache_space_needed; 7670Sstevel@tonic-gate size_t padding; 7680Sstevel@tonic-gate 7690Sstevel@tonic-gate curcpu_func new_curcpu; 7700Sstevel@tonic-gate uint_t new_cpu_mask; 7710Sstevel@tonic-gate percpu_t *new_cpu_list; 7720Sstevel@tonic-gate 7730Sstevel@tonic-gate uint_t i, j; 7740Sstevel@tonic-gate uintptr_t list_addr; 7750Sstevel@tonic-gate 7763866Sraf /* 7773866Sraf * Get a decent "current cpu identifier", to be used to reduce 7783866Sraf * contention. Eventually, this should be replaced by an interface 7793866Sraf * to get the actual CPU sequence number in libthread/liblwp. 7803866Sraf */ 7813866Sraf new_curcpu = (curcpu_func)thr_self; 7823866Sraf if ((ncpus = 2 * sysconf(_SC_NPROCESSORS_CONF)) <= 0) 7833866Sraf ncpus = 4; /* decent default value */ 7840Sstevel@tonic-gate 7850Sstevel@tonic-gate /* round ncpus up to a power of 2 */ 7860Sstevel@tonic-gate while (ncpus & (ncpus - 1)) 7870Sstevel@tonic-gate ncpus++; 7880Sstevel@tonic-gate 7890Sstevel@tonic-gate new_cpu_mask = ncpus - 1; /* create the cpu mask */ 7900Sstevel@tonic-gate 7910Sstevel@tonic-gate /* 7920Sstevel@tonic-gate * We now do some magic with the brk. What we want to get in the 7930Sstevel@tonic-gate * end is a bunch of well-aligned stuff in a big initial allocation. 7940Sstevel@tonic-gate * Along the way, we do sanity checks to make sure no one else has 7950Sstevel@tonic-gate * touched the brk (which shouldn't happen, but it's always good to 7960Sstevel@tonic-gate * check) 7970Sstevel@tonic-gate * 7980Sstevel@tonic-gate * First, make sure sbrk is sane, and store the current brk in oldbrk. 7990Sstevel@tonic-gate */ 8000Sstevel@tonic-gate oldbrk = (uintptr_t)sbrk(0); 8013866Sraf if ((void *)oldbrk == (void *)-1) 8023866Sraf abort(); /* sbrk is broken -- we're doomed. */ 8030Sstevel@tonic-gate 8040Sstevel@tonic-gate /* 8050Sstevel@tonic-gate * Now, align the brk to a multiple of CACHE_COHERENCY_UNIT, so that 8060Sstevel@tonic-gate * the percpu structures and cache lists will be properly aligned. 8070Sstevel@tonic-gate * 8080Sstevel@tonic-gate * 2. All hunks will be page-aligned, assuming HUNKSIZE >= PAGESIZE, 8090Sstevel@tonic-gate * so they can be paged out individually. 8100Sstevel@tonic-gate */ 8110Sstevel@tonic-gate newbrk = ALIGN(oldbrk, CACHE_COHERENCY_UNIT); 8123866Sraf if (newbrk != oldbrk && (uintptr_t)sbrk(newbrk - oldbrk) != oldbrk) 8133866Sraf abort(); /* sbrk is broken -- we're doomed. */ 8140Sstevel@tonic-gate 8150Sstevel@tonic-gate /* 8160Sstevel@tonic-gate * For each cpu, there is one percpu_t and a list of caches 8170Sstevel@tonic-gate */ 8180Sstevel@tonic-gate cache_space_needed = ncpus * (sizeof (percpu_t) + CACHELIST_SIZE); 8190Sstevel@tonic-gate 8200Sstevel@tonic-gate new_cpu_list = (percpu_t *)sbrk(cache_space_needed); 8210Sstevel@tonic-gate 8220Sstevel@tonic-gate if (new_cpu_list == (percpu_t *)-1 || 8233866Sraf (uintptr_t)new_cpu_list != newbrk) 8243866Sraf abort(); /* sbrk is broken -- we're doomed. */ 8250Sstevel@tonic-gate 8260Sstevel@tonic-gate /* 8270Sstevel@tonic-gate * Finally, align the brk to HUNKSIZE so that all hunks are 8280Sstevel@tonic-gate * page-aligned, to avoid edge-effects. 8290Sstevel@tonic-gate */ 8300Sstevel@tonic-gate 8310Sstevel@tonic-gate newbrk = (uintptr_t)new_cpu_list + cache_space_needed; 8320Sstevel@tonic-gate 8330Sstevel@tonic-gate padding = ALIGN(newbrk, HUNKSIZE) - newbrk; 8340Sstevel@tonic-gate 8353866Sraf if (padding > 0 && (uintptr_t)sbrk(padding) != newbrk) 8363866Sraf abort(); /* sbrk is broken -- we're doomed. */ 8370Sstevel@tonic-gate 8380Sstevel@tonic-gate list_addr = ((uintptr_t)new_cpu_list + (sizeof (percpu_t) * ncpus)); 8390Sstevel@tonic-gate 8400Sstevel@tonic-gate /* initialize the percpu list */ 8410Sstevel@tonic-gate for (i = 0; i < ncpus; i++) { 8420Sstevel@tonic-gate new_cpu_list[i].mt_caches = (cache_head_t *)list_addr; 8430Sstevel@tonic-gate for (j = 0; j < NUM_CACHES; j++) { 8440Sstevel@tonic-gate new_cpu_list[i].mt_caches[j].mt_cache = NULL; 8450Sstevel@tonic-gate new_cpu_list[i].mt_caches[j].mt_hint = NULL; 8460Sstevel@tonic-gate } 8470Sstevel@tonic-gate 8483866Sraf (void) mutex_init(&new_cpu_list[i].mt_parent_lock, 8493866Sraf USYNC_THREAD, NULL); 8500Sstevel@tonic-gate 8510Sstevel@tonic-gate /* get the correct cache list alignment */ 8520Sstevel@tonic-gate list_addr += CACHELIST_SIZE; 8530Sstevel@tonic-gate } 8540Sstevel@tonic-gate 8550Sstevel@tonic-gate /* 8560Sstevel@tonic-gate * Initialize oversize listhead 8570Sstevel@tonic-gate */ 8580Sstevel@tonic-gate oversize_list.next_bysize = &oversize_list; 8590Sstevel@tonic-gate oversize_list.prev_bysize = &oversize_list; 8600Sstevel@tonic-gate oversize_list.next_byaddr = &oversize_list; 8610Sstevel@tonic-gate oversize_list.prev_byaddr = &oversize_list; 8620Sstevel@tonic-gate oversize_list.addr = NULL; 8630Sstevel@tonic-gate oversize_list.size = 0; /* sentinal */ 8640Sstevel@tonic-gate 8650Sstevel@tonic-gate /* 8663866Sraf * Now install the global variables. 8670Sstevel@tonic-gate */ 8680Sstevel@tonic-gate curcpu = new_curcpu; 8690Sstevel@tonic-gate cpu_mask = new_cpu_mask; 8700Sstevel@tonic-gate cpu_list = new_cpu_list; 8710Sstevel@tonic-gate } 8720Sstevel@tonic-gate 8730Sstevel@tonic-gate static void 8740Sstevel@tonic-gate create_cache(cache_t *cp, size_t size, uint_t chunksize) 8750Sstevel@tonic-gate { 8760Sstevel@tonic-gate long nblocks; 8770Sstevel@tonic-gate 8783866Sraf (void) mutex_init(&cp->mt_cache_lock, USYNC_THREAD, NULL); 8790Sstevel@tonic-gate cp->mt_size = size; 8800Sstevel@tonic-gate cp->mt_freelist = ((caddr_t)cp + sizeof (cache_t)); 8810Sstevel@tonic-gate cp->mt_span = chunksize * HUNKSIZE - sizeof (cache_t); 8820Sstevel@tonic-gate cp->mt_hunks = chunksize; 8830Sstevel@tonic-gate /* 8840Sstevel@tonic-gate * rough calculation. We will need to adjust later. 8850Sstevel@tonic-gate */ 8860Sstevel@tonic-gate nblocks = cp->mt_span / cp->mt_size; 8870Sstevel@tonic-gate nblocks >>= 3; 8880Sstevel@tonic-gate if (nblocks == 0) { /* less than 8 free blocks in this pool */ 8890Sstevel@tonic-gate int32_t numblocks = 0; 8900Sstevel@tonic-gate long i = cp->mt_span; 8910Sstevel@tonic-gate size_t sub = cp->mt_size; 8920Sstevel@tonic-gate uchar_t mask = 0; 8930Sstevel@tonic-gate 8940Sstevel@tonic-gate while (i > sub) { 8950Sstevel@tonic-gate numblocks++; 8960Sstevel@tonic-gate i -= sub; 8970Sstevel@tonic-gate } 8980Sstevel@tonic-gate nblocks = numblocks; 8990Sstevel@tonic-gate cp->mt_arena = (caddr_t)ALIGN(cp->mt_freelist + 8, 8); 9000Sstevel@tonic-gate cp->mt_nfree = numblocks; 9010Sstevel@tonic-gate while (numblocks--) { 9020Sstevel@tonic-gate mask |= 0x80 >> numblocks; 9030Sstevel@tonic-gate } 9040Sstevel@tonic-gate *(cp->mt_freelist) = mask; 9050Sstevel@tonic-gate } else { 9060Sstevel@tonic-gate cp->mt_arena = (caddr_t)ALIGN((caddr_t)cp->mt_freelist + 9070Sstevel@tonic-gate nblocks, 32); 9080Sstevel@tonic-gate /* recompute nblocks */ 9090Sstevel@tonic-gate nblocks = (uintptr_t)((caddr_t)cp->mt_freelist + 9100Sstevel@tonic-gate cp->mt_span - cp->mt_arena) / cp->mt_size; 9110Sstevel@tonic-gate cp->mt_nfree = ((nblocks >> 3) << 3); 9120Sstevel@tonic-gate /* Set everything to free */ 9130Sstevel@tonic-gate (void) memset(cp->mt_freelist, 0xff, nblocks >> 3); 9140Sstevel@tonic-gate } 9150Sstevel@tonic-gate 9160Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 9170Sstevel@tonic-gate copy_pattern(FREEPATTERN, cp->mt_arena, cp->mt_size * nblocks); 9180Sstevel@tonic-gate 9190Sstevel@tonic-gate cp->mt_next = NULL; 9200Sstevel@tonic-gate } 9210Sstevel@tonic-gate 9220Sstevel@tonic-gate static void 9230Sstevel@tonic-gate reinit_cpu_list(void) 9240Sstevel@tonic-gate { 9250Sstevel@tonic-gate oversize_t *wp = oversize_list.next_bysize; 9260Sstevel@tonic-gate percpu_t *cpuptr; 9270Sstevel@tonic-gate cache_t *thiscache; 9280Sstevel@tonic-gate cache_head_t *cachehead; 9290Sstevel@tonic-gate 9300Sstevel@tonic-gate /* Reinitialize free oversize blocks. */ 9310Sstevel@tonic-gate (void) mutex_lock(&oversize_lock); 9320Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 9330Sstevel@tonic-gate for (; wp != &oversize_list; wp = wp->next_bysize) 9340Sstevel@tonic-gate copy_pattern(FREEPATTERN, wp->addr, wp->size); 9350Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 9360Sstevel@tonic-gate 9370Sstevel@tonic-gate /* Reinitialize free blocks. */ 9380Sstevel@tonic-gate for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) { 9390Sstevel@tonic-gate (void) mutex_lock(&cpuptr->mt_parent_lock); 9400Sstevel@tonic-gate for (cachehead = &cpuptr->mt_caches[0]; cachehead < 9410Sstevel@tonic-gate &cpuptr->mt_caches[NUM_CACHES]; cachehead++) { 9420Sstevel@tonic-gate for (thiscache = cachehead->mt_cache; thiscache != NULL; 9430Sstevel@tonic-gate thiscache = thiscache->mt_next) { 9440Sstevel@tonic-gate (void) mutex_lock(&thiscache->mt_cache_lock); 9450Sstevel@tonic-gate if (thiscache->mt_nfree == 0) { 9460Sstevel@tonic-gate (void) mutex_unlock( 9470Sstevel@tonic-gate &thiscache->mt_cache_lock); 9480Sstevel@tonic-gate continue; 9490Sstevel@tonic-gate } 9500Sstevel@tonic-gate if (thiscache != NULL) 9510Sstevel@tonic-gate reinit_cache(thiscache); 9520Sstevel@tonic-gate (void) mutex_unlock(&thiscache->mt_cache_lock); 9530Sstevel@tonic-gate } 9540Sstevel@tonic-gate } 9550Sstevel@tonic-gate (void) mutex_unlock(&cpuptr->mt_parent_lock); 9560Sstevel@tonic-gate } 9570Sstevel@tonic-gate reinit = 0; 9580Sstevel@tonic-gate } 9590Sstevel@tonic-gate 9600Sstevel@tonic-gate static void 9610Sstevel@tonic-gate reinit_cache(cache_t *thiscache) 9620Sstevel@tonic-gate { 9630Sstevel@tonic-gate uint32_t *freeblocks; /* not a uintptr_t on purpose */ 9640Sstevel@tonic-gate int32_t i, n; 9650Sstevel@tonic-gate caddr_t ret; 9660Sstevel@tonic-gate 9670Sstevel@tonic-gate freeblocks = (uint32_t *)thiscache->mt_freelist; 9680Sstevel@tonic-gate while (freeblocks < (uint32_t *)thiscache->mt_arena) { 9690Sstevel@tonic-gate if (*freeblocks & 0xffffffff) { 9700Sstevel@tonic-gate for (i = 0; i < 32; i++) { 9710Sstevel@tonic-gate if (FLIP_EM(*freeblocks) & (0x80000000 >> i)) { 9720Sstevel@tonic-gate n = (uintptr_t)(((freeblocks - 9730Sstevel@tonic-gate (uint32_t *)thiscache->mt_freelist) << 5) 9740Sstevel@tonic-gate + i) * thiscache->mt_size; 9750Sstevel@tonic-gate ret = thiscache->mt_arena + n; 9760Sstevel@tonic-gate ret += OVERHEAD; 9770Sstevel@tonic-gate copy_pattern(FREEPATTERN, ret, 9780Sstevel@tonic-gate thiscache->mt_size); 9790Sstevel@tonic-gate } 9800Sstevel@tonic-gate } 9810Sstevel@tonic-gate } 9820Sstevel@tonic-gate freeblocks++; 9830Sstevel@tonic-gate } 9840Sstevel@tonic-gate } 9850Sstevel@tonic-gate 9860Sstevel@tonic-gate static void * 9870Sstevel@tonic-gate malloc_internal(size_t size, percpu_t *cpuptr) 9880Sstevel@tonic-gate { 9890Sstevel@tonic-gate cache_head_t *cachehead; 9900Sstevel@tonic-gate cache_t *thiscache, *hintcache; 9910Sstevel@tonic-gate int32_t i, n, logsz, bucket; 9920Sstevel@tonic-gate uint32_t index; 9930Sstevel@tonic-gate uint32_t *freeblocks; /* not a uintptr_t on purpose */ 9940Sstevel@tonic-gate caddr_t ret; 9950Sstevel@tonic-gate 9960Sstevel@tonic-gate logsz = MIN_CACHED_SHIFT; 9970Sstevel@tonic-gate 9980Sstevel@tonic-gate while (size > (1 << logsz)) 9990Sstevel@tonic-gate logsz++; 10000Sstevel@tonic-gate 10010Sstevel@tonic-gate bucket = logsz - MIN_CACHED_SHIFT; 10020Sstevel@tonic-gate 10030Sstevel@tonic-gate (void) mutex_lock(&cpuptr->mt_parent_lock); 10040Sstevel@tonic-gate 10050Sstevel@tonic-gate /* 10060Sstevel@tonic-gate * Find a cache of the appropriate size with free buffers. 10070Sstevel@tonic-gate * 10080Sstevel@tonic-gate * We don't need to lock each cache as we check their mt_nfree count, 10090Sstevel@tonic-gate * since: 10100Sstevel@tonic-gate * 1. We are only looking for caches with mt_nfree > 0. If a 10110Sstevel@tonic-gate * free happens during our search, it will increment mt_nfree, 10120Sstevel@tonic-gate * which will not effect the test. 10130Sstevel@tonic-gate * 2. Allocations can decrement mt_nfree, but they can't happen 10140Sstevel@tonic-gate * as long as we hold mt_parent_lock. 10150Sstevel@tonic-gate */ 10160Sstevel@tonic-gate 10170Sstevel@tonic-gate cachehead = &cpuptr->mt_caches[bucket]; 10180Sstevel@tonic-gate 10190Sstevel@tonic-gate /* Search through the list, starting at the mt_hint */ 10200Sstevel@tonic-gate thiscache = cachehead->mt_hint; 10210Sstevel@tonic-gate 10220Sstevel@tonic-gate while (thiscache != NULL && thiscache->mt_nfree == 0) 10230Sstevel@tonic-gate thiscache = thiscache->mt_next; 10240Sstevel@tonic-gate 10250Sstevel@tonic-gate if (thiscache == NULL) { 10260Sstevel@tonic-gate /* wrap around -- search up to the hint */ 10270Sstevel@tonic-gate thiscache = cachehead->mt_cache; 10280Sstevel@tonic-gate hintcache = cachehead->mt_hint; 10290Sstevel@tonic-gate 10300Sstevel@tonic-gate while (thiscache != NULL && thiscache != hintcache && 10310Sstevel@tonic-gate thiscache->mt_nfree == 0) 10320Sstevel@tonic-gate thiscache = thiscache->mt_next; 10330Sstevel@tonic-gate 10340Sstevel@tonic-gate if (thiscache == hintcache) 10350Sstevel@tonic-gate thiscache = NULL; 10360Sstevel@tonic-gate } 10370Sstevel@tonic-gate 10380Sstevel@tonic-gate 10390Sstevel@tonic-gate if (thiscache == NULL) { /* there are no free caches */ 10400Sstevel@tonic-gate int32_t thisrequest = requestsize; 10410Sstevel@tonic-gate int32_t buffer_size = (1 << logsz) + OVERHEAD; 10420Sstevel@tonic-gate 10430Sstevel@tonic-gate thiscache = (cache_t *)morecore(thisrequest * HUNKSIZE); 10440Sstevel@tonic-gate 10450Sstevel@tonic-gate if (thiscache == (cache_t *)-1) { 10460Sstevel@tonic-gate (void) mutex_unlock(&cpuptr->mt_parent_lock); 10470Sstevel@tonic-gate errno = EAGAIN; 10480Sstevel@tonic-gate return (NULL); 10490Sstevel@tonic-gate } 10500Sstevel@tonic-gate create_cache(thiscache, buffer_size, thisrequest); 10510Sstevel@tonic-gate 10520Sstevel@tonic-gate /* link in the new block at the beginning of the list */ 10530Sstevel@tonic-gate thiscache->mt_next = cachehead->mt_cache; 10540Sstevel@tonic-gate cachehead->mt_cache = thiscache; 10550Sstevel@tonic-gate } 10560Sstevel@tonic-gate 10570Sstevel@tonic-gate /* update the hint to the cache we found or created */ 10580Sstevel@tonic-gate cachehead->mt_hint = thiscache; 10590Sstevel@tonic-gate 10600Sstevel@tonic-gate /* thiscache now points to a cache with available space */ 10610Sstevel@tonic-gate (void) mutex_lock(&thiscache->mt_cache_lock); 10620Sstevel@tonic-gate 10630Sstevel@tonic-gate freeblocks = (uint32_t *)thiscache->mt_freelist; 10640Sstevel@tonic-gate while (freeblocks < (uint32_t *)thiscache->mt_arena) { 10650Sstevel@tonic-gate if (*freeblocks & 0xffffffff) 10660Sstevel@tonic-gate break; 10670Sstevel@tonic-gate freeblocks++; 10680Sstevel@tonic-gate if (freeblocks < (uint32_t *)thiscache->mt_arena && 10690Sstevel@tonic-gate *freeblocks & 0xffffffff) 10700Sstevel@tonic-gate break; 10710Sstevel@tonic-gate freeblocks++; 10720Sstevel@tonic-gate if (freeblocks < (uint32_t *)thiscache->mt_arena && 10730Sstevel@tonic-gate *freeblocks & 0xffffffff) 10740Sstevel@tonic-gate break; 10750Sstevel@tonic-gate freeblocks++; 10760Sstevel@tonic-gate if (freeblocks < (uint32_t *)thiscache->mt_arena && 10770Sstevel@tonic-gate *freeblocks & 0xffffffff) 10780Sstevel@tonic-gate break; 10790Sstevel@tonic-gate freeblocks++; 10800Sstevel@tonic-gate } 10810Sstevel@tonic-gate 10820Sstevel@tonic-gate /* 10830Sstevel@tonic-gate * the offset from mt_freelist to freeblocks is the offset into 10840Sstevel@tonic-gate * the arena. Be sure to include the offset into freeblocks 10850Sstevel@tonic-gate * of the bitmask. n is the offset. 10860Sstevel@tonic-gate */ 10870Sstevel@tonic-gate for (i = 0; i < 32; ) { 10880Sstevel@tonic-gate if (FLIP_EM(*freeblocks) & (0x80000000 >> i++)) 10890Sstevel@tonic-gate break; 10900Sstevel@tonic-gate if (FLIP_EM(*freeblocks) & (0x80000000 >> i++)) 10910Sstevel@tonic-gate break; 10920Sstevel@tonic-gate if (FLIP_EM(*freeblocks) & (0x80000000 >> i++)) 10930Sstevel@tonic-gate break; 10940Sstevel@tonic-gate if (FLIP_EM(*freeblocks) & (0x80000000 >> i++)) 10950Sstevel@tonic-gate break; 10960Sstevel@tonic-gate } 10970Sstevel@tonic-gate index = 0x80000000 >> --i; 10980Sstevel@tonic-gate 10990Sstevel@tonic-gate 11000Sstevel@tonic-gate *freeblocks &= FLIP_EM(~index); 11010Sstevel@tonic-gate 11020Sstevel@tonic-gate thiscache->mt_nfree--; 11030Sstevel@tonic-gate 11040Sstevel@tonic-gate (void) mutex_unlock(&thiscache->mt_cache_lock); 11050Sstevel@tonic-gate (void) mutex_unlock(&cpuptr->mt_parent_lock); 11060Sstevel@tonic-gate 11070Sstevel@tonic-gate n = (uintptr_t)(((freeblocks - (uint32_t *)thiscache->mt_freelist) << 5) 11080Sstevel@tonic-gate + i) * thiscache->mt_size; 11090Sstevel@tonic-gate /* 11100Sstevel@tonic-gate * Now you have the offset in n, you've changed the free mask 11110Sstevel@tonic-gate * in the freelist. Nothing left to do but find the block 11120Sstevel@tonic-gate * in the arena and put the value of thiscache in the word 11130Sstevel@tonic-gate * ahead of the handed out address and return the memory 11140Sstevel@tonic-gate * back to the user. 11150Sstevel@tonic-gate */ 11160Sstevel@tonic-gate ret = thiscache->mt_arena + n; 11170Sstevel@tonic-gate 11180Sstevel@tonic-gate /* Store the cache addr for this buf. Makes free go fast. */ 11190Sstevel@tonic-gate *(uintptr_t *)ret = (uintptr_t)thiscache; 11200Sstevel@tonic-gate 11210Sstevel@tonic-gate /* 11220Sstevel@tonic-gate * This assert makes sure we don't hand out memory that is not 11230Sstevel@tonic-gate * owned by this cache. 11240Sstevel@tonic-gate */ 11250Sstevel@tonic-gate assert(ret + thiscache->mt_size <= thiscache->mt_freelist + 11260Sstevel@tonic-gate thiscache->mt_span); 11270Sstevel@tonic-gate 11280Sstevel@tonic-gate ret += OVERHEAD; 11290Sstevel@tonic-gate 11300Sstevel@tonic-gate assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */ 11310Sstevel@tonic-gate 11320Sstevel@tonic-gate if (reinit == 0 && (debugopt & MTDEBUGPATTERN)) 11330Sstevel@tonic-gate if (verify_pattern(FREEPATTERN, ret, size)) 11340Sstevel@tonic-gate abort(); /* reference after free */ 11350Sstevel@tonic-gate 11360Sstevel@tonic-gate if (debugopt & MTINITBUFFER) 11370Sstevel@tonic-gate copy_pattern(INITPATTERN, ret, size); 11380Sstevel@tonic-gate return ((void *)ret); 11390Sstevel@tonic-gate } 11400Sstevel@tonic-gate 11410Sstevel@tonic-gate static void * 11420Sstevel@tonic-gate morecore(size_t bytes) 11430Sstevel@tonic-gate { 11440Sstevel@tonic-gate void * ret; 11450Sstevel@tonic-gate 11460Sstevel@tonic-gate if (bytes > LONG_MAX) { 11470Sstevel@tonic-gate intptr_t wad; 11480Sstevel@tonic-gate /* 11490Sstevel@tonic-gate * The request size is too big. We need to do this in 11500Sstevel@tonic-gate * chunks. Sbrk only takes an int for an arg. 11510Sstevel@tonic-gate */ 11520Sstevel@tonic-gate if (bytes == ULONG_MAX) 11530Sstevel@tonic-gate return ((void *)-1); 11540Sstevel@tonic-gate 11550Sstevel@tonic-gate ret = sbrk(0); 11560Sstevel@tonic-gate wad = LONG_MAX; 11570Sstevel@tonic-gate while (wad > 0) { 11580Sstevel@tonic-gate if (sbrk(wad) == (void *)-1) { 11590Sstevel@tonic-gate if (ret != sbrk(0)) 11600Sstevel@tonic-gate (void) sbrk(-LONG_MAX); 11610Sstevel@tonic-gate return ((void *)-1); 11620Sstevel@tonic-gate } 11630Sstevel@tonic-gate bytes -= LONG_MAX; 11640Sstevel@tonic-gate wad = bytes; 11650Sstevel@tonic-gate } 11660Sstevel@tonic-gate } else 11670Sstevel@tonic-gate ret = sbrk(bytes); 11680Sstevel@tonic-gate 11690Sstevel@tonic-gate return (ret); 11700Sstevel@tonic-gate } 11710Sstevel@tonic-gate 11720Sstevel@tonic-gate 11730Sstevel@tonic-gate static void * 11740Sstevel@tonic-gate oversize(size_t size) 11750Sstevel@tonic-gate { 11760Sstevel@tonic-gate caddr_t ret; 11770Sstevel@tonic-gate oversize_t *big; 11780Sstevel@tonic-gate 11791412Srm88369 /* make sure we will not overflow */ 11801412Srm88369 if (size > MAX_MTMALLOC) { 11811412Srm88369 errno = ENOMEM; 11821412Srm88369 return (NULL); 11831412Srm88369 } 11840Sstevel@tonic-gate 11850Sstevel@tonic-gate /* 11860Sstevel@tonic-gate * Since we ensure every address we hand back is 11870Sstevel@tonic-gate * MTMALLOC_MIN_ALIGN-byte aligned, ALIGNing size ensures that the 11880Sstevel@tonic-gate * memory handed out is MTMALLOC_MIN_ALIGN-byte aligned at both ends. 11890Sstevel@tonic-gate * This eases the implementation of MTDEBUGPATTERN and MTINITPATTERN, 11900Sstevel@tonic-gate * particularly where coalescing occurs. 11910Sstevel@tonic-gate */ 11920Sstevel@tonic-gate size = ALIGN(size, MTMALLOC_MIN_ALIGN); 11930Sstevel@tonic-gate 11941412Srm88369 /* 11951412Srm88369 * The idea with the global lock is that we are sure to 11961412Srm88369 * block in the kernel anyway since given an oversize alloc 11971412Srm88369 * we are sure to have to call morecore(); 11981412Srm88369 */ 11991412Srm88369 (void) mutex_lock(&oversize_lock); 12001412Srm88369 12010Sstevel@tonic-gate if ((big = find_oversize(size)) != NULL) { 12020Sstevel@tonic-gate if (reinit == 0 && (debugopt & MTDEBUGPATTERN)) 12030Sstevel@tonic-gate if (verify_pattern(FREEPATTERN, big->addr, size)) 12040Sstevel@tonic-gate abort(); /* reference after free */ 12050Sstevel@tonic-gate } else { 12060Sstevel@tonic-gate /* Get more 8-byte aligned memory from heap */ 12070Sstevel@tonic-gate ret = morecore(size + OVSZ_HEADER_SIZE); 12080Sstevel@tonic-gate if (ret == (caddr_t)-1) { 12090Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 12100Sstevel@tonic-gate errno = ENOMEM; 12110Sstevel@tonic-gate return (NULL); 12120Sstevel@tonic-gate } 12130Sstevel@tonic-gate big = oversize_header_alloc((uintptr_t)ret, size); 12140Sstevel@tonic-gate } 12150Sstevel@tonic-gate ret = big->addr; 12160Sstevel@tonic-gate 1217*7166Sraf insert_hash(big); 12180Sstevel@tonic-gate 12190Sstevel@tonic-gate if (debugopt & MTINITBUFFER) 12200Sstevel@tonic-gate copy_pattern(INITPATTERN, ret, size); 12210Sstevel@tonic-gate 12220Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 12230Sstevel@tonic-gate assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */ 12240Sstevel@tonic-gate return ((void *)ret); 12250Sstevel@tonic-gate } 12260Sstevel@tonic-gate 12270Sstevel@tonic-gate static void 12280Sstevel@tonic-gate insert_oversize(oversize_t *op, oversize_t *nx) 12290Sstevel@tonic-gate { 12300Sstevel@tonic-gate oversize_t *sp; 12310Sstevel@tonic-gate 12320Sstevel@tonic-gate /* locate correct insertion point in size-ordered list */ 12330Sstevel@tonic-gate for (sp = oversize_list.next_bysize; 12340Sstevel@tonic-gate sp != &oversize_list && (op->size > sp->size); 12350Sstevel@tonic-gate sp = sp->next_bysize) 12360Sstevel@tonic-gate ; 12370Sstevel@tonic-gate 12380Sstevel@tonic-gate /* link into size-ordered list */ 12390Sstevel@tonic-gate op->next_bysize = sp; 12400Sstevel@tonic-gate op->prev_bysize = sp->prev_bysize; 12410Sstevel@tonic-gate op->prev_bysize->next_bysize = op; 12420Sstevel@tonic-gate op->next_bysize->prev_bysize = op; 12430Sstevel@tonic-gate 12440Sstevel@tonic-gate /* 12450Sstevel@tonic-gate * link item into address-ordered list 12460Sstevel@tonic-gate * (caller provides insertion point as an optimization) 12470Sstevel@tonic-gate */ 12480Sstevel@tonic-gate op->next_byaddr = nx; 12490Sstevel@tonic-gate op->prev_byaddr = nx->prev_byaddr; 12500Sstevel@tonic-gate op->prev_byaddr->next_byaddr = op; 12510Sstevel@tonic-gate op->next_byaddr->prev_byaddr = op; 12520Sstevel@tonic-gate 12530Sstevel@tonic-gate } 12540Sstevel@tonic-gate 12550Sstevel@tonic-gate static void 12560Sstevel@tonic-gate unlink_oversize(oversize_t *lp) 12570Sstevel@tonic-gate { 12580Sstevel@tonic-gate /* unlink from address list */ 12590Sstevel@tonic-gate lp->prev_byaddr->next_byaddr = lp->next_byaddr; 12600Sstevel@tonic-gate lp->next_byaddr->prev_byaddr = lp->prev_byaddr; 12610Sstevel@tonic-gate 12620Sstevel@tonic-gate /* unlink from size list */ 12630Sstevel@tonic-gate lp->prev_bysize->next_bysize = lp->next_bysize; 12640Sstevel@tonic-gate lp->next_bysize->prev_bysize = lp->prev_bysize; 12650Sstevel@tonic-gate } 12660Sstevel@tonic-gate 12670Sstevel@tonic-gate static void 12680Sstevel@tonic-gate position_oversize_by_size(oversize_t *op) 12690Sstevel@tonic-gate { 12700Sstevel@tonic-gate oversize_t *sp; 12710Sstevel@tonic-gate 12720Sstevel@tonic-gate if (op->size > op->next_bysize->size || 12730Sstevel@tonic-gate op->size < op->prev_bysize->size) { 12740Sstevel@tonic-gate 12750Sstevel@tonic-gate /* unlink from size list */ 12760Sstevel@tonic-gate op->prev_bysize->next_bysize = op->next_bysize; 12770Sstevel@tonic-gate op->next_bysize->prev_bysize = op->prev_bysize; 12780Sstevel@tonic-gate 12790Sstevel@tonic-gate /* locate correct insertion point in size-ordered list */ 12800Sstevel@tonic-gate for (sp = oversize_list.next_bysize; 12810Sstevel@tonic-gate sp != &oversize_list && (op->size > sp->size); 12820Sstevel@tonic-gate sp = sp->next_bysize) 12830Sstevel@tonic-gate ; 12840Sstevel@tonic-gate 12850Sstevel@tonic-gate /* link into size-ordered list */ 12860Sstevel@tonic-gate op->next_bysize = sp; 12870Sstevel@tonic-gate op->prev_bysize = sp->prev_bysize; 12880Sstevel@tonic-gate op->prev_bysize->next_bysize = op; 12890Sstevel@tonic-gate op->next_bysize->prev_bysize = op; 12900Sstevel@tonic-gate } 12910Sstevel@tonic-gate } 12920Sstevel@tonic-gate 12930Sstevel@tonic-gate static void 12940Sstevel@tonic-gate add_oversize(oversize_t *lp) 12950Sstevel@tonic-gate { 12960Sstevel@tonic-gate int merge_flags = INSERT_ONLY; 12970Sstevel@tonic-gate oversize_t *nx; /* ptr to item right of insertion point */ 12980Sstevel@tonic-gate oversize_t *pv; /* ptr to item left of insertion point */ 12990Sstevel@tonic-gate uint_t size_lp, size_pv, size_nx; 13000Sstevel@tonic-gate uintptr_t endp_lp, endp_pv, endp_nx; 13010Sstevel@tonic-gate 13020Sstevel@tonic-gate /* 13030Sstevel@tonic-gate * Locate insertion point in address-ordered list 13040Sstevel@tonic-gate */ 13050Sstevel@tonic-gate 13060Sstevel@tonic-gate for (nx = oversize_list.next_byaddr; 13070Sstevel@tonic-gate nx != &oversize_list && (lp->addr > nx->addr); 13080Sstevel@tonic-gate nx = nx->next_byaddr) 13090Sstevel@tonic-gate ; 13100Sstevel@tonic-gate 13110Sstevel@tonic-gate /* 13120Sstevel@tonic-gate * Determine how to add chunk to oversize freelist 13130Sstevel@tonic-gate */ 13140Sstevel@tonic-gate 13150Sstevel@tonic-gate size_lp = OVSZ_HEADER_SIZE + lp->size; 13160Sstevel@tonic-gate endp_lp = ALIGN((uintptr_t)lp + size_lp, MTMALLOC_MIN_ALIGN); 13170Sstevel@tonic-gate size_lp = endp_lp - (uintptr_t)lp; 13180Sstevel@tonic-gate 13190Sstevel@tonic-gate pv = nx->prev_byaddr; 13200Sstevel@tonic-gate 13210Sstevel@tonic-gate if (pv->size) { 13220Sstevel@tonic-gate 13230Sstevel@tonic-gate size_pv = OVSZ_HEADER_SIZE + pv->size; 13240Sstevel@tonic-gate endp_pv = ALIGN((uintptr_t)pv + size_pv, 13250Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 13260Sstevel@tonic-gate size_pv = endp_pv - (uintptr_t)pv; 13270Sstevel@tonic-gate 13280Sstevel@tonic-gate /* Check for adjacency with left chunk */ 13290Sstevel@tonic-gate if ((uintptr_t)lp == endp_pv) 13300Sstevel@tonic-gate merge_flags |= COALESCE_LEFT; 13310Sstevel@tonic-gate } 13320Sstevel@tonic-gate 13330Sstevel@tonic-gate if (nx->size) { 13340Sstevel@tonic-gate 13350Sstevel@tonic-gate /* Check for adjacency with right chunk */ 13360Sstevel@tonic-gate if ((uintptr_t)nx == endp_lp) { 13370Sstevel@tonic-gate size_nx = OVSZ_HEADER_SIZE + nx->size; 13380Sstevel@tonic-gate endp_nx = ALIGN((uintptr_t)nx + size_nx, 13390Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 13400Sstevel@tonic-gate size_nx = endp_nx - (uintptr_t)nx; 13410Sstevel@tonic-gate merge_flags |= COALESCE_RIGHT; 13420Sstevel@tonic-gate } 13430Sstevel@tonic-gate } 13440Sstevel@tonic-gate 13450Sstevel@tonic-gate /* 13460Sstevel@tonic-gate * If MTDEBUGPATTERN==1, lp->addr will have been overwritten with 13470Sstevel@tonic-gate * FREEPATTERN for lp->size bytes. If we can merge, the oversize 13480Sstevel@tonic-gate * header(s) that will also become part of the memory available for 13490Sstevel@tonic-gate * reallocation (ie lp and/or nx) must also be overwritten with 13500Sstevel@tonic-gate * FREEPATTERN or we will SIGABRT when this memory is next reallocated. 13510Sstevel@tonic-gate */ 13520Sstevel@tonic-gate switch (merge_flags) { 13530Sstevel@tonic-gate 13540Sstevel@tonic-gate case INSERT_ONLY: /* Coalescing not possible */ 13550Sstevel@tonic-gate insert_oversize(lp, nx); 13560Sstevel@tonic-gate break; 13570Sstevel@tonic-gate case COALESCE_LEFT: 13580Sstevel@tonic-gate pv->size += size_lp; 13590Sstevel@tonic-gate position_oversize_by_size(pv); 13600Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 13610Sstevel@tonic-gate copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE); 13620Sstevel@tonic-gate break; 13630Sstevel@tonic-gate case COALESCE_RIGHT: 13640Sstevel@tonic-gate unlink_oversize(nx); 13650Sstevel@tonic-gate lp->size += size_nx; 13660Sstevel@tonic-gate insert_oversize(lp, pv->next_byaddr); 13670Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) 13680Sstevel@tonic-gate copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE); 13690Sstevel@tonic-gate break; 13700Sstevel@tonic-gate case COALESCE_WITH_BOTH_SIDES: /* Merge (with right) to the left */ 13710Sstevel@tonic-gate pv->size += size_lp + size_nx; 13720Sstevel@tonic-gate unlink_oversize(nx); 13730Sstevel@tonic-gate position_oversize_by_size(pv); 13740Sstevel@tonic-gate if (debugopt & MTDEBUGPATTERN) { 13750Sstevel@tonic-gate copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE); 13760Sstevel@tonic-gate copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE); 13770Sstevel@tonic-gate } 13780Sstevel@tonic-gate break; 13790Sstevel@tonic-gate } 13800Sstevel@tonic-gate } 13810Sstevel@tonic-gate 13820Sstevel@tonic-gate /* 13830Sstevel@tonic-gate * Find memory on our list that is at least size big. If we find a block that is 13840Sstevel@tonic-gate * big enough, we break it up and return the associated oversize_t struct back 13850Sstevel@tonic-gate * to the calling client. Any leftover piece of that block is returned to the 13860Sstevel@tonic-gate * freelist. 13870Sstevel@tonic-gate */ 13880Sstevel@tonic-gate static oversize_t * 13890Sstevel@tonic-gate find_oversize(size_t size) 13900Sstevel@tonic-gate { 13910Sstevel@tonic-gate oversize_t *wp = oversize_list.next_bysize; 13920Sstevel@tonic-gate while (wp != &oversize_list && size > wp->size) 13930Sstevel@tonic-gate wp = wp->next_bysize; 13940Sstevel@tonic-gate 13950Sstevel@tonic-gate if (wp == &oversize_list) /* empty list or nothing big enough */ 13960Sstevel@tonic-gate return (NULL); 13970Sstevel@tonic-gate /* breaking up a chunk of memory */ 13980Sstevel@tonic-gate if ((long)((wp->size - (size + OVSZ_HEADER_SIZE + MTMALLOC_MIN_ALIGN))) 13990Sstevel@tonic-gate > MAX_CACHED) { 14000Sstevel@tonic-gate caddr_t off; 14010Sstevel@tonic-gate oversize_t *np; 14020Sstevel@tonic-gate size_t osize; 14030Sstevel@tonic-gate off = (caddr_t)ALIGN(wp->addr + size, 14040Sstevel@tonic-gate MTMALLOC_MIN_ALIGN); 14050Sstevel@tonic-gate osize = wp->size; 14060Sstevel@tonic-gate wp->size = (size_t)(off - wp->addr); 14070Sstevel@tonic-gate np = oversize_header_alloc((uintptr_t)off, 14080Sstevel@tonic-gate osize - (wp->size + OVSZ_HEADER_SIZE)); 14090Sstevel@tonic-gate if ((long)np->size < 0) 14100Sstevel@tonic-gate abort(); 14110Sstevel@tonic-gate unlink_oversize(wp); 14120Sstevel@tonic-gate add_oversize(np); 14130Sstevel@tonic-gate } else { 14140Sstevel@tonic-gate unlink_oversize(wp); 14150Sstevel@tonic-gate } 14160Sstevel@tonic-gate return (wp); 14170Sstevel@tonic-gate } 14180Sstevel@tonic-gate 14190Sstevel@tonic-gate static void 14200Sstevel@tonic-gate copy_pattern(uint32_t pattern, void *buf_arg, size_t size) 14210Sstevel@tonic-gate { 14220Sstevel@tonic-gate uint32_t *bufend = (uint32_t *)((char *)buf_arg + size); 14230Sstevel@tonic-gate uint32_t *buf = buf_arg; 14240Sstevel@tonic-gate 14250Sstevel@tonic-gate while (buf < bufend - 3) { 14260Sstevel@tonic-gate buf[3] = buf[2] = buf[1] = buf[0] = pattern; 14270Sstevel@tonic-gate buf += 4; 14280Sstevel@tonic-gate } 14290Sstevel@tonic-gate while (buf < bufend) 14300Sstevel@tonic-gate *buf++ = pattern; 14310Sstevel@tonic-gate } 14320Sstevel@tonic-gate 14330Sstevel@tonic-gate static void * 14340Sstevel@tonic-gate verify_pattern(uint32_t pattern, void *buf_arg, size_t size) 14350Sstevel@tonic-gate { 14360Sstevel@tonic-gate uint32_t *bufend = (uint32_t *)((char *)buf_arg + size); 14370Sstevel@tonic-gate uint32_t *buf; 14380Sstevel@tonic-gate 14390Sstevel@tonic-gate for (buf = buf_arg; buf < bufend; buf++) 14400Sstevel@tonic-gate if (*buf != pattern) 14410Sstevel@tonic-gate return (buf); 14420Sstevel@tonic-gate return (NULL); 14430Sstevel@tonic-gate } 14440Sstevel@tonic-gate 14450Sstevel@tonic-gate static void 14460Sstevel@tonic-gate free_oversize(oversize_t *ovp) 14470Sstevel@tonic-gate { 14480Sstevel@tonic-gate assert(((uintptr_t)ovp->addr & 7) == 0); /* are we 8 byte aligned */ 14490Sstevel@tonic-gate assert(ovp->size > MAX_CACHED); 14500Sstevel@tonic-gate 14510Sstevel@tonic-gate ovp->next_bysize = ovp->prev_bysize = NULL; 14520Sstevel@tonic-gate ovp->next_byaddr = ovp->prev_byaddr = NULL; 14530Sstevel@tonic-gate (void) mutex_lock(&oversize_lock); 14540Sstevel@tonic-gate add_oversize(ovp); 14550Sstevel@tonic-gate (void) mutex_unlock(&oversize_lock); 14560Sstevel@tonic-gate } 14570Sstevel@tonic-gate 14580Sstevel@tonic-gate static oversize_t * 14590Sstevel@tonic-gate oversize_header_alloc(uintptr_t mem, size_t size) 14600Sstevel@tonic-gate { 14610Sstevel@tonic-gate oversize_t *ovsz_hdr; 14620Sstevel@tonic-gate 14630Sstevel@tonic-gate assert(size > MAX_CACHED); 14640Sstevel@tonic-gate 14650Sstevel@tonic-gate ovsz_hdr = (oversize_t *)mem; 14660Sstevel@tonic-gate ovsz_hdr->prev_bysize = NULL; 14670Sstevel@tonic-gate ovsz_hdr->next_bysize = NULL; 14680Sstevel@tonic-gate ovsz_hdr->prev_byaddr = NULL; 14690Sstevel@tonic-gate ovsz_hdr->next_byaddr = NULL; 14700Sstevel@tonic-gate ovsz_hdr->hash_next = NULL; 14710Sstevel@tonic-gate ovsz_hdr->size = size; 14720Sstevel@tonic-gate mem += OVSZ_SIZE; 14730Sstevel@tonic-gate *(uintptr_t *)mem = MTMALLOC_OVERSIZE_MAGIC; 14740Sstevel@tonic-gate mem += OVERHEAD; 14750Sstevel@tonic-gate assert(((uintptr_t)mem & 7) == 0); /* are we 8 byte aligned */ 14760Sstevel@tonic-gate ovsz_hdr->addr = (caddr_t)mem; 14770Sstevel@tonic-gate return (ovsz_hdr); 14780Sstevel@tonic-gate } 14793866Sraf 14803866Sraf static void 14813866Sraf malloc_prepare() 14823866Sraf { 14833866Sraf percpu_t *cpuptr; 14843866Sraf cache_head_t *cachehead; 14853866Sraf cache_t *thiscache; 14863866Sraf 14873866Sraf (void) mutex_lock(&oversize_lock); 14883866Sraf for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) { 14893866Sraf (void) mutex_lock(&cpuptr->mt_parent_lock); 14903866Sraf for (cachehead = &cpuptr->mt_caches[0]; 14913866Sraf cachehead < &cpuptr->mt_caches[NUM_CACHES]; 14923866Sraf cachehead++) { 14933866Sraf for (thiscache = cachehead->mt_cache; 14943866Sraf thiscache != NULL; 14953866Sraf thiscache = thiscache->mt_next) { 14963866Sraf (void) mutex_lock( 14973866Sraf &thiscache->mt_cache_lock); 14983866Sraf } 14993866Sraf } 15003866Sraf } 15013866Sraf } 15023866Sraf 15033866Sraf static void 15043866Sraf malloc_release() 15053866Sraf { 15063866Sraf percpu_t *cpuptr; 15073866Sraf cache_head_t *cachehead; 15083866Sraf cache_t *thiscache; 15093866Sraf 15103866Sraf for (cpuptr = &cpu_list[ncpus - 1]; cpuptr >= &cpu_list[0]; cpuptr--) { 15113866Sraf for (cachehead = &cpuptr->mt_caches[NUM_CACHES - 1]; 15123866Sraf cachehead >= &cpuptr->mt_caches[0]; 15133866Sraf cachehead--) { 15143866Sraf for (thiscache = cachehead->mt_cache; 15153866Sraf thiscache != NULL; 15163866Sraf thiscache = thiscache->mt_next) { 15173866Sraf (void) mutex_unlock( 15183866Sraf &thiscache->mt_cache_lock); 15193866Sraf } 15203866Sraf } 15213866Sraf (void) mutex_unlock(&cpuptr->mt_parent_lock); 15223866Sraf } 15233866Sraf (void) mutex_unlock(&oversize_lock); 15243866Sraf } 15253866Sraf 15263866Sraf #pragma init(malloc_init) 15273866Sraf static void 15283866Sraf malloc_init(void) 15293866Sraf { 15303866Sraf /* 15313866Sraf * This works in the init section for this library 15323866Sraf * because setup_caches() doesn't call anything in libc 15333866Sraf * that calls malloc(). If it did, disaster would ensue. 15343866Sraf * 15353866Sraf * For this to work properly, this library must be the first 15363866Sraf * one to have its init section called (after libc) by the 15373866Sraf * dynamic linker. If some other library's init section 15383866Sraf * ran first and called malloc(), disaster would ensue. 15393866Sraf * Because this is an interposer library for malloc(), the 15403866Sraf * dynamic linker arranges for its init section to run first. 15413866Sraf */ 15423866Sraf (void) setup_caches(); 15433866Sraf 15443866Sraf (void) pthread_atfork(malloc_prepare, malloc_release, malloc_release); 15453866Sraf } 1546