10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 60Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 70Sstevel@tonic-gate * with the License. 80Sstevel@tonic-gate * 90Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 100Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 110Sstevel@tonic-gate * See the License for the specific language governing permissions 120Sstevel@tonic-gate * and limitations under the License. 130Sstevel@tonic-gate * 140Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 150Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 160Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 170Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 180Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 190Sstevel@tonic-gate * 200Sstevel@tonic-gate * CDDL HEADER END 210Sstevel@tonic-gate */ 220Sstevel@tonic-gate /* 23*776Sjwadams * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 240Sstevel@tonic-gate * Use is subject to license terms. 250Sstevel@tonic-gate */ 260Sstevel@tonic-gate 270Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 280Sstevel@tonic-gate 290Sstevel@tonic-gate #include <unistd.h> 300Sstevel@tonic-gate 310Sstevel@tonic-gate #include <errno.h> 320Sstevel@tonic-gate 330Sstevel@tonic-gate #include <string.h> 340Sstevel@tonic-gate 350Sstevel@tonic-gate #include <sys/sysmacros.h> 360Sstevel@tonic-gate 370Sstevel@tonic-gate #include "umem_base.h" 380Sstevel@tonic-gate 390Sstevel@tonic-gate #include "misc.h" 400Sstevel@tonic-gate 410Sstevel@tonic-gate /* 420Sstevel@tonic-gate * malloc_data_t is an 8-byte structure which is located "before" the pointer 430Sstevel@tonic-gate * returned from {m,c,re}alloc and memalign. The first four bytes give 440Sstevel@tonic-gate * information about the buffer, and the second four bytes are a status byte. 450Sstevel@tonic-gate * 460Sstevel@tonic-gate * See umem_impl.h for the various magic numbers used, and the size 470Sstevel@tonic-gate * encode/decode macros. 480Sstevel@tonic-gate * 490Sstevel@tonic-gate * The 'size' of the buffer includes the tags. That is, we encode the 500Sstevel@tonic-gate * argument to umem_alloc(), not the argument to malloc(). 510Sstevel@tonic-gate */ 520Sstevel@tonic-gate 530Sstevel@tonic-gate typedef struct malloc_data { 540Sstevel@tonic-gate uint32_t malloc_size; 550Sstevel@tonic-gate uint32_t malloc_stat; /* = UMEM_MALLOC_ENCODE(state, malloc_size) */ 560Sstevel@tonic-gate } malloc_data_t; 570Sstevel@tonic-gate 580Sstevel@tonic-gate void * 590Sstevel@tonic-gate malloc(size_t size_arg) 600Sstevel@tonic-gate { 610Sstevel@tonic-gate #ifdef _LP64 620Sstevel@tonic-gate uint32_t high_size = 0; 630Sstevel@tonic-gate #endif 640Sstevel@tonic-gate size_t size; 650Sstevel@tonic-gate 660Sstevel@tonic-gate malloc_data_t *ret; 670Sstevel@tonic-gate size = size_arg + sizeof (malloc_data_t); 680Sstevel@tonic-gate 690Sstevel@tonic-gate #ifdef _LP64 700Sstevel@tonic-gate if (size > UMEM_SECOND_ALIGN) { 710Sstevel@tonic-gate size += sizeof (malloc_data_t); 720Sstevel@tonic-gate high_size = (size >> 32); 730Sstevel@tonic-gate } 740Sstevel@tonic-gate #endif 750Sstevel@tonic-gate if (size < size_arg) { 760Sstevel@tonic-gate errno = ENOMEM; /* overflow */ 770Sstevel@tonic-gate return (NULL); 780Sstevel@tonic-gate } 790Sstevel@tonic-gate ret = (malloc_data_t *)_umem_alloc(size, UMEM_DEFAULT); 800Sstevel@tonic-gate if (ret == NULL) { 810Sstevel@tonic-gate if (size <= UMEM_MAXBUF) 820Sstevel@tonic-gate errno = EAGAIN; 830Sstevel@tonic-gate else 840Sstevel@tonic-gate errno = ENOMEM; 850Sstevel@tonic-gate return (NULL); 860Sstevel@tonic-gate #ifdef _LP64 870Sstevel@tonic-gate } else if (high_size > 0) { 880Sstevel@tonic-gate uint32_t low_size = (uint32_t)size; 890Sstevel@tonic-gate 900Sstevel@tonic-gate /* 910Sstevel@tonic-gate * uses different magic numbers to make it harder to 920Sstevel@tonic-gate * undetectably corrupt 930Sstevel@tonic-gate */ 940Sstevel@tonic-gate ret->malloc_size = high_size; 950Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_MAGIC, high_size); 960Sstevel@tonic-gate ret++; 970Sstevel@tonic-gate 980Sstevel@tonic-gate ret->malloc_size = low_size; 990Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_OVERSIZE_MAGIC, 1000Sstevel@tonic-gate low_size); 1010Sstevel@tonic-gate ret++; 1020Sstevel@tonic-gate } else if (size > UMEM_SECOND_ALIGN) { 1030Sstevel@tonic-gate uint32_t low_size = (uint32_t)size; 1040Sstevel@tonic-gate 1050Sstevel@tonic-gate ret++; /* leave the first 8 bytes alone */ 1060Sstevel@tonic-gate 1070Sstevel@tonic-gate ret->malloc_size = low_size; 1080Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, 1090Sstevel@tonic-gate low_size); 1100Sstevel@tonic-gate ret++; 1110Sstevel@tonic-gate #endif 1120Sstevel@tonic-gate } else { 1130Sstevel@tonic-gate ret->malloc_size = size; 1140Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MALLOC_MAGIC, size); 1150Sstevel@tonic-gate ret++; 1160Sstevel@tonic-gate } 1170Sstevel@tonic-gate return ((void *)ret); 1180Sstevel@tonic-gate } 1190Sstevel@tonic-gate 1200Sstevel@tonic-gate void * 1210Sstevel@tonic-gate calloc(size_t nelem, size_t elsize) 1220Sstevel@tonic-gate { 1230Sstevel@tonic-gate size_t size = nelem * elsize; 1240Sstevel@tonic-gate void *retval; 1250Sstevel@tonic-gate 1260Sstevel@tonic-gate if (nelem > 0 && elsize > 0 && size/nelem != elsize) { 1270Sstevel@tonic-gate errno = ENOMEM; /* overflow */ 1280Sstevel@tonic-gate return (NULL); 1290Sstevel@tonic-gate } 1300Sstevel@tonic-gate 1310Sstevel@tonic-gate retval = malloc(size); 1320Sstevel@tonic-gate if (retval == NULL) 1330Sstevel@tonic-gate return (NULL); 1340Sstevel@tonic-gate 1350Sstevel@tonic-gate (void) memset(retval, 0, size); 1360Sstevel@tonic-gate return (retval); 1370Sstevel@tonic-gate } 1380Sstevel@tonic-gate 1390Sstevel@tonic-gate /* 1400Sstevel@tonic-gate * memalign uses vmem_xalloc to do its work. 1410Sstevel@tonic-gate * 1420Sstevel@tonic-gate * in 64-bit, the memaligned buffer always has two tags. This simplifies the 1430Sstevel@tonic-gate * code. 1440Sstevel@tonic-gate */ 1450Sstevel@tonic-gate 1460Sstevel@tonic-gate void * 1470Sstevel@tonic-gate memalign(size_t align, size_t size_arg) 1480Sstevel@tonic-gate { 1490Sstevel@tonic-gate size_t size; 1500Sstevel@tonic-gate uintptr_t phase; 1510Sstevel@tonic-gate 1520Sstevel@tonic-gate void *buf; 1530Sstevel@tonic-gate malloc_data_t *ret; 1540Sstevel@tonic-gate 1550Sstevel@tonic-gate size_t overhead; 1560Sstevel@tonic-gate 1570Sstevel@tonic-gate if (size_arg == 0 || align == 0 || (align & (align - 1)) != 0) { 1580Sstevel@tonic-gate errno = EINVAL; 1590Sstevel@tonic-gate return (NULL); 1600Sstevel@tonic-gate } 1610Sstevel@tonic-gate 1620Sstevel@tonic-gate /* 1630Sstevel@tonic-gate * if malloc provides the required alignment, use it. 1640Sstevel@tonic-gate */ 1650Sstevel@tonic-gate if (align <= UMEM_ALIGN || 1660Sstevel@tonic-gate (align <= UMEM_SECOND_ALIGN && size_arg >= UMEM_SECOND_ALIGN)) 1670Sstevel@tonic-gate return (malloc(size_arg)); 1680Sstevel@tonic-gate 1690Sstevel@tonic-gate #ifdef _LP64 1700Sstevel@tonic-gate overhead = 2 * sizeof (malloc_data_t); 1710Sstevel@tonic-gate #else 1720Sstevel@tonic-gate overhead = sizeof (malloc_data_t); 1730Sstevel@tonic-gate #endif 1740Sstevel@tonic-gate 1750Sstevel@tonic-gate ASSERT(overhead <= align); 1760Sstevel@tonic-gate 1770Sstevel@tonic-gate size = size_arg + overhead; 1780Sstevel@tonic-gate phase = align - overhead; 1790Sstevel@tonic-gate 1800Sstevel@tonic-gate if (umem_memalign_arena == NULL && umem_init() == 0) { 1810Sstevel@tonic-gate errno = ENOMEM; 1820Sstevel@tonic-gate return (NULL); 1830Sstevel@tonic-gate } 1840Sstevel@tonic-gate 1850Sstevel@tonic-gate if (size < size_arg) { 1860Sstevel@tonic-gate errno = ENOMEM; /* overflow */ 1870Sstevel@tonic-gate return (NULL); 1880Sstevel@tonic-gate } 1890Sstevel@tonic-gate 1900Sstevel@tonic-gate buf = vmem_xalloc(umem_memalign_arena, size, align, phase, 1910Sstevel@tonic-gate 0, NULL, NULL, VM_NOSLEEP); 1920Sstevel@tonic-gate 1930Sstevel@tonic-gate if (buf == NULL) { 1940Sstevel@tonic-gate if ((size_arg + align) <= UMEM_MAXBUF) 1950Sstevel@tonic-gate errno = EAGAIN; 1960Sstevel@tonic-gate else 1970Sstevel@tonic-gate errno = ENOMEM; 1980Sstevel@tonic-gate 1990Sstevel@tonic-gate return (NULL); 2000Sstevel@tonic-gate } 2010Sstevel@tonic-gate 2020Sstevel@tonic-gate ret = (malloc_data_t *)buf; 2030Sstevel@tonic-gate { 2040Sstevel@tonic-gate uint32_t low_size = (uint32_t)size; 2050Sstevel@tonic-gate 2060Sstevel@tonic-gate #ifdef _LP64 2070Sstevel@tonic-gate uint32_t high_size = (uint32_t)(size >> 32); 2080Sstevel@tonic-gate 2090Sstevel@tonic-gate ret->malloc_size = high_size; 2100Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MEMALIGN_MAGIC, 2110Sstevel@tonic-gate high_size); 2120Sstevel@tonic-gate ret++; 2130Sstevel@tonic-gate #endif 2140Sstevel@tonic-gate 2150Sstevel@tonic-gate ret->malloc_size = low_size; 2160Sstevel@tonic-gate ret->malloc_stat = UMEM_MALLOC_ENCODE(MEMALIGN_MAGIC, low_size); 2170Sstevel@tonic-gate ret++; 2180Sstevel@tonic-gate } 2190Sstevel@tonic-gate 2200Sstevel@tonic-gate ASSERT(P2PHASE((uintptr_t)ret, align) == 0); 2210Sstevel@tonic-gate ASSERT((void *)((uintptr_t)ret - overhead) == buf); 2220Sstevel@tonic-gate 2230Sstevel@tonic-gate return ((void *)ret); 2240Sstevel@tonic-gate } 2250Sstevel@tonic-gate 2260Sstevel@tonic-gate void * 2270Sstevel@tonic-gate valloc(size_t size) 2280Sstevel@tonic-gate { 2290Sstevel@tonic-gate return (memalign(pagesize, size)); 2300Sstevel@tonic-gate } 2310Sstevel@tonic-gate 2320Sstevel@tonic-gate /* 2330Sstevel@tonic-gate * process_free: 2340Sstevel@tonic-gate * 2350Sstevel@tonic-gate * Pulls information out of a buffer pointer, and optionally free it. 2360Sstevel@tonic-gate * This is used by free() and realloc() to process buffers. 2370Sstevel@tonic-gate * 2380Sstevel@tonic-gate * On failure, calls umem_err_recoverable() with an appropriate message 2390Sstevel@tonic-gate * On success, returns the data size through *data_size_arg, if (!is_free). 2400Sstevel@tonic-gate * 2410Sstevel@tonic-gate * Preserves errno, since free()'s semantics require it. 2420Sstevel@tonic-gate */ 2430Sstevel@tonic-gate 2440Sstevel@tonic-gate static int 2450Sstevel@tonic-gate process_free(void *buf_arg, 2460Sstevel@tonic-gate int do_free, /* free the buffer, or just get its size? */ 2470Sstevel@tonic-gate size_t *data_size_arg) /* output: bytes of data in buf_arg */ 2480Sstevel@tonic-gate { 2490Sstevel@tonic-gate malloc_data_t *buf; 2500Sstevel@tonic-gate 2510Sstevel@tonic-gate void *base; 2520Sstevel@tonic-gate size_t size; 2530Sstevel@tonic-gate size_t data_size; 2540Sstevel@tonic-gate 2550Sstevel@tonic-gate const char *message; 2560Sstevel@tonic-gate int old_errno = errno; 2570Sstevel@tonic-gate 2580Sstevel@tonic-gate buf = (malloc_data_t *)buf_arg; 2590Sstevel@tonic-gate 2600Sstevel@tonic-gate buf--; 2610Sstevel@tonic-gate size = buf->malloc_size; 2620Sstevel@tonic-gate 2630Sstevel@tonic-gate switch (UMEM_MALLOC_DECODE(buf->malloc_stat, size)) { 2640Sstevel@tonic-gate 2650Sstevel@tonic-gate case MALLOC_MAGIC: 2660Sstevel@tonic-gate base = (void *)buf; 2670Sstevel@tonic-gate data_size = size - sizeof (malloc_data_t); 2680Sstevel@tonic-gate 2690Sstevel@tonic-gate if (do_free) 2700Sstevel@tonic-gate buf->malloc_stat = UMEM_FREE_PATTERN_32; 2710Sstevel@tonic-gate 2720Sstevel@tonic-gate goto process_malloc; 2730Sstevel@tonic-gate 2740Sstevel@tonic-gate #ifdef _LP64 2750Sstevel@tonic-gate case MALLOC_SECOND_MAGIC: 2760Sstevel@tonic-gate base = (void *)(buf - 1); 2770Sstevel@tonic-gate data_size = size - 2 * sizeof (malloc_data_t); 2780Sstevel@tonic-gate 2790Sstevel@tonic-gate if (do_free) 2800Sstevel@tonic-gate buf->malloc_stat = UMEM_FREE_PATTERN_32; 2810Sstevel@tonic-gate 2820Sstevel@tonic-gate goto process_malloc; 2830Sstevel@tonic-gate 2840Sstevel@tonic-gate case MALLOC_OVERSIZE_MAGIC: { 2850Sstevel@tonic-gate size_t high_size; 2860Sstevel@tonic-gate 2870Sstevel@tonic-gate buf--; 2880Sstevel@tonic-gate high_size = buf->malloc_size; 2890Sstevel@tonic-gate 2900Sstevel@tonic-gate if (UMEM_MALLOC_DECODE(buf->malloc_stat, high_size) != 2910Sstevel@tonic-gate MALLOC_MAGIC) { 2920Sstevel@tonic-gate message = "invalid or corrupted buffer"; 2930Sstevel@tonic-gate break; 2940Sstevel@tonic-gate } 2950Sstevel@tonic-gate 2960Sstevel@tonic-gate size += high_size << 32; 2970Sstevel@tonic-gate 2980Sstevel@tonic-gate base = (void *)buf; 2990Sstevel@tonic-gate data_size = size - 2 * sizeof (malloc_data_t); 3000Sstevel@tonic-gate 3010Sstevel@tonic-gate if (do_free) { 3020Sstevel@tonic-gate buf->malloc_stat = UMEM_FREE_PATTERN_32; 3030Sstevel@tonic-gate (buf + 1)->malloc_stat = UMEM_FREE_PATTERN_32; 3040Sstevel@tonic-gate } 3050Sstevel@tonic-gate 3060Sstevel@tonic-gate goto process_malloc; 3070Sstevel@tonic-gate } 3080Sstevel@tonic-gate #endif 3090Sstevel@tonic-gate 3100Sstevel@tonic-gate case MEMALIGN_MAGIC: { 3110Sstevel@tonic-gate size_t overhead = sizeof (malloc_data_t); 3120Sstevel@tonic-gate 3130Sstevel@tonic-gate #ifdef _LP64 3140Sstevel@tonic-gate size_t high_size; 3150Sstevel@tonic-gate 3160Sstevel@tonic-gate overhead += sizeof (malloc_data_t); 3170Sstevel@tonic-gate 3180Sstevel@tonic-gate buf--; 3190Sstevel@tonic-gate high_size = buf->malloc_size; 3200Sstevel@tonic-gate 3210Sstevel@tonic-gate if (UMEM_MALLOC_DECODE(buf->malloc_stat, high_size) != 3220Sstevel@tonic-gate MEMALIGN_MAGIC) { 3230Sstevel@tonic-gate message = "invalid or corrupted buffer"; 3240Sstevel@tonic-gate break; 3250Sstevel@tonic-gate } 3260Sstevel@tonic-gate size += high_size << 32; 3270Sstevel@tonic-gate 3280Sstevel@tonic-gate /* 3290Sstevel@tonic-gate * destroy the main tag's malloc_stat 3300Sstevel@tonic-gate */ 3310Sstevel@tonic-gate if (do_free) 3320Sstevel@tonic-gate (buf + 1)->malloc_stat = UMEM_FREE_PATTERN_32; 3330Sstevel@tonic-gate #endif 3340Sstevel@tonic-gate 3350Sstevel@tonic-gate base = (void *)buf; 3360Sstevel@tonic-gate data_size = size - overhead; 3370Sstevel@tonic-gate 3380Sstevel@tonic-gate if (do_free) 3390Sstevel@tonic-gate buf->malloc_stat = UMEM_FREE_PATTERN_32; 3400Sstevel@tonic-gate 3410Sstevel@tonic-gate goto process_memalign; 3420Sstevel@tonic-gate } 3430Sstevel@tonic-gate default: 3440Sstevel@tonic-gate if (buf->malloc_stat == UMEM_FREE_PATTERN_32) 3450Sstevel@tonic-gate message = "double-free or invalid buffer"; 3460Sstevel@tonic-gate else 3470Sstevel@tonic-gate message = "invalid or corrupted buffer"; 3480Sstevel@tonic-gate break; 3490Sstevel@tonic-gate } 3500Sstevel@tonic-gate 3510Sstevel@tonic-gate umem_err_recoverable("%s(%p): %s\n", 3520Sstevel@tonic-gate do_free? "free" : "realloc", buf_arg, message); 3530Sstevel@tonic-gate 3540Sstevel@tonic-gate errno = old_errno; 3550Sstevel@tonic-gate return (0); 3560Sstevel@tonic-gate 3570Sstevel@tonic-gate process_malloc: 3580Sstevel@tonic-gate if (do_free) 3590Sstevel@tonic-gate _umem_free(base, size); 3600Sstevel@tonic-gate else 3610Sstevel@tonic-gate *data_size_arg = data_size; 3620Sstevel@tonic-gate 3630Sstevel@tonic-gate errno = old_errno; 3640Sstevel@tonic-gate return (1); 3650Sstevel@tonic-gate 3660Sstevel@tonic-gate process_memalign: 3670Sstevel@tonic-gate if (do_free) 3680Sstevel@tonic-gate vmem_xfree(umem_memalign_arena, base, size); 3690Sstevel@tonic-gate else 3700Sstevel@tonic-gate *data_size_arg = data_size; 3710Sstevel@tonic-gate 3720Sstevel@tonic-gate errno = old_errno; 3730Sstevel@tonic-gate return (1); 3740Sstevel@tonic-gate } 3750Sstevel@tonic-gate 3760Sstevel@tonic-gate void 3770Sstevel@tonic-gate free(void *buf) 3780Sstevel@tonic-gate { 3790Sstevel@tonic-gate if (buf == NULL) 3800Sstevel@tonic-gate return; 3810Sstevel@tonic-gate 3820Sstevel@tonic-gate /* 3830Sstevel@tonic-gate * Process buf, freeing it if it is not corrupt. 3840Sstevel@tonic-gate */ 3850Sstevel@tonic-gate (void) process_free(buf, 1, NULL); 3860Sstevel@tonic-gate } 3870Sstevel@tonic-gate 3880Sstevel@tonic-gate void * 3890Sstevel@tonic-gate realloc(void *buf_arg, size_t newsize) 3900Sstevel@tonic-gate { 3910Sstevel@tonic-gate size_t oldsize; 3920Sstevel@tonic-gate void *buf; 3930Sstevel@tonic-gate 3940Sstevel@tonic-gate if (buf_arg == NULL) 3950Sstevel@tonic-gate return (malloc(newsize)); 3960Sstevel@tonic-gate 397*776Sjwadams if (newsize == 0) { 398*776Sjwadams free(buf_arg); 399*776Sjwadams return (NULL); 400*776Sjwadams } 401*776Sjwadams 4020Sstevel@tonic-gate /* 4030Sstevel@tonic-gate * get the old data size without freeing the buffer 4040Sstevel@tonic-gate */ 4050Sstevel@tonic-gate if (process_free(buf_arg, 0, &oldsize) == 0) { 4060Sstevel@tonic-gate errno = EINVAL; 4070Sstevel@tonic-gate return (NULL); 4080Sstevel@tonic-gate } 4090Sstevel@tonic-gate 4100Sstevel@tonic-gate if (newsize == oldsize) /* size didn't change */ 4110Sstevel@tonic-gate return (buf_arg); 4120Sstevel@tonic-gate 4130Sstevel@tonic-gate buf = malloc(newsize); 4140Sstevel@tonic-gate if (buf == NULL) 4150Sstevel@tonic-gate return (NULL); 4160Sstevel@tonic-gate 4170Sstevel@tonic-gate (void) memcpy(buf, buf_arg, MIN(newsize, oldsize)); 4180Sstevel@tonic-gate free(buf_arg); 4190Sstevel@tonic-gate return (buf); 4200Sstevel@tonic-gate } 421