1 /* $NetBSD: kern_malloc.c,v 1.138 2012/02/06 12:13:44 drochner Exp $ */ 2 3 /* 4 * Copyright (c) 1987, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * @(#)kern_malloc.c 8.4 (Berkeley) 5/20/95 32 */ 33 34 /* 35 * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the University of 48 * California, Berkeley and its contributors. 49 * 4. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)kern_malloc.c 8.4 (Berkeley) 5/20/95 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: kern_malloc.c,v 1.138 2012/02/06 12:13:44 drochner Exp $"); 70 71 #include <sys/param.h> 72 #include <sys/proc.h> 73 #include <sys/kernel.h> 74 #include <sys/malloc.h> 75 #include <sys/kmem.h> 76 #include <sys/systm.h> 77 #include <sys/debug.h> 78 #include <sys/mutex.h> 79 #include <sys/lockdebug.h> 80 81 #include <uvm/uvm_extern.h> 82 83 #include "opt_kmemstats.h" 84 #include "opt_malloclog.h" 85 #include "opt_malloc_debug.h" 86 87 struct kmembuckets kmembuckets[MINBUCKET + 16]; 88 struct kmemusage *kmemusage; 89 struct malloc_type *kmemstatistics; 90 91 kmutex_t malloc_lock; 92 93 struct malloc_header { 94 /* Total size, include the header. */ 95 size_t mh_size; 96 } __aligned(ALIGNBYTES+1); 97 98 /* 99 * Allocate a block of memory 100 */ 101 #ifdef MALLOCLOG 102 void * 103 _kern_malloc(unsigned long size, struct malloc_type *ksp, int flags, 104 const char *file, long line) 105 #else 106 void * 107 kern_malloc(unsigned long size, struct malloc_type *ksp, int flags) 108 #endif /* MALLOCLOG */ 109 { 110 const int kmflags = (flags & M_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 111 size_t allocsize, hdroffset; 112 struct malloc_header *mh; 113 void *p; 114 115 if (size >= PAGE_SIZE) { 116 allocsize = PAGE_SIZE + size; /* for page alignment */ 117 hdroffset = PAGE_SIZE - sizeof(struct malloc_header); 118 } else { 119 allocsize = sizeof(struct malloc_header) + size; 120 hdroffset = 0; 121 } 122 123 p = kmem_intr_alloc(allocsize, kmflags); 124 if (p == NULL) 125 return NULL; 126 127 if ((flags & M_ZERO) != 0) { 128 memset(p, 0, allocsize); 129 } 130 mh = (void *)((char *)p + hdroffset); 131 mh->mh_size = allocsize - hdroffset; 132 133 return mh + 1; 134 } 135 136 /* 137 * Free a block of memory allocated by malloc. 138 */ 139 #ifdef MALLOCLOG 140 void 141 _kern_free(void *addr, struct malloc_type *ksp, const char *file, long line) 142 #else 143 void 144 kern_free(void *addr, struct malloc_type *ksp) 145 #endif /* MALLOCLOG */ 146 { 147 struct malloc_header *mh; 148 149 mh = addr; 150 mh--; 151 152 if (mh->mh_size >= PAGE_SIZE + sizeof(struct malloc_header)) 153 kmem_intr_free((char *)addr - PAGE_SIZE, 154 mh->mh_size + PAGE_SIZE - sizeof(struct malloc_header)); 155 else 156 kmem_intr_free(mh, mh->mh_size); 157 } 158 159 /* 160 * Change the size of a block of memory. 161 */ 162 void * 163 kern_realloc(void *curaddr, unsigned long newsize, struct malloc_type *ksp, 164 int flags) 165 { 166 struct malloc_header *mh; 167 unsigned long cursize; 168 void *newaddr; 169 170 /* 171 * realloc() with a NULL pointer is the same as malloc(). 172 */ 173 if (curaddr == NULL) 174 return (malloc(newsize, ksp, flags)); 175 176 /* 177 * realloc() with zero size is the same as free(). 178 */ 179 if (newsize == 0) { 180 free(curaddr, ksp); 181 return (NULL); 182 } 183 184 #ifdef LOCKDEBUG 185 if ((flags & M_NOWAIT) == 0) { 186 ASSERT_SLEEPABLE(); 187 } 188 #endif 189 190 mh = curaddr; 191 mh--; 192 193 cursize = mh->mh_size - sizeof(struct malloc_header); 194 195 /* 196 * If we already actually have as much as they want, we're done. 197 */ 198 if (newsize <= cursize) 199 return (curaddr); 200 201 /* 202 * Can't satisfy the allocation with the existing block. 203 * Allocate a new one and copy the data. 204 */ 205 newaddr = malloc(newsize, ksp, flags); 206 if (__predict_false(newaddr == NULL)) { 207 /* 208 * malloc() failed, because flags included M_NOWAIT. 209 * Return NULL to indicate that failure. The old 210 * pointer is still valid. 211 */ 212 return (NULL); 213 } 214 memcpy(newaddr, curaddr, cursize); 215 216 /* 217 * We were successful: free the old allocation and return 218 * the new one. 219 */ 220 free(curaddr, ksp); 221 return (newaddr); 222 } 223 224 /* 225 * Add a malloc type to the system. 226 */ 227 void 228 malloc_type_attach(struct malloc_type *type) 229 { 230 231 if (type->ks_magic != M_MAGIC) 232 panic("malloc_type_attach: bad magic"); 233 234 #ifdef DIAGNOSTIC 235 { 236 struct malloc_type *ksp; 237 for (ksp = kmemstatistics; ksp != NULL; ksp = ksp->ks_next) { 238 if (ksp == type) 239 panic("%s: `%s' already on list", __func__, 240 type->ks_shortdesc); 241 } 242 } 243 #endif 244 245 #ifdef KMEMSTATS 246 #else 247 type->ks_limit = 0; 248 #endif 249 250 type->ks_next = kmemstatistics; 251 kmemstatistics = type; 252 } 253 254 /* 255 * Remove a malloc type from the system.. 256 */ 257 void 258 malloc_type_detach(struct malloc_type *type) 259 { 260 struct malloc_type *ksp; 261 262 #ifdef DIAGNOSTIC 263 if (type->ks_magic != M_MAGIC) 264 panic("malloc_type_detach: bad magic"); 265 #endif 266 267 if (type == kmemstatistics) 268 kmemstatistics = type->ks_next; 269 else { 270 for (ksp = kmemstatistics; ksp->ks_next != NULL; 271 ksp = ksp->ks_next) { 272 if (ksp->ks_next == type) { 273 ksp->ks_next = type->ks_next; 274 break; 275 } 276 } 277 #ifdef DIAGNOSTIC 278 if (ksp->ks_next == NULL) 279 panic("malloc_type_detach: not on list"); 280 #endif 281 } 282 type->ks_next = NULL; 283 } 284 285 /* 286 * Set the limit on a malloc type. 287 */ 288 void 289 malloc_type_setlimit(struct malloc_type *type, u_long limit) 290 { 291 #ifdef KMEMSTATS 292 mutex_spin_enter(&malloc_lock); 293 type->ks_limit = limit; 294 mutex_spin_exit(&malloc_lock); 295 #endif 296 } 297 298 /* 299 * Initialize the kernel memory allocator 300 */ 301 void 302 kmeminit(void) 303 { 304 __link_set_decl(malloc_types, struct malloc_type); 305 struct malloc_type * const *ksp; 306 #ifdef KMEMSTATS 307 long indx; 308 #endif 309 310 mutex_init(&malloc_lock, MUTEX_DEFAULT, IPL_VM); 311 312 #ifdef KMEMSTATS 313 for (indx = 0; indx < MINBUCKET + 16; indx++) { 314 if (1 << indx >= PAGE_SIZE) 315 kmembuckets[indx].kb_elmpercl = 1; 316 else 317 kmembuckets[indx].kb_elmpercl = PAGE_SIZE / (1 << indx); 318 kmembuckets[indx].kb_highwat = 319 5 * kmembuckets[indx].kb_elmpercl; 320 } 321 #endif 322 323 /* Attach all of the statically-linked malloc types. */ 324 __link_set_foreach(ksp, malloc_types) 325 malloc_type_attach(*ksp); 326 327 #ifdef MALLOC_DEBUG 328 debug_malloc_init(); 329 #endif 330 } 331 332 #ifdef DDB 333 #include <ddb/db_output.h> 334 335 /* 336 * Dump kmem statistics from ddb. 337 * 338 * usage: call dump_kmemstats 339 */ 340 void dump_kmemstats(void); 341 342 void 343 dump_kmemstats(void) 344 { 345 #ifdef KMEMSTATS 346 struct malloc_type *ksp; 347 348 for (ksp = kmemstatistics; ksp != NULL; ksp = ksp->ks_next) { 349 if (ksp->ks_memuse == 0) 350 continue; 351 db_printf("%s%.*s %ld\n", ksp->ks_shortdesc, 352 (int)(20 - strlen(ksp->ks_shortdesc)), 353 " ", 354 ksp->ks_memuse); 355 } 356 #else 357 db_printf("Kmem stats are not being collected.\n"); 358 #endif /* KMEMSTATS */ 359 } 360 #endif /* DDB */ 361 362 363 #if 0 364 /* 365 * Diagnostic messages about "Data modified on 366 * freelist" indicate a memory corruption, but 367 * they do not help tracking it down. 368 * This function can be called at various places 369 * to sanity check malloc's freelist and discover 370 * where does the corruption take place. 371 */ 372 int 373 freelist_sanitycheck(void) { 374 int i,j; 375 struct kmembuckets *kbp; 376 struct freelist *freep; 377 int rv = 0; 378 379 for (i = MINBUCKET; i <= MINBUCKET + 15; i++) { 380 kbp = &kmembuckets[i]; 381 freep = (struct freelist *)kbp->kb_next; 382 j = 0; 383 while(freep) { 384 vm_map_lock(kmem_map); 385 rv = uvm_map_checkprot(kmem_map, (vaddr_t)freep, 386 (vaddr_t)freep + sizeof(struct freelist), 387 VM_PROT_WRITE); 388 vm_map_unlock(kmem_map); 389 390 if ((rv == 0) || (*(int *)freep != WEIRD_ADDR)) { 391 printf("bucket %i, chunck %d at %p modified\n", 392 i, j, freep); 393 return 1; 394 } 395 freep = (struct freelist *)freep->next; 396 j++; 397 } 398 } 399 400 return 0; 401 } 402 #endif 403