80 static void *zget(vm_zone_t z, int *tryagainp);
89 zalloc(vm_zone_t z)  in zalloc()  argument
98 	if (z == NULL)  in zalloc()
101 	zpcpu = &z->zpcpu[gd->gd_cpuid];  in zalloc()
131 	spin_lock(&z->zspin);  in zalloc()
132 	if (z->zfreecnt > z->zfreemin) {  in zalloc()
135 			item = z->zitems;  in zalloc()
141 			z->zitems = ((void **)item)[0];  in zalloc()
142 			--z->zfreecnt;  in zalloc()
146 		} while (--n > 0 && z->zfreecnt > z->zfreemin);  in zalloc()
147 		spin_unlock(&z->zspin);  in zalloc()
150 		spin_unlock(&z->zspin);  in zalloc()
152 		item = zget(z, &tryagain);  in zalloc()
160 		if (item == NULL && (z->zflags & ZONE_PANICFAIL))  in zalloc()
161 			panic("zalloc(%s) failed", z->zname);  in zalloc()
172 zfree(vm_zone_t z, void *item)  in zfree()  argument
180 	zpcpu = &z->zpcpu[gd->gd_cpuid];  in zfree()
185 	zmax = z->zmax_pcpu;  in zfree()
229 	spin_lock(&z->zspin);  in zfree()
230 	((void **)tail_item)[0] = z->zitems;  in zfree()
231 	z->zitems = item;  in zfree()
232 	z->zfreecnt += count;  in zfree()
233 	spin_unlock(&z->zspin);  in zfree()
284 zinitna(vm_zone_t z, char *name, size_t size, long nentries, uint32_t flags)  in zinitna()  argument
291 	if (z->zflags & ZONE_DESTROYABLE)  in zinitna()
298 	if ((z->zflags & ZONE_BOOT) == 0) {  in zinitna()
299 		z->zsize = roundup2(size, ZONE_ROUNDING);  in zinitna()
300 		spin_init(&z->zspin, "zinitna");  in zinitna()
301 		lockinit(&z->zgetlk, "zgetlk", 0, LK_CANRECURSE);  in zinitna()
303 		z->zfreecnt = 0;  in zinitna()
304 		z->ztotal = 0;  in zinitna()
305 		z->zmax = 0;  in zinitna()
306 		z->zname = name;  in zinitna()
307 		z->zitems = NULL;  in zinitna()
310 		LIST_INSERT_HEAD(&zlist, z, zlink);  in zinitna()
313 		bzero(z->zpcpu, sizeof(z->zpcpu));  in zinitna()
316 	z->zkmvec = NULL;  in zinitna()
317 	z->zkmcur = z->zkmmax = 0;  in zinitna()
318 	z->zflags |= flags;  in zinitna()
327 	if (z->zflags & ZONE_INTERRUPT) {  in zinitna()
328 		totsize = round_page((size_t)z->zsize * nentries);  in zinitna()
331 		z->zkva = kmem_alloc_pageable(kernel_map, totsize,  in zinitna()
333 		if (z->zkva == 0) {  in zinitna()
334 			LIST_REMOVE(z, zlink);  in zinitna()
338 		z->zpagemax = totsize / PAGE_SIZE;  in zinitna()
339 		z->zallocflag = VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT |  in zinitna()
341 		z->zmax += nentries;  in zinitna()
351 		z->zmax_pcpu = z->zmax / ncpus / 64;  in zinitna()
352 		if (z->zmax_pcpu < 1024)  in zinitna()
353 			z->zmax_pcpu = 1024;  in zinitna()
354 		if (z->zmax_pcpu * z->zsize > 16*1024*1024)  in zinitna()
355 			z->zmax_pcpu = 16*1024*1024 / z->zsize;  in zinitna()
357 		z->zallocflag = VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM;  in zinitna()
358 		z->zmax = 0;  in zinitna()
359 		z->zmax_pcpu = 8192;  in zinitna()
363 	if (z->zsize > PAGE_SIZE)  in zinitna()
364 		z->zfreemin = 1;  in zinitna()
366 		z->zfreemin = PAGE_SIZE / z->zsize;  in zinitna()
368 	z->zpagecount = 0;  in zinitna()
373 	z->zalloc = ZONE_MAXPGLOAD;  in zinitna()
379 	if (z->zflags & ZONE_INTERRUPT) {  in zinitna()
382 		buf = zget(z, NULL);  in zinitna()
384 			zfree(z, buf);  in zinitna()
402 	vm_zone_t z;  in zinit()  local
404 	z = (vm_zone_t) kmalloc(sizeof (struct vm_zone), M_ZONE, M_NOWAIT);  in zinit()
405 	if (z == NULL)  in zinit()
408 	z->zflags = 0;  in zinit()
409 	if (zinitna(z, name, size, nentries, flags & ~ZONE_DESTROYABLE) == 0) {  in zinit()
410 		kfree(z, M_ZONE);  in zinit()
415 		z->zflags |= ZONE_DESTROYABLE;  in zinit()
417 	return z;  in zinit()
427 zbootinit(vm_zone_t z, char *name, size_t size, void *item, long nitems)  in zbootinit()  argument
431 	spin_init(&z->zspin, "zbootinit");  in zbootinit()
432 	lockinit(&z->zgetlk, "zgetlk", 0, LK_CANRECURSE);  in zbootinit()
433 	bzero(z->zpcpu, sizeof(z->zpcpu));  in zbootinit()
434 	z->zname = name;  in zbootinit()
435 	z->zsize = size;  in zbootinit()
436 	z->zpagemax = 0;  in zbootinit()
437 	z->zflags = ZONE_BOOT;  in zbootinit()
438 	z->zfreemin = 0;  in zbootinit()
439 	z->zallocflag = 0;  in zbootinit()
440 	z->zpagecount = 0;  in zbootinit()
441 	z->zalloc = 0;  in zbootinit()
443 	bzero(item, (size_t)nitems * z->zsize);  in zbootinit()
444 	z->zitems = NULL;  in zbootinit()
446 		((void **)item)[0] = z->zitems;  in zbootinit()
450 		z->zitems = item;  in zbootinit()
451 		item = (uint8_t *)item + z->zsize;  in zbootinit()
453 	z->zfreecnt = nitems;  in zbootinit()
454 	z->zmax = nitems;  in zbootinit()
455 	z->ztotal = nitems;  in zbootinit()
458 	LIST_INSERT_HEAD(&zlist, z, zlink);  in zbootinit()
468 zdestroy(vm_zone_t z)  in zdestroy()  argument
472 	if (z == NULL)  in zdestroy()
474 	if ((z->zflags & ZONE_DESTROYABLE) == 0)  in zdestroy()
478 	LIST_REMOVE(z, zlink);  in zdestroy()
484 	KKASSERT((z->zflags & ZONE_INTERRUPT) == 0);  in zdestroy()
485 	for (i = 0; i < z->zkmcur; i++) {  in zdestroy()
486 		kmem_free(kernel_map, z->zkmvec[i],  in zdestroy()
487 			  (size_t)z->zalloc * PAGE_SIZE);  in zdestroy()
488 		atomic_subtract_long(&zone_kern_pages, z->zalloc);  in zdestroy()
490 	if (z->zkmvec != NULL)  in zdestroy()
491 		kfree(z->zkmvec, M_ZONE);  in zdestroy()
493 	spin_uninit(&z->zspin);  in zdestroy()
494 	kfree(z, M_ZONE);  in zdestroy()
516 zget(vm_zone_t z, int *tryagainp)  in zget()  argument
529 	if (z == NULL)  in zget()
540 	if ((z->zflags & ZONE_INTERRUPT) == 0) {  in zget()
541 		if (lockmgr(&z->zgetlk, LK_EXCLUSIVE | LK_SLEEPFAIL)) {  in zget()
547 	if (z->zflags & ZONE_INTERRUPT) {  in zget()
557 		for (i = 0; i < ZONE_MAXPGLOAD && i < z->zalloc; ++i) {  in zget()
561 						  z->zallocflag);  in zget()
581 		spin_lock(&z->zspin);  in zget()
582 		noffset = (size_t)z->zpagecount * PAGE_SIZE;  in zget()
584 		savezpc = z->zpagecount;  in zget()
589 		if (z->zpagecount + nalloc > z->zpagemax)  in zget()
590 			z->zpagecount = z->zpagemax;  in zget()
592 			z->zpagecount += nalloc;  in zget()
594 		item = (char *)z->zkva + noffset;  in zget()
595 		npages = z->zpagecount - savezpc;  in zget()
597 			 z->zsize;  in zget()
599 		spin_unlock(&z->zspin);  in zget()
613 			zkva = z->zkva + (size_t)(savezpc + i) * PAGE_SIZE;  in zget()
621 	} else if (z->zflags & ZONE_SPECIAL) {  in zget()
629 		nbytes = (size_t)z->zalloc * PAGE_SIZE;  in zget()
630 		z->zpagecount += z->zalloc;	/* Track total memory use */  in zget()
637 			atomic_add_long(&zone_kern_pages, z->zalloc);  in zget()
642 		nitems = nbytes / z->zsize;  in zget()
647 		nbytes = (size_t)z->zalloc * PAGE_SIZE;  in zget()
648 		z->zpagecount += z->zalloc;	/* Track total memory use */  in zget()
655 			atomic_add_long(&zone_kern_pages, z->zalloc);  in zget()
658 			if (z->zflags & ZONE_DESTROYABLE) {  in zget()
659 				if (z->zkmcur == z->zkmmax) {  in zget()
660 					z->zkmmax =  in zget()
661 						z->zkmmax==0 ? 1 : z->zkmmax*2;  in zget()
662 					z->zkmvec = krealloc(z->zkmvec,  in zget()
663 					    z->zkmmax * sizeof(z->zkmvec[0]),  in zget()
666 				z->zkmvec[z->zkmcur++] = (vm_offset_t)item;  in zget()
671 		nitems = nbytes / z->zsize;  in zget()
678 	spin_lock(&z->zspin);  in zget()
679 	z->ztotal += nitems;  in zget()
687 	if (nitems > 1 && (z->zflags & ZONE_SPECIAL)) {  in zget()
703 			item = (uint8_t *)item + z->zsize;  in zget()
716 			((void **)item)[0] = z->zitems;  in zget()
720 			z->zitems = item;  in zget()
721 			item = (uint8_t *)item + z->zsize;  in zget()
723 		z->zfreecnt += nitems;  in zget()
724 		++z->znalloc;  in zget()
725 	} else if (z->zfreecnt > 0) {  in zget()
729 		item = z->zitems;  in zget()
730 		z->zitems = ((void **)item)[0];  in zget()
736 		--z->zfreecnt;  in zget()
737 		++z->znalloc;  in zget()
744 	spin_unlock(&z->zspin);  in zget()
753 	if ((z->zflags & ZONE_INTERRUPT) == 0)  in zget()
754 		lockmgr(&z->zgetlk, LK_RELEASE);  in zget()