10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51253Slq150181 * Common Development and Distribution License (the "License"). 61455Sandrei * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 211253Slq150181 220Sstevel@tonic-gate /* 233446Smrj * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 240Sstevel@tonic-gate * Use is subject to license terms. 250Sstevel@tonic-gate */ 260Sstevel@tonic-gate 270Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 280Sstevel@tonic-gate 290Sstevel@tonic-gate #include <sys/machsystm.h> 300Sstevel@tonic-gate #include <sys/archsystm.h> 310Sstevel@tonic-gate #include <sys/vm.h> 320Sstevel@tonic-gate #include <sys/cpu.h> 330Sstevel@tonic-gate #include <sys/atomic.h> 340Sstevel@tonic-gate #include <sys/reboot.h> 350Sstevel@tonic-gate #include <sys/kdi.h> 360Sstevel@tonic-gate #include <sys/bootconf.h> 370Sstevel@tonic-gate #include <sys/memlist_plat.h> 380Sstevel@tonic-gate #include <sys/memlist_impl.h> 390Sstevel@tonic-gate #include <sys/prom_plat.h> 400Sstevel@tonic-gate #include <sys/prom_isa.h> 410Sstevel@tonic-gate #include <sys/autoconf.h> 420Sstevel@tonic-gate #include <sys/intreg.h> 430Sstevel@tonic-gate #include <sys/ivintr.h> 440Sstevel@tonic-gate #include <sys/fpu/fpusystm.h> 450Sstevel@tonic-gate #include <sys/iommutsb.h> 460Sstevel@tonic-gate #include <vm/vm_dep.h> 470Sstevel@tonic-gate #include <vm/seg_dev.h> 480Sstevel@tonic-gate #include <vm/seg_kmem.h> 490Sstevel@tonic-gate #include <vm/seg_kpm.h> 500Sstevel@tonic-gate #include <vm/seg_map.h> 510Sstevel@tonic-gate #include <vm/seg_kp.h> 520Sstevel@tonic-gate #include <sys/sysconf.h> 530Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 540Sstevel@tonic-gate #include <sys/kobj.h> 550Sstevel@tonic-gate #include <sys/sun4asi.h> 560Sstevel@tonic-gate #include <sys/clconf.h> 570Sstevel@tonic-gate #include <sys/platform_module.h> 580Sstevel@tonic-gate #include <sys/panic.h> 590Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h> 600Sstevel@tonic-gate #include <sys/clock.h> 610Sstevel@tonic-gate #include <sys/cmn_err.h> 620Sstevel@tonic-gate #include <sys/promif.h> 630Sstevel@tonic-gate #include <sys/prom_debug.h> 640Sstevel@tonic-gate #include <sys/traptrace.h> 650Sstevel@tonic-gate #include <sys/memnode.h> 660Sstevel@tonic-gate #include <sys/mem_cage.h> 672241Shuah #include <sys/mmu.h> 680Sstevel@tonic-gate 690Sstevel@tonic-gate extern void setup_trap_table(void); 700Sstevel@tonic-gate extern void cpu_intrq_setup(struct cpu *); 710Sstevel@tonic-gate extern void cpu_intrq_register(struct cpu *); 720Sstevel@tonic-gate extern void contig_mem_init(void); 730Sstevel@tonic-gate extern void mach_dump_buffer_init(void); 740Sstevel@tonic-gate extern void mach_descrip_init(void); 751991Sheppo extern void mach_descrip_startup_fini(void); 760Sstevel@tonic-gate extern void mach_memscrub(void); 770Sstevel@tonic-gate extern void mach_fpras(void); 780Sstevel@tonic-gate extern void mach_cpu_halt_idle(void); 790Sstevel@tonic-gate extern void mach_hw_copy_limit(void); 801991Sheppo extern void load_mach_drivers(void); 810Sstevel@tonic-gate extern void load_tod_module(void); 820Sstevel@tonic-gate #pragma weak load_tod_module 830Sstevel@tonic-gate 840Sstevel@tonic-gate extern int ndata_alloc_mmfsa(struct memlist *ndata); 850Sstevel@tonic-gate #pragma weak ndata_alloc_mmfsa 860Sstevel@tonic-gate 871991Sheppo extern void cif_init(void); 881991Sheppo #pragma weak cif_init 891991Sheppo 900Sstevel@tonic-gate extern void parse_idprom(void); 910Sstevel@tonic-gate extern void add_vx_handler(char *, int, void (*)(cell_t *)); 920Sstevel@tonic-gate extern void mem_config_init(void); 930Sstevel@tonic-gate extern void memseg_remap_init(void); 940Sstevel@tonic-gate 952296Sae112802 extern void mach_kpm_init(void); 962296Sae112802 970Sstevel@tonic-gate /* 980Sstevel@tonic-gate * External Data: 990Sstevel@tonic-gate */ 1000Sstevel@tonic-gate extern int vac_size; /* cache size in bytes */ 1010Sstevel@tonic-gate extern uint_t vac_mask; /* VAC alignment consistency mask */ 1020Sstevel@tonic-gate extern uint_t vac_colors; 1030Sstevel@tonic-gate 1040Sstevel@tonic-gate /* 1050Sstevel@tonic-gate * Global Data Definitions: 1060Sstevel@tonic-gate */ 1070Sstevel@tonic-gate 1080Sstevel@tonic-gate /* 1090Sstevel@tonic-gate * XXX - Don't port this to new architectures 1100Sstevel@tonic-gate * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 1110Sstevel@tonic-gate * 'romp' has no use with a prom with an IEEE 1275 client interface. 1120Sstevel@tonic-gate * The driver doesn't use the value, but it depends on the symbol. 1130Sstevel@tonic-gate */ 1140Sstevel@tonic-gate void *romp; /* veritas driver won't load without romp 4154976 */ 1150Sstevel@tonic-gate /* 1160Sstevel@tonic-gate * Declare these as initialized data so we can patch them. 1170Sstevel@tonic-gate */ 1180Sstevel@tonic-gate pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 1190Sstevel@tonic-gate pgcnt_t segkpsize = 1200Sstevel@tonic-gate btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 1210Sstevel@tonic-gate uint_t segmap_percent = 12; /* Size of segmap segment */ 1220Sstevel@tonic-gate 1230Sstevel@tonic-gate int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 1240Sstevel@tonic-gate int vac_copyback = 1; 1250Sstevel@tonic-gate char *cache_mode = NULL; 1260Sstevel@tonic-gate int use_mix = 1; 1270Sstevel@tonic-gate int prom_debug = 0; 1280Sstevel@tonic-gate 1290Sstevel@tonic-gate struct bootops *bootops = 0; /* passed in from boot in %o2 */ 1300Sstevel@tonic-gate caddr_t boot_tba; /* %tba at boot - used by kmdb */ 1310Sstevel@tonic-gate uint_t tba_taken_over = 0; 1320Sstevel@tonic-gate 1330Sstevel@tonic-gate caddr_t s_text; /* start of kernel text segment */ 1340Sstevel@tonic-gate caddr_t e_text; /* end of kernel text segment */ 1350Sstevel@tonic-gate caddr_t s_data; /* start of kernel data segment */ 1360Sstevel@tonic-gate caddr_t e_data; /* end of kernel data segment */ 1370Sstevel@tonic-gate 1380Sstevel@tonic-gate caddr_t modtext; /* beginning of module text */ 1390Sstevel@tonic-gate size_t modtext_sz; /* size of module text */ 1400Sstevel@tonic-gate caddr_t moddata; /* beginning of module data reserve */ 1410Sstevel@tonic-gate caddr_t e_moddata; /* end of module data reserve */ 1420Sstevel@tonic-gate 1430Sstevel@tonic-gate /* 1440Sstevel@tonic-gate * End of first block of contiguous kernel in 32-bit virtual address space 1450Sstevel@tonic-gate */ 1460Sstevel@tonic-gate caddr_t econtig32; /* end of first blk of contiguous kernel */ 1470Sstevel@tonic-gate 1480Sstevel@tonic-gate caddr_t ncbase; /* beginning of non-cached segment */ 1490Sstevel@tonic-gate caddr_t ncend; /* end of non-cached segment */ 1500Sstevel@tonic-gate caddr_t sdata; /* beginning of data segment */ 1510Sstevel@tonic-gate 1520Sstevel@tonic-gate caddr_t extra_etva; /* beginning of unused nucleus text */ 1530Sstevel@tonic-gate pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 1540Sstevel@tonic-gate 1550Sstevel@tonic-gate size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 1560Sstevel@tonic-gate caddr_t nalloc_base; /* beginning of nucleus allocation */ 1570Sstevel@tonic-gate caddr_t nalloc_end; /* end of nucleus allocatable memory */ 1580Sstevel@tonic-gate caddr_t valloc_base; /* beginning of kvalloc segment */ 1590Sstevel@tonic-gate 1600Sstevel@tonic-gate caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 1610Sstevel@tonic-gate caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 1620Sstevel@tonic-gate 1632296Sae112802 uintptr_t shm_alignment; /* VAC address consistency modulus */ 1640Sstevel@tonic-gate struct memlist *phys_install; /* Total installed physical memory */ 1650Sstevel@tonic-gate struct memlist *phys_avail; /* Available (unreserved) physical memory */ 1660Sstevel@tonic-gate struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 1670Sstevel@tonic-gate struct memlist ndata; /* memlist of nucleus allocatable memory */ 1680Sstevel@tonic-gate int memexp_flag; /* memory expansion card flag */ 1690Sstevel@tonic-gate uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 1700Sstevel@tonic-gate pgcnt_t obp_pages; /* Physical pages used by OBP */ 1710Sstevel@tonic-gate 1720Sstevel@tonic-gate /* 1730Sstevel@tonic-gate * VM data structures 1740Sstevel@tonic-gate */ 1750Sstevel@tonic-gate long page_hashsz; /* Size of page hash table (power of two) */ 1760Sstevel@tonic-gate struct page *pp_base; /* Base of system page struct array */ 1770Sstevel@tonic-gate size_t pp_sz; /* Size in bytes of page struct array */ 1780Sstevel@tonic-gate struct page **page_hash; /* Page hash table */ 1790Sstevel@tonic-gate struct seg ktextseg; /* Segment used for kernel executable image */ 1800Sstevel@tonic-gate struct seg kvalloc; /* Segment used for "valloc" mapping */ 1810Sstevel@tonic-gate struct seg kpseg; /* Segment used for pageable kernel virt mem */ 1820Sstevel@tonic-gate struct seg ktexthole; /* Segment used for nucleus text hole */ 1830Sstevel@tonic-gate struct seg kmapseg; /* Segment used for generic kernel mappings */ 1840Sstevel@tonic-gate struct seg kpmseg; /* Segment used for physical mapping */ 1850Sstevel@tonic-gate struct seg kdebugseg; /* Segment used for the kernel debugger */ 1860Sstevel@tonic-gate 1870Sstevel@tonic-gate uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 1880Sstevel@tonic-gate size_t kpm_pp_sz; /* Size of system kpm_page array */ 1890Sstevel@tonic-gate pgcnt_t kpm_npages; /* How many kpm pages are managed */ 1900Sstevel@tonic-gate 1910Sstevel@tonic-gate struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 1920Sstevel@tonic-gate struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 1930Sstevel@tonic-gate struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 1940Sstevel@tonic-gate 1953290Sjohansen int segzio_fromheap = 0; /* zio allocations occur from heap */ 1963290Sjohansen caddr_t segzio_base; /* Base address of segzio */ 1973290Sjohansen pgcnt_t segziosize = 0; /* size of zio segment in pages */ 1983290Sjohansen 1990Sstevel@tonic-gate /* 2000Sstevel@tonic-gate * debugger pages (if allocated) 2010Sstevel@tonic-gate */ 2020Sstevel@tonic-gate struct vnode kdebugvp; 2030Sstevel@tonic-gate 2040Sstevel@tonic-gate /* 2053446Smrj * VA range available to the debugger 2063446Smrj */ 2073446Smrj const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 2083446Smrj const size_t kdi_segdebugsize = SEGDEBUGSIZE; 2093446Smrj 2103446Smrj /* 2110Sstevel@tonic-gate * Segment for relocated kernel structures in 64-bit large RAM kernels 2120Sstevel@tonic-gate */ 2130Sstevel@tonic-gate struct seg kmem64; 2140Sstevel@tonic-gate 2150Sstevel@tonic-gate struct memseg *memseg_base; 2160Sstevel@tonic-gate size_t memseg_sz; /* Used to translate a va to page */ 2170Sstevel@tonic-gate struct vnode unused_pages_vp; 2180Sstevel@tonic-gate 2190Sstevel@tonic-gate /* 2200Sstevel@tonic-gate * VM data structures allocated early during boot. 2210Sstevel@tonic-gate */ 2220Sstevel@tonic-gate size_t pagehash_sz; 2230Sstevel@tonic-gate uint64_t memlist_sz; 2240Sstevel@tonic-gate 2250Sstevel@tonic-gate char tbr_wr_addr_inited = 0; 2260Sstevel@tonic-gate 2270Sstevel@tonic-gate 2280Sstevel@tonic-gate /* 2290Sstevel@tonic-gate * Static Routines: 2300Sstevel@tonic-gate */ 2310Sstevel@tonic-gate static void memlist_add(uint64_t, uint64_t, struct memlist **, 2320Sstevel@tonic-gate struct memlist **); 2330Sstevel@tonic-gate static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 2340Sstevel@tonic-gate pgcnt_t); 2350Sstevel@tonic-gate static void kvm_init(void); 2360Sstevel@tonic-gate 2370Sstevel@tonic-gate static void startup_init(void); 2380Sstevel@tonic-gate static void startup_memlist(void); 2390Sstevel@tonic-gate static void startup_modules(void); 2400Sstevel@tonic-gate static void startup_bop_gone(void); 2410Sstevel@tonic-gate static void startup_vm(void); 2420Sstevel@tonic-gate static void startup_end(void); 2430Sstevel@tonic-gate static void setup_cage_params(void); 2441253Slq150181 static void startup_create_io_node(void); 2450Sstevel@tonic-gate 2460Sstevel@tonic-gate static pgcnt_t npages; 2470Sstevel@tonic-gate static struct memlist *memlist; 2480Sstevel@tonic-gate void *memlist_end; 2490Sstevel@tonic-gate 2500Sstevel@tonic-gate static pgcnt_t bop_alloc_pages; 2510Sstevel@tonic-gate static caddr_t hblk_base; 2520Sstevel@tonic-gate uint_t hblk_alloc_dynamic = 0; 2530Sstevel@tonic-gate uint_t hblk1_min = H1MIN; 2540Sstevel@tonic-gate uint_t hblk8_min; 2550Sstevel@tonic-gate 2560Sstevel@tonic-gate 2570Sstevel@tonic-gate /* 2580Sstevel@tonic-gate * Hooks for unsupported platforms and down-rev firmware 2590Sstevel@tonic-gate */ 2600Sstevel@tonic-gate int iam_positron(void); 2610Sstevel@tonic-gate #pragma weak iam_positron 2620Sstevel@tonic-gate static void do_prom_version_check(void); 2630Sstevel@tonic-gate static void kpm_init(void); 2640Sstevel@tonic-gate static void kpm_npages_setup(int); 2650Sstevel@tonic-gate static void kpm_memseg_init(void); 2660Sstevel@tonic-gate 2670Sstevel@tonic-gate /* 2680Sstevel@tonic-gate * After receiving a thermal interrupt, this is the number of seconds 2690Sstevel@tonic-gate * to delay before shutting off the system, assuming 2700Sstevel@tonic-gate * shutdown fails. Use /etc/system to change the delay if this isn't 2710Sstevel@tonic-gate * large enough. 2720Sstevel@tonic-gate */ 2730Sstevel@tonic-gate int thermal_powerdown_delay = 1200; 2740Sstevel@tonic-gate 2750Sstevel@tonic-gate /* 2760Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed 2770Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel. 2780Sstevel@tonic-gate */ 2790Sstevel@tonic-gate int page_relocate_ready = 0; 2800Sstevel@tonic-gate 2810Sstevel@tonic-gate /* 2820Sstevel@tonic-gate * Enable some debugging messages concerning memory usage... 2830Sstevel@tonic-gate */ 2840Sstevel@tonic-gate #ifdef DEBUGGING_MEM 2850Sstevel@tonic-gate static int debugging_mem; 2860Sstevel@tonic-gate static void 2870Sstevel@tonic-gate printmemlist(char *title, struct memlist *list) 2880Sstevel@tonic-gate { 2890Sstevel@tonic-gate if (!debugging_mem) 2900Sstevel@tonic-gate return; 2910Sstevel@tonic-gate 2920Sstevel@tonic-gate printf("%s\n", title); 2930Sstevel@tonic-gate 2940Sstevel@tonic-gate while (list) { 2950Sstevel@tonic-gate prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 2960Sstevel@tonic-gate (uint32_t)(list->address >> 32), (uint32_t)list->address, 2970Sstevel@tonic-gate (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 2980Sstevel@tonic-gate list = list->next; 2990Sstevel@tonic-gate } 3000Sstevel@tonic-gate } 3010Sstevel@tonic-gate 3020Sstevel@tonic-gate void 3030Sstevel@tonic-gate printmemseg(struct memseg *memseg) 3040Sstevel@tonic-gate { 3050Sstevel@tonic-gate if (!debugging_mem) 3060Sstevel@tonic-gate return; 3070Sstevel@tonic-gate 3080Sstevel@tonic-gate printf("memseg\n"); 3090Sstevel@tonic-gate 3100Sstevel@tonic-gate while (memseg) { 3110Sstevel@tonic-gate prom_printf("\tpage = 0x%p, epage = 0x%p, " 3120Sstevel@tonic-gate "pfn = 0x%x, epfn = 0x%x\n", 3130Sstevel@tonic-gate memseg->pages, memseg->epages, 3140Sstevel@tonic-gate memseg->pages_base, memseg->pages_end); 3150Sstevel@tonic-gate memseg = memseg->next; 3160Sstevel@tonic-gate } 3170Sstevel@tonic-gate } 3180Sstevel@tonic-gate 3190Sstevel@tonic-gate #define debug_pause(str) halt((str)) 3200Sstevel@tonic-gate #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 3210Sstevel@tonic-gate #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 3220Sstevel@tonic-gate #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 3230Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) \ 3240Sstevel@tonic-gate if (debugging_mem) prom_printf((str), (a), (b), (c)) 3250Sstevel@tonic-gate #else /* DEBUGGING_MEM */ 3260Sstevel@tonic-gate #define MPRINTF(str) 3270Sstevel@tonic-gate #define MPRINTF1(str, a) 3280Sstevel@tonic-gate #define MPRINTF2(str, a, b) 3290Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) 3300Sstevel@tonic-gate #endif /* DEBUGGING_MEM */ 3310Sstevel@tonic-gate 3320Sstevel@tonic-gate /* Simple message to indicate that the bootops pointer has been zeroed */ 3330Sstevel@tonic-gate #ifdef DEBUG 3340Sstevel@tonic-gate static int bootops_gone_on = 0; 3350Sstevel@tonic-gate #define BOOTOPS_GONE() \ 3360Sstevel@tonic-gate if (bootops_gone_on) \ 3370Sstevel@tonic-gate prom_printf("The bootops vec is zeroed now!\n"); 3380Sstevel@tonic-gate #else 3390Sstevel@tonic-gate #define BOOTOPS_GONE() 3400Sstevel@tonic-gate #endif /* DEBUG */ 3410Sstevel@tonic-gate 3420Sstevel@tonic-gate /* 3430Sstevel@tonic-gate * Monitor pages may not be where this says they are. 3440Sstevel@tonic-gate * and the debugger may not be there either. 3450Sstevel@tonic-gate * 3460Sstevel@tonic-gate * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 3470Sstevel@tonic-gate * 3480Sstevel@tonic-gate * Physical memory layout 3490Sstevel@tonic-gate * (not necessarily contiguous) 3500Sstevel@tonic-gate * (THIS IS SOMEWHAT WRONG) 3510Sstevel@tonic-gate * /-----------------------\ 3520Sstevel@tonic-gate * | monitor pages | 3530Sstevel@tonic-gate * availmem -|-----------------------| 3540Sstevel@tonic-gate * | | 3550Sstevel@tonic-gate * | page pool | 3560Sstevel@tonic-gate * | | 3570Sstevel@tonic-gate * |-----------------------| 3580Sstevel@tonic-gate * | configured tables | 3590Sstevel@tonic-gate * | buffers | 3600Sstevel@tonic-gate * firstaddr -|-----------------------| 3610Sstevel@tonic-gate * | hat data structures | 3620Sstevel@tonic-gate * |-----------------------| 3630Sstevel@tonic-gate * | kernel data, bss | 3640Sstevel@tonic-gate * |-----------------------| 3650Sstevel@tonic-gate * | interrupt stack | 3660Sstevel@tonic-gate * |-----------------------| 3670Sstevel@tonic-gate * | kernel text (RO) | 3680Sstevel@tonic-gate * |-----------------------| 3690Sstevel@tonic-gate * | trap table (4k) | 3700Sstevel@tonic-gate * |-----------------------| 3710Sstevel@tonic-gate * page 1 | panicbuf | 3720Sstevel@tonic-gate * |-----------------------| 3730Sstevel@tonic-gate * page 0 | reclaimed | 3740Sstevel@tonic-gate * |_______________________| 3750Sstevel@tonic-gate * 3760Sstevel@tonic-gate * 3770Sstevel@tonic-gate * 3780Sstevel@tonic-gate * Kernel's Virtual Memory Layout. 3790Sstevel@tonic-gate * /-----------------------\ 3800Sstevel@tonic-gate * 0xFFFFFFFF.FFFFFFFF -| |- 3810Sstevel@tonic-gate * | OBP's virtual page | 3820Sstevel@tonic-gate * | tables | 3830Sstevel@tonic-gate * 0xFFFFFFFC.00000000 -|-----------------------|- 3840Sstevel@tonic-gate * : : 3850Sstevel@tonic-gate * : : 3863290Sjohansen * -|-----------------------|- 3873290Sjohansen * | segzio | (base and size vary) 3880Sstevel@tonic-gate * 0xFFFFFE00.00000000 -|-----------------------|- 3890Sstevel@tonic-gate * | | Ultrasparc I/II support 3900Sstevel@tonic-gate * | segkpm segment | up to 2TB of physical 3910Sstevel@tonic-gate * | (64-bit kernel ONLY) | memory, VAC has 2 colors 3920Sstevel@tonic-gate * | | 3930Sstevel@tonic-gate * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 3940Sstevel@tonic-gate * : : 3950Sstevel@tonic-gate * : : 3960Sstevel@tonic-gate * 0xFFFFF810.00000000 -|-----------------------|- hole_end 3970Sstevel@tonic-gate * | | ^ 3980Sstevel@tonic-gate * | UltraSPARC I/II call | | 3990Sstevel@tonic-gate * | bug requires an extra | | 4000Sstevel@tonic-gate * | 4 GB of space between | | 4010Sstevel@tonic-gate * | hole and used RAM | | 4020Sstevel@tonic-gate * | | | 4030Sstevel@tonic-gate * 0xFFFFF800.00000000 -|-----------------------|- | 4040Sstevel@tonic-gate * | | | 4050Sstevel@tonic-gate * | Virtual Address Hole | UltraSPARC 4060Sstevel@tonic-gate * | on UltraSPARC I/II | I/II * ONLY * 4070Sstevel@tonic-gate * | | | 4080Sstevel@tonic-gate * 0x00000800.00000000 -|-----------------------|- | 4090Sstevel@tonic-gate * | | | 4100Sstevel@tonic-gate * | UltraSPARC I/II call | | 4110Sstevel@tonic-gate * | bug requires an extra | | 4120Sstevel@tonic-gate * | 4 GB of space between | | 4130Sstevel@tonic-gate * | hole and used RAM | | 4140Sstevel@tonic-gate * | | v 4150Sstevel@tonic-gate * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 4160Sstevel@tonic-gate * : : ^ 4170Sstevel@tonic-gate * : : | 4180Sstevel@tonic-gate * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 4190Sstevel@tonic-gate * | | | 4200Sstevel@tonic-gate * | 64-bit kernel ONLY | | 4210Sstevel@tonic-gate * | | | 4220Sstevel@tonic-gate * | kmem64 segment | | 4230Sstevel@tonic-gate * | | | 4240Sstevel@tonic-gate * | (Relocated extra HME | Approximately 4250Sstevel@tonic-gate * | block allocations, | 1 TB of virtual 4260Sstevel@tonic-gate * | memnode freelists, | address space 4270Sstevel@tonic-gate * | HME hash buckets, | | 4280Sstevel@tonic-gate * | mml_table, kpmp_table,| | 4290Sstevel@tonic-gate * | page_t array and | | 4300Sstevel@tonic-gate * | hashblock pool to | | 4310Sstevel@tonic-gate * | avoid hard-coded | | 4320Sstevel@tonic-gate * | 32-bit vaddr | | 4330Sstevel@tonic-gate * | limitations) | | 4340Sstevel@tonic-gate * | | v 4350Sstevel@tonic-gate * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 4360Sstevel@tonic-gate * | | 4370Sstevel@tonic-gate * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 4380Sstevel@tonic-gate * | | 4390Sstevel@tonic-gate * 0x00000300.00000000 -|-----------------------|- SYSBASE 4400Sstevel@tonic-gate * : : 4410Sstevel@tonic-gate * : : 4420Sstevel@tonic-gate * -|-----------------------|- 4430Sstevel@tonic-gate * | | 4440Sstevel@tonic-gate * | segmap segment | SEGMAPSIZE (1/8th physmem, 4450Sstevel@tonic-gate * | | 256G MAX) 4460Sstevel@tonic-gate * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 4470Sstevel@tonic-gate * : : 4480Sstevel@tonic-gate * : : 4490Sstevel@tonic-gate * -|-----------------------|- 4500Sstevel@tonic-gate * | | 4510Sstevel@tonic-gate * | segkp | SEGKPSIZE (2GB) 4520Sstevel@tonic-gate * | | 4530Sstevel@tonic-gate * | | 4540Sstevel@tonic-gate * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 4550Sstevel@tonic-gate * | | 4560Sstevel@tonic-gate * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 4570Sstevel@tonic-gate * | | (SEGKPBASE - 0x400000) 4580Sstevel@tonic-gate * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 4590Sstevel@tonic-gate * | | (MEMSCRUBBASE - NCARGS) 4600Sstevel@tonic-gate * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 4610Sstevel@tonic-gate * | | (ARGSBASE - PPMAPSIZE) 4620Sstevel@tonic-gate * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 4630Sstevel@tonic-gate * | | 4640Sstevel@tonic-gate * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 4650Sstevel@tonic-gate * | | 4660Sstevel@tonic-gate * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 4670Sstevel@tonic-gate * : : 4680Sstevel@tonic-gate * : : 4690Sstevel@tonic-gate * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 4700Sstevel@tonic-gate * | | 4710Sstevel@tonic-gate * | OBP | 4720Sstevel@tonic-gate * | | 4730Sstevel@tonic-gate * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 4740Sstevel@tonic-gate * | kmdb | 4750Sstevel@tonic-gate * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 4760Sstevel@tonic-gate * : : 4770Sstevel@tonic-gate * : : 4780Sstevel@tonic-gate * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 4790Sstevel@tonic-gate * | | 4800Sstevel@tonic-gate * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 4810Sstevel@tonic-gate * | | ~64MB) 4820Sstevel@tonic-gate * 0x00000000.78002000 -|-----------------------| 4830Sstevel@tonic-gate * | panicbuf | 4840Sstevel@tonic-gate * 0x00000000.78000000 -|-----------------------|- SYSBASE32 4850Sstevel@tonic-gate * : : 4860Sstevel@tonic-gate * : : 4870Sstevel@tonic-gate * | | 4880Sstevel@tonic-gate * |-----------------------|- econtig32 4890Sstevel@tonic-gate * | vm structures | 4900Sstevel@tonic-gate * 0x00000000.01C00000 |-----------------------|- nalloc_end 4910Sstevel@tonic-gate * | TSBs | 4920Sstevel@tonic-gate * |-----------------------|- end/nalloc_base 4930Sstevel@tonic-gate * | kernel data & bss | 4940Sstevel@tonic-gate * 0x00000000.01800000 -|-----------------------| 4950Sstevel@tonic-gate * : nucleus text hole : 4960Sstevel@tonic-gate * 0x00000000.01400000 -|-----------------------| 4970Sstevel@tonic-gate * : : 4980Sstevel@tonic-gate * |-----------------------| 4990Sstevel@tonic-gate * | module text | 5000Sstevel@tonic-gate * |-----------------------|- e_text/modtext 5010Sstevel@tonic-gate * | kernel text | 5020Sstevel@tonic-gate * |-----------------------| 5030Sstevel@tonic-gate * | trap table (48k) | 5040Sstevel@tonic-gate * 0x00000000.01000000 -|-----------------------|- KERNELBASE 5050Sstevel@tonic-gate * | reserved for trapstat |} TSTAT_TOTAL_SIZE 5060Sstevel@tonic-gate * |-----------------------| 5070Sstevel@tonic-gate * | | 5080Sstevel@tonic-gate * | invalid | 5090Sstevel@tonic-gate * | | 5100Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 5110Sstevel@tonic-gate * 5120Sstevel@tonic-gate * 5130Sstevel@tonic-gate * 5140Sstevel@tonic-gate * 32-bit User Virtual Memory Layout. 5150Sstevel@tonic-gate * /-----------------------\ 5160Sstevel@tonic-gate * | | 5170Sstevel@tonic-gate * | invalid | 5180Sstevel@tonic-gate * | | 5190Sstevel@tonic-gate * 0xFFC00000 -|-----------------------|- USERLIMIT 5200Sstevel@tonic-gate * | user stack | 5210Sstevel@tonic-gate * : : 5220Sstevel@tonic-gate * : : 5230Sstevel@tonic-gate * : : 5240Sstevel@tonic-gate * | user data | 5250Sstevel@tonic-gate * -|-----------------------|- 5260Sstevel@tonic-gate * | user text | 5270Sstevel@tonic-gate * 0x00002000 -|-----------------------|- 5280Sstevel@tonic-gate * | invalid | 5290Sstevel@tonic-gate * 0x00000000 _|_______________________| 5300Sstevel@tonic-gate * 5310Sstevel@tonic-gate * 5320Sstevel@tonic-gate * 5330Sstevel@tonic-gate * 64-bit User Virtual Memory Layout. 5340Sstevel@tonic-gate * /-----------------------\ 5350Sstevel@tonic-gate * | | 5360Sstevel@tonic-gate * | invalid | 5370Sstevel@tonic-gate * | | 5380Sstevel@tonic-gate * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 5390Sstevel@tonic-gate * | user stack | 5400Sstevel@tonic-gate * : : 5410Sstevel@tonic-gate * : : 5420Sstevel@tonic-gate * : : 5430Sstevel@tonic-gate * | user data | 5440Sstevel@tonic-gate * -|-----------------------|- 5450Sstevel@tonic-gate * | user text | 5460Sstevel@tonic-gate * 0x00000000.00100000 -|-----------------------|- 5470Sstevel@tonic-gate * | invalid | 5480Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 5490Sstevel@tonic-gate */ 5500Sstevel@tonic-gate 5510Sstevel@tonic-gate extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 5520Sstevel@tonic-gate extern uint64_t ecache_flush_address(void); 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate #pragma weak load_platform_modules 5551772Sjl139090 #pragma weak plat_startup_memlist 5560Sstevel@tonic-gate #pragma weak ecache_init_scrub_flush_area 5570Sstevel@tonic-gate #pragma weak ecache_flush_address 5580Sstevel@tonic-gate 5590Sstevel@tonic-gate 5600Sstevel@tonic-gate /* 5610Sstevel@tonic-gate * By default the DR Cage is enabled for maximum OS 5620Sstevel@tonic-gate * MPSS performance. Users needing to disable the cage mechanism 5630Sstevel@tonic-gate * can set this variable to zero via /etc/system. 5640Sstevel@tonic-gate * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 5650Sstevel@tonic-gate * will result in loss of DR functionality. 5660Sstevel@tonic-gate * Platforms wishing to disable kernel Cage by default 5670Sstevel@tonic-gate * should do so in their set_platform_defaults() routine. 5680Sstevel@tonic-gate */ 5690Sstevel@tonic-gate int kernel_cage_enable = 1; 5700Sstevel@tonic-gate 5710Sstevel@tonic-gate static void 5720Sstevel@tonic-gate setup_cage_params(void) 5730Sstevel@tonic-gate { 5740Sstevel@tonic-gate void (*func)(void); 5750Sstevel@tonic-gate 5760Sstevel@tonic-gate func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 5770Sstevel@tonic-gate if (func != NULL) { 5780Sstevel@tonic-gate (*func)(); 5790Sstevel@tonic-gate return; 5800Sstevel@tonic-gate } 5810Sstevel@tonic-gate 5820Sstevel@tonic-gate if (kernel_cage_enable == 0) { 5830Sstevel@tonic-gate return; 5840Sstevel@tonic-gate } 5850Sstevel@tonic-gate kcage_range_lock(); 5860Sstevel@tonic-gate if (kcage_range_init(phys_avail, 1) == 0) { 5870Sstevel@tonic-gate kcage_init(total_pages / 256); 5880Sstevel@tonic-gate } 5890Sstevel@tonic-gate kcage_range_unlock(); 5900Sstevel@tonic-gate 5910Sstevel@tonic-gate if (kcage_on) { 5920Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 5930Sstevel@tonic-gate } else { 5940Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 5950Sstevel@tonic-gate } 5960Sstevel@tonic-gate 5970Sstevel@tonic-gate } 5980Sstevel@tonic-gate 5990Sstevel@tonic-gate /* 6000Sstevel@tonic-gate * Machine-dependent startup code 6010Sstevel@tonic-gate */ 6020Sstevel@tonic-gate void 6030Sstevel@tonic-gate startup(void) 6040Sstevel@tonic-gate { 6050Sstevel@tonic-gate startup_init(); 6060Sstevel@tonic-gate if (&startup_platform) 6070Sstevel@tonic-gate startup_platform(); 6080Sstevel@tonic-gate startup_memlist(); 6090Sstevel@tonic-gate startup_modules(); 6100Sstevel@tonic-gate setup_cage_params(); 6110Sstevel@tonic-gate startup_bop_gone(); 6120Sstevel@tonic-gate startup_vm(); 6130Sstevel@tonic-gate startup_end(); 6140Sstevel@tonic-gate } 6150Sstevel@tonic-gate 6160Sstevel@tonic-gate struct regs sync_reg_buf; 6170Sstevel@tonic-gate uint64_t sync_tt; 6180Sstevel@tonic-gate 6190Sstevel@tonic-gate void 6200Sstevel@tonic-gate sync_handler(void) 6210Sstevel@tonic-gate { 6220Sstevel@tonic-gate struct trap_info ti; 6230Sstevel@tonic-gate int i; 6240Sstevel@tonic-gate 6250Sstevel@tonic-gate /* 6260Sstevel@tonic-gate * Prevent trying to talk to the other CPUs since they are 6270Sstevel@tonic-gate * sitting in the prom and won't reply. 6280Sstevel@tonic-gate */ 6290Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 6300Sstevel@tonic-gate if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 6310Sstevel@tonic-gate cpu[i]->cpu_flags &= ~CPU_READY; 6320Sstevel@tonic-gate cpu[i]->cpu_flags |= CPU_QUIESCED; 6330Sstevel@tonic-gate CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 6340Sstevel@tonic-gate } 6350Sstevel@tonic-gate } 6360Sstevel@tonic-gate 6370Sstevel@tonic-gate /* 6380Sstevel@tonic-gate * We've managed to get here without going through the 6390Sstevel@tonic-gate * normal panic code path. Try and save some useful 6400Sstevel@tonic-gate * information. 6410Sstevel@tonic-gate */ 6420Sstevel@tonic-gate if (!panicstr && (curthread->t_panic_trap == NULL)) { 6430Sstevel@tonic-gate ti.trap_type = sync_tt; 6440Sstevel@tonic-gate ti.trap_regs = &sync_reg_buf; 6450Sstevel@tonic-gate ti.trap_addr = NULL; 6460Sstevel@tonic-gate ti.trap_mmu_fsr = 0x0; 6470Sstevel@tonic-gate 6480Sstevel@tonic-gate curthread->t_panic_trap = &ti; 6490Sstevel@tonic-gate } 6500Sstevel@tonic-gate 6510Sstevel@tonic-gate /* 6520Sstevel@tonic-gate * If we're re-entering the panic path, update the signature 6530Sstevel@tonic-gate * block so that the SC knows we're in the second part of panic. 6540Sstevel@tonic-gate */ 6550Sstevel@tonic-gate if (panicstr) 6560Sstevel@tonic-gate CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 6570Sstevel@tonic-gate 6580Sstevel@tonic-gate nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 6590Sstevel@tonic-gate panic("sync initiated"); 6600Sstevel@tonic-gate } 6610Sstevel@tonic-gate 6620Sstevel@tonic-gate 6630Sstevel@tonic-gate static void 6640Sstevel@tonic-gate startup_init(void) 6650Sstevel@tonic-gate { 6660Sstevel@tonic-gate /* 6670Sstevel@tonic-gate * We want to save the registers while we're still in OBP 6680Sstevel@tonic-gate * so that we know they haven't been fiddled with since. 6690Sstevel@tonic-gate * (In principle, OBP can't change them just because it 6700Sstevel@tonic-gate * makes a callback, but we'd rather not depend on that 6710Sstevel@tonic-gate * behavior.) 6720Sstevel@tonic-gate */ 6730Sstevel@tonic-gate char sync_str[] = 6740Sstevel@tonic-gate "warning @ warning off : sync " 6750Sstevel@tonic-gate "%%tl-c %%tstate h# %p x! " 6760Sstevel@tonic-gate "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 6770Sstevel@tonic-gate "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 6780Sstevel@tonic-gate "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 6790Sstevel@tonic-gate "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 6800Sstevel@tonic-gate "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 6810Sstevel@tonic-gate "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 6820Sstevel@tonic-gate "%%y h# %p l! %%tl-c %%tt h# %p x! " 6830Sstevel@tonic-gate "sync ; warning !"; 6840Sstevel@tonic-gate 6850Sstevel@tonic-gate /* 6860Sstevel@tonic-gate * 20 == num of %p substrings 6870Sstevel@tonic-gate * 16 == max num of chars %p will expand to. 6880Sstevel@tonic-gate */ 6890Sstevel@tonic-gate char bp[sizeof (sync_str) + 16 * 20]; 6900Sstevel@tonic-gate 6910Sstevel@tonic-gate (void) check_boot_version(BOP_GETVERSION(bootops)); 6920Sstevel@tonic-gate 6930Sstevel@tonic-gate /* 6940Sstevel@tonic-gate * Initialize ptl1 stack for the 1st CPU. 6950Sstevel@tonic-gate */ 6960Sstevel@tonic-gate ptl1_init_cpu(&cpu0); 6970Sstevel@tonic-gate 6980Sstevel@tonic-gate /* 6990Sstevel@tonic-gate * Initialize the address map for cache consistent mappings 7000Sstevel@tonic-gate * to random pages; must be done after vac_size is set. 7010Sstevel@tonic-gate */ 7020Sstevel@tonic-gate ppmapinit(); 7030Sstevel@tonic-gate 7040Sstevel@tonic-gate /* 7050Sstevel@tonic-gate * Initialize the PROM callback handler. 7060Sstevel@tonic-gate */ 7070Sstevel@tonic-gate init_vx_handler(); 7080Sstevel@tonic-gate 7090Sstevel@tonic-gate /* 7100Sstevel@tonic-gate * have prom call sync_callback() to handle the sync and 7110Sstevel@tonic-gate * save some useful information which will be stored in the 7120Sstevel@tonic-gate * core file later. 7130Sstevel@tonic-gate */ 7140Sstevel@tonic-gate (void) sprintf((char *)bp, sync_str, 7150Sstevel@tonic-gate (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 7160Sstevel@tonic-gate (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 7170Sstevel@tonic-gate (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 7180Sstevel@tonic-gate (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 7190Sstevel@tonic-gate (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 7200Sstevel@tonic-gate (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 7210Sstevel@tonic-gate (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 7220Sstevel@tonic-gate (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 7230Sstevel@tonic-gate (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 7240Sstevel@tonic-gate (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 7250Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 7260Sstevel@tonic-gate add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 7270Sstevel@tonic-gate } 7280Sstevel@tonic-gate 7290Sstevel@tonic-gate static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 7300Sstevel@tonic-gate static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 7310Sstevel@tonic-gate 7322973Sgovinda #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 7332973Sgovinda (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 7342973Sgovinda (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 7350Sstevel@tonic-gate 7360Sstevel@tonic-gate /* 7370Sstevel@tonic-gate * As OBP takes up some RAM when the system boots, pages will already be "lost" 7380Sstevel@tonic-gate * to the system and reflected in npages by the time we see it. 7390Sstevel@tonic-gate * 7400Sstevel@tonic-gate * We only want to allocate kernel structures in the 64-bit virtual address 7410Sstevel@tonic-gate * space on systems with enough RAM to make the overhead of keeping track of 7420Sstevel@tonic-gate * an extra kernel memory segment worthwhile. 7430Sstevel@tonic-gate * 7440Sstevel@tonic-gate * Since OBP has already performed its memory allocations by this point, if we 7450Sstevel@tonic-gate * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 7460Sstevel@tonic-gate * memory in the 64-bit virtual address space; otherwise keep allocations 7470Sstevel@tonic-gate * contiguous with we've mapped so far in the 32-bit virtual address space. 7480Sstevel@tonic-gate */ 7490Sstevel@tonic-gate #define MINMOVE_RAM_MB ((size_t)1900) 7500Sstevel@tonic-gate #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 7510Sstevel@tonic-gate 7520Sstevel@tonic-gate pgcnt_t tune_npages = (pgcnt_t) 7530Sstevel@tonic-gate (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 7540Sstevel@tonic-gate 7550Sstevel@tonic-gate static void 7560Sstevel@tonic-gate startup_memlist(void) 7570Sstevel@tonic-gate { 7580Sstevel@tonic-gate size_t alloc_sz; 7590Sstevel@tonic-gate size_t ctrs_sz; 7600Sstevel@tonic-gate caddr_t alloc_base; 7610Sstevel@tonic-gate caddr_t ctrs_base, ctrs_end; 7620Sstevel@tonic-gate caddr_t memspace; 7630Sstevel@tonic-gate caddr_t va; 7640Sstevel@tonic-gate int memblocks = 0; 7650Sstevel@tonic-gate struct memlist *cur; 7660Sstevel@tonic-gate size_t syslimit = (size_t)SYSLIMIT; 7670Sstevel@tonic-gate size_t sysbase = (size_t)SYSBASE; 7680Sstevel@tonic-gate int alloc_alignsize = MMU_PAGESIZE; 7690Sstevel@tonic-gate extern void page_coloring_init(void); 770*3717Sdp78419 extern void page_set_colorequiv_arr(void); 7710Sstevel@tonic-gate 7720Sstevel@tonic-gate /* 7730Sstevel@tonic-gate * Initialize enough of the system to allow kmem_alloc to work by 7740Sstevel@tonic-gate * calling boot to allocate its memory until the time that 7750Sstevel@tonic-gate * kvm_init is completed. The page structs are allocated after 7760Sstevel@tonic-gate * rounding up end to the nearest page boundary; the memsegs are 7770Sstevel@tonic-gate * initialized and the space they use comes from the kernel heap. 7780Sstevel@tonic-gate * With appropriate initialization, they can be reallocated later 7790Sstevel@tonic-gate * to a size appropriate for the machine's configuration. 7800Sstevel@tonic-gate * 7810Sstevel@tonic-gate * At this point, memory is allocated for things that will never 7820Sstevel@tonic-gate * need to be freed, this used to be "valloced". This allows a 7830Sstevel@tonic-gate * savings as the pages don't need page structures to describe 7840Sstevel@tonic-gate * them because them will not be managed by the vm system. 7850Sstevel@tonic-gate */ 7860Sstevel@tonic-gate 7870Sstevel@tonic-gate /* 7880Sstevel@tonic-gate * We're loaded by boot with the following configuration (as 7890Sstevel@tonic-gate * specified in the sun4u/conf/Mapfile): 7900Sstevel@tonic-gate * 7910Sstevel@tonic-gate * text: 4 MB chunk aligned on a 4MB boundary 7920Sstevel@tonic-gate * data & bss: 4 MB chunk aligned on a 4MB boundary 7930Sstevel@tonic-gate * 7940Sstevel@tonic-gate * These two chunks will eventually be mapped by 2 locked 4MB 7950Sstevel@tonic-gate * ttes and will represent the nucleus of the kernel. This gives 7960Sstevel@tonic-gate * us some free space that is already allocated, some or all of 7970Sstevel@tonic-gate * which is made available to kernel module text. 7980Sstevel@tonic-gate * 7990Sstevel@tonic-gate * The free space in the data-bss chunk is used for nucleus 8000Sstevel@tonic-gate * allocatable data structures and we reserve it using the 8010Sstevel@tonic-gate * nalloc_base and nalloc_end variables. This space is currently 8020Sstevel@tonic-gate * being used for hat data structures required for tlb miss 8030Sstevel@tonic-gate * handling operations. We align nalloc_base to a l2 cache 8040Sstevel@tonic-gate * linesize because this is the line size the hardware uses to 8050Sstevel@tonic-gate * maintain cache coherency. 8060Sstevel@tonic-gate * 256K is carved out for module data. 8070Sstevel@tonic-gate */ 8080Sstevel@tonic-gate 8090Sstevel@tonic-gate nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 8100Sstevel@tonic-gate moddata = nalloc_base; 8110Sstevel@tonic-gate e_moddata = nalloc_base + MODDATA; 8120Sstevel@tonic-gate nalloc_base = e_moddata; 8130Sstevel@tonic-gate 8140Sstevel@tonic-gate nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 8150Sstevel@tonic-gate valloc_base = nalloc_base; 8160Sstevel@tonic-gate 8170Sstevel@tonic-gate /* 8180Sstevel@tonic-gate * Calculate the start of the data segment. 8190Sstevel@tonic-gate */ 8200Sstevel@tonic-gate sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 8210Sstevel@tonic-gate 8220Sstevel@tonic-gate PRM_DEBUG(moddata); 8230Sstevel@tonic-gate PRM_DEBUG(nalloc_base); 8240Sstevel@tonic-gate PRM_DEBUG(nalloc_end); 8250Sstevel@tonic-gate PRM_DEBUG(sdata); 8260Sstevel@tonic-gate 8270Sstevel@tonic-gate /* 8280Sstevel@tonic-gate * Remember any slop after e_text so we can give it to the modules. 8290Sstevel@tonic-gate */ 8300Sstevel@tonic-gate PRM_DEBUG(e_text); 8310Sstevel@tonic-gate modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 8320Sstevel@tonic-gate if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 8330Sstevel@tonic-gate panic("nucleus text overflow"); 8340Sstevel@tonic-gate modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 8350Sstevel@tonic-gate modtext; 8360Sstevel@tonic-gate PRM_DEBUG(modtext); 8370Sstevel@tonic-gate PRM_DEBUG(modtext_sz); 8380Sstevel@tonic-gate 8390Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 8400Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 8410Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 8420Sstevel@tonic-gate /* 8430Sstevel@tonic-gate * Remember what the physically available highest page is 8440Sstevel@tonic-gate * so that dumpsys works properly, and find out how much 8450Sstevel@tonic-gate * memory is installed. 8460Sstevel@tonic-gate */ 8470Sstevel@tonic-gate installed_top_size_memlist_array(boot_physinstalled, 8480Sstevel@tonic-gate boot_physinstalled_len, &physmax, &physinstalled); 8490Sstevel@tonic-gate PRM_DEBUG(physinstalled); 8500Sstevel@tonic-gate PRM_DEBUG(physmax); 8510Sstevel@tonic-gate 8520Sstevel@tonic-gate /* Fill out memory nodes config structure */ 8530Sstevel@tonic-gate startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 8540Sstevel@tonic-gate 8550Sstevel@tonic-gate /* 8560Sstevel@tonic-gate * Get the list of physically available memory to size 8570Sstevel@tonic-gate * the number of page structures needed. 8580Sstevel@tonic-gate */ 8590Sstevel@tonic-gate size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 8600Sstevel@tonic-gate /* 8610Sstevel@tonic-gate * This first snap shot of npages can represent the pages used 8620Sstevel@tonic-gate * by OBP's text and data approximately. This is used in the 8630Sstevel@tonic-gate * the calculation of the kernel size 8640Sstevel@tonic-gate */ 8650Sstevel@tonic-gate obp_pages = physinstalled - npages; 8660Sstevel@tonic-gate 8670Sstevel@tonic-gate 8680Sstevel@tonic-gate /* 8690Sstevel@tonic-gate * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 8700Sstevel@tonic-gate * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 8710Sstevel@tonic-gate * 2MB) and any excess pages are put on physavail. The assumption is 8720Sstevel@tonic-gate * that small-memory systems will need more pages more than they'll 8730Sstevel@tonic-gate * need efficiently-mapped module texts. 8740Sstevel@tonic-gate */ 8750Sstevel@tonic-gate if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 8760Sstevel@tonic-gate modtext_sz > MODTEXT_SM_CAP) { 8770Sstevel@tonic-gate extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 8780Sstevel@tonic-gate modtext_sz = MODTEXT_SM_CAP; 8790Sstevel@tonic-gate } else 8800Sstevel@tonic-gate extra_etpg = 0; 8810Sstevel@tonic-gate PRM_DEBUG(extra_etpg); 8820Sstevel@tonic-gate PRM_DEBUG(modtext_sz); 8830Sstevel@tonic-gate extra_etva = modtext + modtext_sz; 8840Sstevel@tonic-gate PRM_DEBUG(extra_etva); 8850Sstevel@tonic-gate 8860Sstevel@tonic-gate /* 8870Sstevel@tonic-gate * Account for any pages after e_text and e_data. 8880Sstevel@tonic-gate */ 8890Sstevel@tonic-gate npages += extra_etpg; 8900Sstevel@tonic-gate npages += mmu_btopr(nalloc_end - nalloc_base); 8910Sstevel@tonic-gate PRM_DEBUG(npages); 8920Sstevel@tonic-gate 8930Sstevel@tonic-gate /* 8940Sstevel@tonic-gate * npages is the maximum of available physical memory possible. 8950Sstevel@tonic-gate * (ie. it will never be more than this) 8960Sstevel@tonic-gate */ 8970Sstevel@tonic-gate 8980Sstevel@tonic-gate /* 8990Sstevel@tonic-gate * initialize the nucleus memory allocator. 9000Sstevel@tonic-gate */ 9010Sstevel@tonic-gate ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 9020Sstevel@tonic-gate 9030Sstevel@tonic-gate /* 9040Sstevel@tonic-gate * Allocate mmu fault status area from the nucleus data area. 9050Sstevel@tonic-gate */ 9060Sstevel@tonic-gate if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 9070Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 9080Sstevel@tonic-gate 9090Sstevel@tonic-gate /* 9100Sstevel@tonic-gate * Allocate kernel TSBs from the nucleus data area. 9110Sstevel@tonic-gate */ 9120Sstevel@tonic-gate if (ndata_alloc_tsbs(&ndata, npages) != 0) 9130Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 9140Sstevel@tonic-gate 9150Sstevel@tonic-gate /* 9160Sstevel@tonic-gate * Allocate dmv dispatch table from the nucleus data area. 9170Sstevel@tonic-gate */ 9180Sstevel@tonic-gate if (ndata_alloc_dmv(&ndata) != 0) 9190Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 9200Sstevel@tonic-gate 9210Sstevel@tonic-gate 9220Sstevel@tonic-gate page_coloring_init(); 9230Sstevel@tonic-gate 9240Sstevel@tonic-gate /* 9250Sstevel@tonic-gate * Allocate page_freelists bin headers for memnode 0 from the 9260Sstevel@tonic-gate * nucleus data area. 9270Sstevel@tonic-gate */ 9280Sstevel@tonic-gate if (ndata_alloc_page_freelists(&ndata, 0) != 0) 9290Sstevel@tonic-gate cmn_err(CE_PANIC, 9300Sstevel@tonic-gate "no more nucleus memory after page free lists alloc"); 9310Sstevel@tonic-gate 9320Sstevel@tonic-gate if (kpm_enable) { 9330Sstevel@tonic-gate kpm_init(); 9340Sstevel@tonic-gate /* 9350Sstevel@tonic-gate * kpm page space -- Update kpm_npages and make the 9360Sstevel@tonic-gate * same assumption about fragmenting as it is done 9370Sstevel@tonic-gate * for memseg_sz. 9380Sstevel@tonic-gate */ 9390Sstevel@tonic-gate kpm_npages_setup(memblocks + 4); 9400Sstevel@tonic-gate } 9410Sstevel@tonic-gate 9420Sstevel@tonic-gate /* 9430Sstevel@tonic-gate * Allocate hat related structs from the nucleus data area. 9440Sstevel@tonic-gate */ 9450Sstevel@tonic-gate if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 9460Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 9470Sstevel@tonic-gate 9480Sstevel@tonic-gate /* 9490Sstevel@tonic-gate * We want to do the BOP_ALLOCs before the real allocation of page 9500Sstevel@tonic-gate * structs in order to not have to allocate page structs for this 9510Sstevel@tonic-gate * memory. We need to calculate a virtual address because we want 9520Sstevel@tonic-gate * the page structs to come before other allocations in virtual address 9530Sstevel@tonic-gate * space. This is so some (if not all) of page structs can actually 9540Sstevel@tonic-gate * live in the nucleus. 9550Sstevel@tonic-gate */ 9560Sstevel@tonic-gate 9570Sstevel@tonic-gate /* 9580Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 9590Sstevel@tonic-gate * 9600Sstevel@tonic-gate * There are comments all over the SFMMU code warning of dire 9610Sstevel@tonic-gate * consequences if the TSBs are moved out of 32-bit space. This 9620Sstevel@tonic-gate * is largely because the asm code uses "sethi %hi(addr)"-type 9630Sstevel@tonic-gate * instructions which will not provide the expected result if the 9640Sstevel@tonic-gate * address is a 64-bit one. 9650Sstevel@tonic-gate * 9660Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 9670Sstevel@tonic-gate */ 9680Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 9690Sstevel@tonic-gate alloc_base = sfmmu_ktsb_alloc(alloc_base); 9700Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 9710Sstevel@tonic-gate PRM_DEBUG(alloc_base); 9720Sstevel@tonic-gate 9730Sstevel@tonic-gate /* 9740Sstevel@tonic-gate * Allocate IOMMU TSB array. We do this here so that the physical 9750Sstevel@tonic-gate * memory gets deducted from the PROM's physical memory list. 9760Sstevel@tonic-gate */ 9770Sstevel@tonic-gate alloc_base = iommu_tsb_init(alloc_base); 9780Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 9790Sstevel@tonic-gate ecache_alignsize); 9800Sstevel@tonic-gate PRM_DEBUG(alloc_base); 9810Sstevel@tonic-gate 9820Sstevel@tonic-gate /* 9831772Sjl139090 * Platforms like Starcat and OPL need special structures assigned in 9841772Sjl139090 * 32-bit virtual address space because their probing routines execute 9851772Sjl139090 * FCode, and FCode can't handle 64-bit virtual addresses... 9860Sstevel@tonic-gate */ 9871772Sjl139090 if (&plat_startup_memlist) { 9881772Sjl139090 alloc_base = plat_startup_memlist(alloc_base); 9890Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 9900Sstevel@tonic-gate ecache_alignsize); 9910Sstevel@tonic-gate PRM_DEBUG(alloc_base); 9920Sstevel@tonic-gate } 9930Sstevel@tonic-gate 9940Sstevel@tonic-gate /* 9950Sstevel@tonic-gate * If we have enough memory, use 4M pages for alignment because it 9960Sstevel@tonic-gate * greatly reduces the number of TLB misses we take albeit at the cost 9970Sstevel@tonic-gate * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 9980Sstevel@tonic-gate * allocation.) Still, the speedup on large memory systems (e.g. > 64 9990Sstevel@tonic-gate * GB) is quite noticeable, so it is worth the effort to do if we can. 10000Sstevel@tonic-gate * 10010Sstevel@tonic-gate * Note, however, that this speedup will only occur if the boot PROM 10020Sstevel@tonic-gate * uses the largest possible MMU page size possible to map memory 10030Sstevel@tonic-gate * requests that are properly aligned and sized (for example, a request 10040Sstevel@tonic-gate * for a multiple of 4MB of memory aligned to a 4MB boundary will 10050Sstevel@tonic-gate * result in a mapping using a 4MB MMU page.) 10060Sstevel@tonic-gate * 10070Sstevel@tonic-gate * Even then, the large page mappings will only speed things up until 10080Sstevel@tonic-gate * the startup process proceeds a bit further, as when 10090Sstevel@tonic-gate * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 10100Sstevel@tonic-gate * kernel it remaps everything but the TSBs using 8K pages anyway... 10110Sstevel@tonic-gate * 10120Sstevel@tonic-gate * At some point in the future, sfmmu_map_prom_mappings() will be 10130Sstevel@tonic-gate * rewritten to copy memory mappings to the kernel using the same MMU 10140Sstevel@tonic-gate * page sizes the PROM used. When that occurs, if the PROM did use 10150Sstevel@tonic-gate * large MMU pages to map memory, the alignment/sizing work we're 10160Sstevel@tonic-gate * doing now should give us a nice extra performance boost, albeit at 10170Sstevel@tonic-gate * the cost of greater RAM usage... 10180Sstevel@tonic-gate */ 10190Sstevel@tonic-gate alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 10200Sstevel@tonic-gate MMU_PAGESIZE); 10210Sstevel@tonic-gate 10220Sstevel@tonic-gate PRM_DEBUG(tune_npages); 10230Sstevel@tonic-gate PRM_DEBUG(alloc_alignsize); 10240Sstevel@tonic-gate 10250Sstevel@tonic-gate /* 10260Sstevel@tonic-gate * Save off where the contiguous allocations to date have ended 10270Sstevel@tonic-gate * in econtig32. 10280Sstevel@tonic-gate */ 10290Sstevel@tonic-gate econtig32 = alloc_base; 10300Sstevel@tonic-gate PRM_DEBUG(econtig32); 10310Sstevel@tonic-gate 10320Sstevel@tonic-gate if (econtig32 > (caddr_t)KERNEL_LIMIT32) 10330Sstevel@tonic-gate cmn_err(CE_PANIC, "econtig32 too big"); 10340Sstevel@tonic-gate 10350Sstevel@tonic-gate /* 10360Sstevel@tonic-gate * To avoid memory allocation collisions in the 32-bit virtual address 10370Sstevel@tonic-gate * space, make allocations from this point forward in 64-bit virtual 10380Sstevel@tonic-gate * address space starting at syslimit and working up. Also use the 10390Sstevel@tonic-gate * alignment specified by alloc_alignsize, as we may be able to save 10400Sstevel@tonic-gate * ourselves TLB misses by using larger page sizes if they're 10410Sstevel@tonic-gate * available. 10420Sstevel@tonic-gate * 10430Sstevel@tonic-gate * All this is needed because on large memory systems, the default 10440Sstevel@tonic-gate * Solaris allocations will collide with SYSBASE32, which is hard 10450Sstevel@tonic-gate * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 10460Sstevel@tonic-gate * kernels, move the allocations to a location in the 64-bit virtual 10470Sstevel@tonic-gate * address space space, allowing those structures to grow without 10480Sstevel@tonic-gate * worry. 10490Sstevel@tonic-gate * 10500Sstevel@tonic-gate * On current CPUs we'll run out of physical memory address bits before 10510Sstevel@tonic-gate * we need to worry about the allocations running into anything else in 10520Sstevel@tonic-gate * VM or the virtual address holes on US-I and II, as there's currently 10530Sstevel@tonic-gate * about 1 TB of addressable space before the US-I/II VA hole. 10540Sstevel@tonic-gate */ 10550Sstevel@tonic-gate kmem64_base = (caddr_t)syslimit; 10560Sstevel@tonic-gate PRM_DEBUG(kmem64_base); 10570Sstevel@tonic-gate 10580Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 10590Sstevel@tonic-gate 10600Sstevel@tonic-gate /* 10610Sstevel@tonic-gate * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 10620Sstevel@tonic-gate * them here. 10630Sstevel@tonic-gate */ 10640Sstevel@tonic-gate if (khme_hash == NULL || uhme_hash == NULL) { 10650Sstevel@tonic-gate /* 10660Sstevel@tonic-gate * alloc_hme_buckets() will align alloc_base properly before 10670Sstevel@tonic-gate * assigning the hash buckets, so we don't need to do it 10680Sstevel@tonic-gate * before the call... 10690Sstevel@tonic-gate */ 10700Sstevel@tonic-gate alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 10710Sstevel@tonic-gate 10720Sstevel@tonic-gate PRM_DEBUG(alloc_base); 10730Sstevel@tonic-gate PRM_DEBUG(khme_hash); 10740Sstevel@tonic-gate PRM_DEBUG(uhme_hash); 10750Sstevel@tonic-gate } 10760Sstevel@tonic-gate 10770Sstevel@tonic-gate /* 10780Sstevel@tonic-gate * Allocate the remaining page freelists. NUMA systems can 10790Sstevel@tonic-gate * have lots of page freelists, one per node, which quickly 10800Sstevel@tonic-gate * outgrow the amount of nucleus memory available. 10810Sstevel@tonic-gate */ 10820Sstevel@tonic-gate if (max_mem_nodes > 1) { 10830Sstevel@tonic-gate int mnode; 10840Sstevel@tonic-gate caddr_t alloc_start = alloc_base; 10850Sstevel@tonic-gate 10860Sstevel@tonic-gate for (mnode = 1; mnode < max_mem_nodes; mnode++) { 10870Sstevel@tonic-gate alloc_base = alloc_page_freelists(mnode, alloc_base, 10880Sstevel@tonic-gate ecache_alignsize); 10890Sstevel@tonic-gate } 10900Sstevel@tonic-gate 10910Sstevel@tonic-gate if (alloc_base > alloc_start) { 10920Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 10930Sstevel@tonic-gate alloc_alignsize); 10940Sstevel@tonic-gate if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 10950Sstevel@tonic-gate alloc_base - alloc_start, 10960Sstevel@tonic-gate alloc_alignsize) != alloc_start) 10970Sstevel@tonic-gate cmn_err(CE_PANIC, 10980Sstevel@tonic-gate "Unable to alloc page freelists\n"); 10990Sstevel@tonic-gate } 11000Sstevel@tonic-gate 11010Sstevel@tonic-gate PRM_DEBUG(alloc_base); 11020Sstevel@tonic-gate } 11030Sstevel@tonic-gate 11040Sstevel@tonic-gate if (!mml_table) { 11050Sstevel@tonic-gate size_t mmltable_sz; 11060Sstevel@tonic-gate 11070Sstevel@tonic-gate /* 11080Sstevel@tonic-gate * We need to allocate the mml_table here because there 11090Sstevel@tonic-gate * was not enough space within the nucleus. 11100Sstevel@tonic-gate */ 11110Sstevel@tonic-gate mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 11120Sstevel@tonic-gate alloc_sz = roundup(mmltable_sz, alloc_alignsize); 11130Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 11140Sstevel@tonic-gate alloc_alignsize); 11150Sstevel@tonic-gate 11160Sstevel@tonic-gate if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 11170Sstevel@tonic-gate alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 11180Sstevel@tonic-gate panic("mml_table alloc failure"); 11190Sstevel@tonic-gate 11200Sstevel@tonic-gate alloc_base += alloc_sz; 11210Sstevel@tonic-gate PRM_DEBUG(mml_table); 11220Sstevel@tonic-gate PRM_DEBUG(alloc_base); 11230Sstevel@tonic-gate } 11240Sstevel@tonic-gate 11250Sstevel@tonic-gate if (kpm_enable && !(kpmp_table || kpmp_stable)) { 11260Sstevel@tonic-gate size_t kpmptable_sz; 11270Sstevel@tonic-gate caddr_t table; 11280Sstevel@tonic-gate 11290Sstevel@tonic-gate /* 11300Sstevel@tonic-gate * We need to allocate either kpmp_table or kpmp_stable here 11310Sstevel@tonic-gate * because there was not enough space within the nucleus. 11320Sstevel@tonic-gate */ 11330Sstevel@tonic-gate kpmptable_sz = (kpm_smallpages == 0) ? 11340Sstevel@tonic-gate sizeof (kpm_hlk_t) * kpmp_table_sz : 11350Sstevel@tonic-gate sizeof (kpm_shlk_t) * kpmp_stable_sz; 11360Sstevel@tonic-gate 11370Sstevel@tonic-gate alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 11380Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 11390Sstevel@tonic-gate alloc_alignsize); 11400Sstevel@tonic-gate 11410Sstevel@tonic-gate table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 11420Sstevel@tonic-gate alloc_alignsize); 11430Sstevel@tonic-gate 11440Sstevel@tonic-gate if (table != alloc_base) 11450Sstevel@tonic-gate panic("kpmp_table or kpmp_stable alloc failure"); 11460Sstevel@tonic-gate 11470Sstevel@tonic-gate if (kpm_smallpages == 0) { 11480Sstevel@tonic-gate kpmp_table = (kpm_hlk_t *)table; 11490Sstevel@tonic-gate PRM_DEBUG(kpmp_table); 11500Sstevel@tonic-gate } else { 11510Sstevel@tonic-gate kpmp_stable = (kpm_shlk_t *)table; 11520Sstevel@tonic-gate PRM_DEBUG(kpmp_stable); 11530Sstevel@tonic-gate } 11540Sstevel@tonic-gate 11550Sstevel@tonic-gate alloc_base += alloc_sz; 11560Sstevel@tonic-gate PRM_DEBUG(alloc_base); 11570Sstevel@tonic-gate } 11580Sstevel@tonic-gate 11590Sstevel@tonic-gate if (&ecache_init_scrub_flush_area) { 11600Sstevel@tonic-gate /* 11610Sstevel@tonic-gate * Pass alloc_base directly, as the routine itself is 11620Sstevel@tonic-gate * responsible for any special alignment requirements... 11630Sstevel@tonic-gate */ 11640Sstevel@tonic-gate alloc_base = ecache_init_scrub_flush_area(alloc_base); 11650Sstevel@tonic-gate PRM_DEBUG(alloc_base); 11660Sstevel@tonic-gate } 11670Sstevel@tonic-gate 11680Sstevel@tonic-gate /* 11690Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 11700Sstevel@tonic-gate */ 11710Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 11720Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 11730Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 11740Sstevel@tonic-gate 11750Sstevel@tonic-gate /* 11760Sstevel@tonic-gate * Reset npages and memblocks based on boot_physavail list. 11770Sstevel@tonic-gate */ 11780Sstevel@tonic-gate size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 11790Sstevel@tonic-gate PRM_DEBUG(npages); 11800Sstevel@tonic-gate 11810Sstevel@tonic-gate /* 11820Sstevel@tonic-gate * Account for extra memory after e_text. 11830Sstevel@tonic-gate */ 11840Sstevel@tonic-gate npages += extra_etpg; 11850Sstevel@tonic-gate 11860Sstevel@tonic-gate /* 11870Sstevel@tonic-gate * Calculate the largest free memory chunk in the nucleus data area. 11880Sstevel@tonic-gate * We need to figure out if page structs can fit in there or not. 11890Sstevel@tonic-gate * We also make sure enough page structs get created for any physical 11900Sstevel@tonic-gate * memory we might be returning to the system. 11910Sstevel@tonic-gate */ 11920Sstevel@tonic-gate ndata_remain_sz = ndata_maxsize(&ndata); 11930Sstevel@tonic-gate PRM_DEBUG(ndata_remain_sz); 11940Sstevel@tonic-gate 11950Sstevel@tonic-gate pp_sz = sizeof (struct page) * npages; 11960Sstevel@tonic-gate 11970Sstevel@tonic-gate /* 11980Sstevel@tonic-gate * Here's a nice bit of code based on somewhat recursive logic: 11990Sstevel@tonic-gate * 12000Sstevel@tonic-gate * If the page array would fit within the nucleus, we want to 12010Sstevel@tonic-gate * add npages to cover any extra memory we may be returning back 12020Sstevel@tonic-gate * to the system. 12030Sstevel@tonic-gate * 12040Sstevel@tonic-gate * HOWEVER, the page array is sized by calculating the size of 12050Sstevel@tonic-gate * (struct page * npages), as are the pagehash table, ctrs and 12060Sstevel@tonic-gate * memseg_list, so the very act of performing the calculation below may 12070Sstevel@tonic-gate * in fact make the array large enough that it no longer fits in the 12080Sstevel@tonic-gate * nucleus, meaning there would now be a much larger area of the 12090Sstevel@tonic-gate * nucleus free that should really be added to npages, which would 12100Sstevel@tonic-gate * make the page array that much larger, and so on. 12110Sstevel@tonic-gate * 12120Sstevel@tonic-gate * This also ignores the memory possibly used in the nucleus for the 12130Sstevel@tonic-gate * the page hash, ctrs and memseg list and the fact that whether they 12140Sstevel@tonic-gate * fit there or not varies with the npages calculation below, but we 12150Sstevel@tonic-gate * don't even factor them into the equation at this point; perhaps we 12160Sstevel@tonic-gate * should or perhaps we should just take the approach that the few 12170Sstevel@tonic-gate * extra pages we could add via this calculation REALLY aren't worth 12180Sstevel@tonic-gate * the hassle... 12190Sstevel@tonic-gate */ 12200Sstevel@tonic-gate if (ndata_remain_sz > pp_sz) { 12210Sstevel@tonic-gate size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 12220Sstevel@tonic-gate 12230Sstevel@tonic-gate npages += mmu_btop(spare); 12240Sstevel@tonic-gate 12250Sstevel@tonic-gate pp_sz = npages * sizeof (struct page); 12260Sstevel@tonic-gate 12270Sstevel@tonic-gate pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 12280Sstevel@tonic-gate } 12290Sstevel@tonic-gate 12300Sstevel@tonic-gate /* 12310Sstevel@tonic-gate * If physmem is patched to be non-zero, use it instead of 12320Sstevel@tonic-gate * the monitor value unless physmem is larger than the total 12330Sstevel@tonic-gate * amount of memory on hand. 12340Sstevel@tonic-gate */ 12350Sstevel@tonic-gate if (physmem == 0 || physmem > npages) 12360Sstevel@tonic-gate physmem = npages; 12370Sstevel@tonic-gate 12380Sstevel@tonic-gate /* 12390Sstevel@tonic-gate * If pp_base is NULL that means the routines above have determined 12400Sstevel@tonic-gate * the page array will not fit in the nucleus; we'll have to 12410Sstevel@tonic-gate * BOP_ALLOC() ourselves some space for them. 12420Sstevel@tonic-gate */ 12430Sstevel@tonic-gate if (pp_base == NULL) { 12440Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 12450Sstevel@tonic-gate alloc_alignsize); 12460Sstevel@tonic-gate 12470Sstevel@tonic-gate alloc_sz = roundup(pp_sz, alloc_alignsize); 12480Sstevel@tonic-gate 12490Sstevel@tonic-gate if ((pp_base = (struct page *)BOP_ALLOC(bootops, 12500Sstevel@tonic-gate alloc_base, alloc_sz, alloc_alignsize)) != 12510Sstevel@tonic-gate (struct page *)alloc_base) 12520Sstevel@tonic-gate panic("page alloc failure"); 12530Sstevel@tonic-gate 12540Sstevel@tonic-gate alloc_base += alloc_sz; 12550Sstevel@tonic-gate } 12560Sstevel@tonic-gate 12570Sstevel@tonic-gate /* 12580Sstevel@tonic-gate * The page structure hash table size is a power of 2 12590Sstevel@tonic-gate * such that the average hash chain length is PAGE_HASHAVELEN. 12600Sstevel@tonic-gate */ 12610Sstevel@tonic-gate page_hashsz = npages / PAGE_HASHAVELEN; 12620Sstevel@tonic-gate page_hashsz = 1 << highbit((ulong_t)page_hashsz); 12630Sstevel@tonic-gate pagehash_sz = sizeof (struct page *) * page_hashsz; 12640Sstevel@tonic-gate 12650Sstevel@tonic-gate /* 12660Sstevel@tonic-gate * We want to TRY to fit the page structure hash table, 12670Sstevel@tonic-gate * the page size free list counters, the memseg list and 12680Sstevel@tonic-gate * and the kpm page space in the nucleus if possible. 12690Sstevel@tonic-gate * 12700Sstevel@tonic-gate * alloc_sz counts how much memory needs to be allocated by 12710Sstevel@tonic-gate * BOP_ALLOC(). 12720Sstevel@tonic-gate */ 12730Sstevel@tonic-gate page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 12740Sstevel@tonic-gate 12750Sstevel@tonic-gate alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 12760Sstevel@tonic-gate 12770Sstevel@tonic-gate /* 12780Sstevel@tonic-gate * Size up per page size free list counters. 12790Sstevel@tonic-gate */ 12800Sstevel@tonic-gate ctrs_sz = page_ctrs_sz(); 12810Sstevel@tonic-gate ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 12820Sstevel@tonic-gate 12830Sstevel@tonic-gate if (ctrs_base == NULL) 12840Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 12850Sstevel@tonic-gate 12860Sstevel@tonic-gate /* 12870Sstevel@tonic-gate * The memseg list is for the chunks of physical memory that 12880Sstevel@tonic-gate * will be managed by the vm system. The number calculated is 12890Sstevel@tonic-gate * a guess as boot may fragment it more when memory allocations 12900Sstevel@tonic-gate * are made before kphysm_init(). Currently, there are two 12910Sstevel@tonic-gate * allocations before then, so we assume each causes fragmen- 12920Sstevel@tonic-gate * tation, and add a couple more for good measure. 12930Sstevel@tonic-gate */ 12940Sstevel@tonic-gate memseg_sz = sizeof (struct memseg) * (memblocks + 4); 12950Sstevel@tonic-gate memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 12960Sstevel@tonic-gate 12970Sstevel@tonic-gate if (memseg_base == NULL) 12980Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 12990Sstevel@tonic-gate 13000Sstevel@tonic-gate 13010Sstevel@tonic-gate if (kpm_enable) { 13020Sstevel@tonic-gate /* 13030Sstevel@tonic-gate * kpm page space -- Update kpm_npages and make the 13040Sstevel@tonic-gate * same assumption about fragmenting as it is done 13050Sstevel@tonic-gate * for memseg_sz above. 13060Sstevel@tonic-gate */ 13070Sstevel@tonic-gate kpm_npages_setup(memblocks + 4); 13080Sstevel@tonic-gate kpm_pp_sz = (kpm_smallpages == 0) ? 13090Sstevel@tonic-gate kpm_npages * sizeof (kpm_page_t): 13100Sstevel@tonic-gate kpm_npages * sizeof (kpm_spage_t); 13110Sstevel@tonic-gate 13120Sstevel@tonic-gate kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 13130Sstevel@tonic-gate ecache_alignsize); 13140Sstevel@tonic-gate 13150Sstevel@tonic-gate if (kpm_pp_base == NULL) 13160Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + 13170Sstevel@tonic-gate kpm_pp_sz; 13180Sstevel@tonic-gate } 13190Sstevel@tonic-gate 13200Sstevel@tonic-gate if (alloc_sz > 0) { 13210Sstevel@tonic-gate uintptr_t bop_base; 13220Sstevel@tonic-gate 13230Sstevel@tonic-gate /* 13240Sstevel@tonic-gate * We need extra memory allocated through BOP_ALLOC. 13250Sstevel@tonic-gate */ 13260Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 13270Sstevel@tonic-gate alloc_alignsize); 13280Sstevel@tonic-gate 13290Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, alloc_alignsize); 13300Sstevel@tonic-gate 13310Sstevel@tonic-gate if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 13320Sstevel@tonic-gate alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 13330Sstevel@tonic-gate panic("system page struct alloc failure"); 13340Sstevel@tonic-gate 13350Sstevel@tonic-gate alloc_base += alloc_sz; 13360Sstevel@tonic-gate 13370Sstevel@tonic-gate if (page_hash == NULL) { 13380Sstevel@tonic-gate page_hash = (struct page **)bop_base; 13390Sstevel@tonic-gate bop_base = roundup(bop_base + pagehash_sz, 13400Sstevel@tonic-gate ecache_alignsize); 13410Sstevel@tonic-gate } 13420Sstevel@tonic-gate 13430Sstevel@tonic-gate if (ctrs_base == NULL) { 13440Sstevel@tonic-gate ctrs_base = (caddr_t)bop_base; 13450Sstevel@tonic-gate bop_base = roundup(bop_base + ctrs_sz, 13460Sstevel@tonic-gate ecache_alignsize); 13470Sstevel@tonic-gate } 13480Sstevel@tonic-gate 13490Sstevel@tonic-gate if (memseg_base == NULL) { 13500Sstevel@tonic-gate memseg_base = (struct memseg *)bop_base; 13510Sstevel@tonic-gate bop_base = roundup(bop_base + memseg_sz, 13520Sstevel@tonic-gate ecache_alignsize); 13530Sstevel@tonic-gate } 13540Sstevel@tonic-gate 13550Sstevel@tonic-gate if (kpm_enable && kpm_pp_base == NULL) { 13560Sstevel@tonic-gate kpm_pp_base = (uintptr_t)bop_base; 13570Sstevel@tonic-gate bop_base = roundup(bop_base + kpm_pp_sz, 13580Sstevel@tonic-gate ecache_alignsize); 13590Sstevel@tonic-gate } 13600Sstevel@tonic-gate 13610Sstevel@tonic-gate ASSERT(bop_base <= (uintptr_t)alloc_base); 13620Sstevel@tonic-gate } 13630Sstevel@tonic-gate 13640Sstevel@tonic-gate /* 13650Sstevel@tonic-gate * Initialize per page size free list counters. 13660Sstevel@tonic-gate */ 13670Sstevel@tonic-gate ctrs_end = page_ctrs_alloc(ctrs_base); 13680Sstevel@tonic-gate ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 13690Sstevel@tonic-gate 13700Sstevel@tonic-gate PRM_DEBUG(page_hash); 13710Sstevel@tonic-gate PRM_DEBUG(memseg_base); 13720Sstevel@tonic-gate PRM_DEBUG(kpm_pp_base); 13730Sstevel@tonic-gate PRM_DEBUG(kpm_pp_sz); 13740Sstevel@tonic-gate PRM_DEBUG(pp_base); 13750Sstevel@tonic-gate PRM_DEBUG(pp_sz); 13760Sstevel@tonic-gate PRM_DEBUG(alloc_base); 13770Sstevel@tonic-gate 13780Sstevel@tonic-gate #ifdef TRAPTRACE 13790Sstevel@tonic-gate /* 13800Sstevel@tonic-gate * Allocate trap trace buffer last so as not to affect 13810Sstevel@tonic-gate * the 4M alignments of the allocations above on V9 SPARCs... 13820Sstevel@tonic-gate */ 13830Sstevel@tonic-gate alloc_base = trap_trace_alloc(alloc_base); 13840Sstevel@tonic-gate PRM_DEBUG(alloc_base); 13850Sstevel@tonic-gate #endif /* TRAPTRACE */ 13860Sstevel@tonic-gate 13870Sstevel@tonic-gate if (kmem64_base) { 13880Sstevel@tonic-gate /* 13890Sstevel@tonic-gate * Set the end of the kmem64 segment for V9 SPARCs, if 13900Sstevel@tonic-gate * appropriate... 13910Sstevel@tonic-gate */ 13920Sstevel@tonic-gate kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 13930Sstevel@tonic-gate alloc_alignsize); 13940Sstevel@tonic-gate 13950Sstevel@tonic-gate PRM_DEBUG(kmem64_base); 13960Sstevel@tonic-gate PRM_DEBUG(kmem64_end); 13970Sstevel@tonic-gate } 13980Sstevel@tonic-gate 13990Sstevel@tonic-gate /* 14002973Sgovinda * Allocate space for the interrupt vector table and also for the 14012973Sgovinda * reserved interrupt vector data structures. 14020Sstevel@tonic-gate */ 14032973Sgovinda memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 14040Sstevel@tonic-gate IVSIZE, MMU_PAGESIZE); 14052973Sgovinda if (memspace != (caddr_t)intr_vec_table) 14062973Sgovinda panic("interrupt vector table allocation failure"); 14070Sstevel@tonic-gate 14080Sstevel@tonic-gate /* 14090Sstevel@tonic-gate * The memory lists from boot are allocated from the heap arena 14100Sstevel@tonic-gate * so that later they can be freed and/or reallocated. 14110Sstevel@tonic-gate */ 14120Sstevel@tonic-gate if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 14130Sstevel@tonic-gate panic("could not retrieve property \"extent\""); 14140Sstevel@tonic-gate 14150Sstevel@tonic-gate /* 14160Sstevel@tonic-gate * Between now and when we finish copying in the memory lists, 14170Sstevel@tonic-gate * allocations happen so the space gets fragmented and the 14180Sstevel@tonic-gate * lists longer. Leave enough space for lists twice as long 14190Sstevel@tonic-gate * as what boot says it has now; roundup to a pagesize. 14200Sstevel@tonic-gate * Also add space for the final phys-avail copy in the fixup 14210Sstevel@tonic-gate * routine. 14220Sstevel@tonic-gate */ 14230Sstevel@tonic-gate va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 14240Sstevel@tonic-gate roundup(IVSIZE, MMU_PAGESIZE)); 14250Sstevel@tonic-gate memlist_sz *= 4; 14260Sstevel@tonic-gate memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 14270Sstevel@tonic-gate memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 14280Sstevel@tonic-gate if (memspace == NULL) 14290Sstevel@tonic-gate halt("Boot allocation failed."); 14300Sstevel@tonic-gate 14310Sstevel@tonic-gate memlist = (struct memlist *)memspace; 14320Sstevel@tonic-gate memlist_end = (char *)memspace + memlist_sz; 14330Sstevel@tonic-gate 14340Sstevel@tonic-gate PRM_DEBUG(memlist); 14350Sstevel@tonic-gate PRM_DEBUG(memlist_end); 14360Sstevel@tonic-gate PRM_DEBUG(sysbase); 14370Sstevel@tonic-gate PRM_DEBUG(syslimit); 14380Sstevel@tonic-gate 14390Sstevel@tonic-gate kernelheap_init((void *)sysbase, (void *)syslimit, 14400Sstevel@tonic-gate (caddr_t)sysbase + PAGESIZE, NULL, NULL); 14410Sstevel@tonic-gate 14420Sstevel@tonic-gate /* 14430Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 14440Sstevel@tonic-gate */ 14450Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 14460Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 14470Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 14480Sstevel@tonic-gate 14490Sstevel@tonic-gate /* 14500Sstevel@tonic-gate * Remove the space used by BOP_ALLOC from the kernel heap 14510Sstevel@tonic-gate * plus the area actually used by the OBP (if any) 14520Sstevel@tonic-gate * ignoring virtual addresses in virt_avail, above syslimit. 14530Sstevel@tonic-gate */ 14540Sstevel@tonic-gate virt_avail = memlist; 14550Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 14560Sstevel@tonic-gate 14570Sstevel@tonic-gate for (cur = virt_avail; cur->next; cur = cur->next) { 14580Sstevel@tonic-gate uint64_t range_base, range_size; 14590Sstevel@tonic-gate 14600Sstevel@tonic-gate if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 14610Sstevel@tonic-gate continue; 14620Sstevel@tonic-gate if (range_base >= (uint64_t)syslimit) 14630Sstevel@tonic-gate break; 14640Sstevel@tonic-gate /* 14650Sstevel@tonic-gate * Limit the range to end at syslimit. 14660Sstevel@tonic-gate */ 14670Sstevel@tonic-gate range_size = MIN(cur->next->address, 14680Sstevel@tonic-gate (uint64_t)syslimit) - range_base; 14690Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 14700Sstevel@tonic-gate 0, 0, (void *)range_base, (void *)(range_base + range_size), 14710Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 14720Sstevel@tonic-gate } 14730Sstevel@tonic-gate 14740Sstevel@tonic-gate phys_avail = memlist; 14750Sstevel@tonic-gate (void) copy_physavail(boot_physavail, boot_physavail_len, 14760Sstevel@tonic-gate &memlist, 0, 0); 14770Sstevel@tonic-gate 14780Sstevel@tonic-gate /* 14790Sstevel@tonic-gate * Add any extra memory after e_text to the phys_avail list, as long 14800Sstevel@tonic-gate * as there's at least a page to add. 14810Sstevel@tonic-gate */ 14820Sstevel@tonic-gate if (extra_etpg) 14830Sstevel@tonic-gate memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 14840Sstevel@tonic-gate &memlist, &phys_avail); 14850Sstevel@tonic-gate 14860Sstevel@tonic-gate /* 14870Sstevel@tonic-gate * Add any extra memory after e_data to the phys_avail list as long 14880Sstevel@tonic-gate * as there's at least a page to add. Usually, there isn't any, 14890Sstevel@tonic-gate * since extra HME blocks typically get allocated there first before 14900Sstevel@tonic-gate * using RAM elsewhere. 14910Sstevel@tonic-gate */ 14920Sstevel@tonic-gate if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 14930Sstevel@tonic-gate nalloc_base = nalloc_end; 14940Sstevel@tonic-gate ndata_remain_sz = nalloc_end - nalloc_base; 14950Sstevel@tonic-gate 14960Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE) 14970Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base), 14980Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 14990Sstevel@tonic-gate 15000Sstevel@tonic-gate PRM_DEBUG(memlist); 15010Sstevel@tonic-gate PRM_DEBUG(memlist_sz); 15020Sstevel@tonic-gate PRM_DEBUG(memspace); 15030Sstevel@tonic-gate 15040Sstevel@tonic-gate if ((caddr_t)memlist > (memspace + memlist_sz)) 15050Sstevel@tonic-gate panic("memlist overflow"); 15060Sstevel@tonic-gate 15070Sstevel@tonic-gate PRM_DEBUG(pp_base); 15080Sstevel@tonic-gate PRM_DEBUG(memseg_base); 15090Sstevel@tonic-gate PRM_DEBUG(npages); 15100Sstevel@tonic-gate 15110Sstevel@tonic-gate /* 15120Sstevel@tonic-gate * Initialize the page structures from the memory lists. 15130Sstevel@tonic-gate */ 15140Sstevel@tonic-gate kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 15150Sstevel@tonic-gate 15160Sstevel@tonic-gate availrmem_initial = availrmem = freemem; 15170Sstevel@tonic-gate PRM_DEBUG(availrmem); 15180Sstevel@tonic-gate 15190Sstevel@tonic-gate /* 15200Sstevel@tonic-gate * Some of the locks depend on page_hashsz being set! 15210Sstevel@tonic-gate * kmem_init() depends on this; so, keep it here. 15220Sstevel@tonic-gate */ 15230Sstevel@tonic-gate page_lock_init(); 15240Sstevel@tonic-gate 15250Sstevel@tonic-gate /* 15260Sstevel@tonic-gate * Initialize kernel memory allocator. 15270Sstevel@tonic-gate */ 15280Sstevel@tonic-gate kmem_init(); 15290Sstevel@tonic-gate 15300Sstevel@tonic-gate /* 1531*3717Sdp78419 * Factor in colorequiv to check additional 'equivalent' bins 1532*3717Sdp78419 */ 1533*3717Sdp78419 page_set_colorequiv_arr(); 1534*3717Sdp78419 1535*3717Sdp78419 /* 15360Sstevel@tonic-gate * Initialize bp_mapin(). 15370Sstevel@tonic-gate */ 15380Sstevel@tonic-gate bp_init(shm_alignment, HAT_STRICTORDER); 15390Sstevel@tonic-gate 15400Sstevel@tonic-gate /* 15412973Sgovinda * Reserve space for panicbuf, intr_vec_table and reserved interrupt 15422973Sgovinda * vector data structures from the 32-bit heap. 15430Sstevel@tonic-gate */ 15440Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 15450Sstevel@tonic-gate panicbuf, panicbuf + PANICBUFSIZE, 15460Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 15470Sstevel@tonic-gate 15480Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 15492973Sgovinda intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 15500Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 15510Sstevel@tonic-gate 15520Sstevel@tonic-gate mem_config_init(); 15530Sstevel@tonic-gate } 15540Sstevel@tonic-gate 15550Sstevel@tonic-gate static void 15560Sstevel@tonic-gate startup_modules(void) 15570Sstevel@tonic-gate { 15580Sstevel@tonic-gate int proplen, nhblk1, nhblk8; 15590Sstevel@tonic-gate size_t nhblksz; 15600Sstevel@tonic-gate pgcnt_t hblk_pages, pages_per_hblk; 15610Sstevel@tonic-gate size_t hme8blk_sz, hme1blk_sz; 15620Sstevel@tonic-gate 15630Sstevel@tonic-gate /* 15640Sstevel@tonic-gate * Log any optional messages from the boot program 15650Sstevel@tonic-gate */ 15660Sstevel@tonic-gate proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 15670Sstevel@tonic-gate if (proplen > 0) { 15680Sstevel@tonic-gate char *msg; 15690Sstevel@tonic-gate size_t len = (size_t)proplen; 15700Sstevel@tonic-gate 15710Sstevel@tonic-gate msg = kmem_zalloc(len, KM_SLEEP); 15720Sstevel@tonic-gate (void) BOP_GETPROP(bootops, "boot-message", msg); 15730Sstevel@tonic-gate cmn_err(CE_CONT, "?%s\n", msg); 15740Sstevel@tonic-gate kmem_free(msg, len); 15750Sstevel@tonic-gate } 15760Sstevel@tonic-gate 15770Sstevel@tonic-gate /* 15780Sstevel@tonic-gate * Let the platforms have a chance to change default 15790Sstevel@tonic-gate * values before reading system file. 15800Sstevel@tonic-gate */ 15810Sstevel@tonic-gate if (&set_platform_defaults) 15820Sstevel@tonic-gate set_platform_defaults(); 15830Sstevel@tonic-gate 15840Sstevel@tonic-gate /* 15850Sstevel@tonic-gate * Calculate default settings of system parameters based upon 15860Sstevel@tonic-gate * maxusers, yet allow to be overridden via the /etc/system file. 15870Sstevel@tonic-gate */ 15880Sstevel@tonic-gate param_calc(0); 15890Sstevel@tonic-gate 15900Sstevel@tonic-gate mod_setup(); 15910Sstevel@tonic-gate 15920Sstevel@tonic-gate /* 15930Sstevel@tonic-gate * If this is a positron, complain and halt. 15940Sstevel@tonic-gate */ 15950Sstevel@tonic-gate if (&iam_positron && iam_positron()) { 15960Sstevel@tonic-gate cmn_err(CE_WARN, "This hardware platform is not supported" 15970Sstevel@tonic-gate " by this release of Solaris.\n"); 15980Sstevel@tonic-gate #ifdef DEBUG 15990Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to resume */ 16000Sstevel@tonic-gate cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 16010Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 16020Sstevel@tonic-gate 16030Sstevel@tonic-gate #else /* DEBUG */ 16040Sstevel@tonic-gate halt(0); 16050Sstevel@tonic-gate #endif /* DEBUG */ 16060Sstevel@tonic-gate } 16070Sstevel@tonic-gate 16080Sstevel@tonic-gate /* 16090Sstevel@tonic-gate * If we are running firmware that isn't 64-bit ready 16100Sstevel@tonic-gate * then complain and halt. 16110Sstevel@tonic-gate */ 16120Sstevel@tonic-gate do_prom_version_check(); 16130Sstevel@tonic-gate 16140Sstevel@tonic-gate /* 16150Sstevel@tonic-gate * Initialize system parameters 16160Sstevel@tonic-gate */ 16170Sstevel@tonic-gate param_init(); 16180Sstevel@tonic-gate 16190Sstevel@tonic-gate /* 16200Sstevel@tonic-gate * maxmem is the amount of physical memory we're playing with. 16210Sstevel@tonic-gate */ 16220Sstevel@tonic-gate maxmem = physmem; 16230Sstevel@tonic-gate 16240Sstevel@tonic-gate /* Set segkp limits. */ 16253446Smrj ncbase = kdi_segdebugbase; 16263446Smrj ncend = kdi_segdebugbase; 16270Sstevel@tonic-gate 16280Sstevel@tonic-gate /* 16290Sstevel@tonic-gate * Initialize the hat layer. 16300Sstevel@tonic-gate */ 16310Sstevel@tonic-gate hat_init(); 16320Sstevel@tonic-gate 16330Sstevel@tonic-gate /* 16340Sstevel@tonic-gate * Initialize segment management stuff. 16350Sstevel@tonic-gate */ 16360Sstevel@tonic-gate seg_init(); 16370Sstevel@tonic-gate 16380Sstevel@tonic-gate /* 16390Sstevel@tonic-gate * Create the va>tte handler, so the prom can understand 16400Sstevel@tonic-gate * kernel translations. The handler is installed later, just 16410Sstevel@tonic-gate * as we are about to take over the trap table from the prom. 16420Sstevel@tonic-gate */ 16430Sstevel@tonic-gate create_va_to_tte(); 16440Sstevel@tonic-gate 16450Sstevel@tonic-gate /* 16460Sstevel@tonic-gate * Load the forthdebugger (optional) 16470Sstevel@tonic-gate */ 16480Sstevel@tonic-gate forthdebug_init(); 16490Sstevel@tonic-gate 16500Sstevel@tonic-gate /* 16510Sstevel@tonic-gate * Create OBP node for console input callbacks 16520Sstevel@tonic-gate * if it is needed. 16530Sstevel@tonic-gate */ 16541253Slq150181 startup_create_io_node(); 16550Sstevel@tonic-gate 16560Sstevel@tonic-gate if (modloadonly("fs", "specfs") == -1) 16570Sstevel@tonic-gate halt("Can't load specfs"); 16580Sstevel@tonic-gate 16590Sstevel@tonic-gate if (modloadonly("fs", "devfs") == -1) 16600Sstevel@tonic-gate halt("Can't load devfs"); 16610Sstevel@tonic-gate 16620Sstevel@tonic-gate if (modloadonly("misc", "swapgeneric") == -1) 16630Sstevel@tonic-gate halt("Can't load swapgeneric"); 16640Sstevel@tonic-gate 16651676Sjpk (void) modloadonly("sys", "lbl_edition"); 16661676Sjpk 16670Sstevel@tonic-gate dispinit(); 16680Sstevel@tonic-gate 16690Sstevel@tonic-gate /* 16700Sstevel@tonic-gate * Infer meanings to the members of the idprom buffer. 16710Sstevel@tonic-gate */ 16720Sstevel@tonic-gate parse_idprom(); 16730Sstevel@tonic-gate 16740Sstevel@tonic-gate /* Read cluster configuration data. */ 16750Sstevel@tonic-gate clconf_init(); 16760Sstevel@tonic-gate 16770Sstevel@tonic-gate setup_ddi(); 16780Sstevel@tonic-gate 16790Sstevel@tonic-gate /* 16800Sstevel@tonic-gate * Lets take this opportunity to load the root device. 16810Sstevel@tonic-gate */ 16820Sstevel@tonic-gate if (loadrootmodules() != 0) 16830Sstevel@tonic-gate debug_enter("Can't load the root filesystem"); 16840Sstevel@tonic-gate 16850Sstevel@tonic-gate /* 16860Sstevel@tonic-gate * Load tod driver module for the tod part found on this system. 16870Sstevel@tonic-gate * Recompute the cpu frequency/delays based on tod as tod part 16880Sstevel@tonic-gate * tends to keep time more accurately. 16890Sstevel@tonic-gate */ 16900Sstevel@tonic-gate if (&load_tod_module) 16910Sstevel@tonic-gate load_tod_module(); 16920Sstevel@tonic-gate 16930Sstevel@tonic-gate /* 16940Sstevel@tonic-gate * Allow platforms to load modules which might 16950Sstevel@tonic-gate * be needed after bootops are gone. 16960Sstevel@tonic-gate */ 16970Sstevel@tonic-gate if (&load_platform_modules) 16980Sstevel@tonic-gate load_platform_modules(); 16990Sstevel@tonic-gate 17000Sstevel@tonic-gate setcpudelay(); 17010Sstevel@tonic-gate 17020Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 17030Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 17040Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 17050Sstevel@tonic-gate 17060Sstevel@tonic-gate bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 17070Sstevel@tonic-gate 17080Sstevel@tonic-gate /* 17090Sstevel@tonic-gate * Calculation and allocation of hmeblks needed to remap 17100Sstevel@tonic-gate * the memory allocated by PROM till now: 17110Sstevel@tonic-gate * 17120Sstevel@tonic-gate * (1) calculate how much virtual memory has been bop_alloc'ed. 17130Sstevel@tonic-gate * (2) roundup this memory to span of hme8blk, i.e. 64KB 17140Sstevel@tonic-gate * (3) calculate number of hme8blk's needed to remap this memory 17150Sstevel@tonic-gate * (4) calculate amount of memory that's consumed by these hme8blk's 17160Sstevel@tonic-gate * (5) add memory calculated in steps (2) and (4) above. 17170Sstevel@tonic-gate * (6) roundup this memory to span of hme8blk, i.e. 64KB 17180Sstevel@tonic-gate * (7) calculate number of hme8blk's needed to remap this memory 17190Sstevel@tonic-gate * (8) calculate amount of memory that's consumed by these hme8blk's 17200Sstevel@tonic-gate * (9) allocate additional hme1blk's to hold large mappings. 17210Sstevel@tonic-gate * H8TOH1 determines this. The current SWAG gives enough hblk1's 17220Sstevel@tonic-gate * to remap everything with 4M mappings. 17230Sstevel@tonic-gate * (10) account for partially used hblk8's due to non-64K aligned 17240Sstevel@tonic-gate * PROM mapping entries. 17250Sstevel@tonic-gate * (11) add memory calculated in steps (8), (9), and (10) above. 17260Sstevel@tonic-gate * (12) kmem_zalloc the memory calculated in (11); since segkmem 17270Sstevel@tonic-gate * is not ready yet, this gets bop_alloc'ed. 17280Sstevel@tonic-gate * (13) there will be very few bop_alloc's after this point before 17290Sstevel@tonic-gate * trap table takes over 17300Sstevel@tonic-gate */ 17310Sstevel@tonic-gate 17320Sstevel@tonic-gate /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 17330Sstevel@tonic-gate hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 17340Sstevel@tonic-gate hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 17350Sstevel@tonic-gate 17360Sstevel@tonic-gate pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 17370Sstevel@tonic-gate bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 17380Sstevel@tonic-gate nhblk8 = bop_alloc_pages / pages_per_hblk; 17390Sstevel@tonic-gate nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 17400Sstevel@tonic-gate hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 17410Sstevel@tonic-gate bop_alloc_pages += hblk_pages; 17420Sstevel@tonic-gate bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 17430Sstevel@tonic-gate nhblk8 = bop_alloc_pages / pages_per_hblk; 17440Sstevel@tonic-gate nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 17450Sstevel@tonic-gate if (nhblk1 < hblk1_min) 17460Sstevel@tonic-gate nhblk1 = hblk1_min; 17470Sstevel@tonic-gate if (nhblk8 < hblk8_min) 17480Sstevel@tonic-gate nhblk8 = hblk8_min; 17490Sstevel@tonic-gate 17500Sstevel@tonic-gate /* 17510Sstevel@tonic-gate * Since hblk8's can hold up to 64k of mappings aligned on a 64k 17520Sstevel@tonic-gate * boundary, the number of hblk8's needed to map the entries in the 17530Sstevel@tonic-gate * boot_virtavail list needs to be adjusted to take this into 17540Sstevel@tonic-gate * consideration. Thus, we need to add additional hblk8's since it 17550Sstevel@tonic-gate * is possible that an hblk8 will not have all 8 slots used due to 17560Sstevel@tonic-gate * alignment constraints. Since there were boot_virtavail_len entries 17570Sstevel@tonic-gate * in that list, we need to add that many hblk8's to the number 17580Sstevel@tonic-gate * already calculated to make sure we don't underestimate. 17590Sstevel@tonic-gate */ 17600Sstevel@tonic-gate nhblk8 += boot_virtavail_len; 17610Sstevel@tonic-gate nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 17620Sstevel@tonic-gate 17630Sstevel@tonic-gate /* Allocate in pagesize chunks */ 17640Sstevel@tonic-gate nhblksz = roundup(nhblksz, MMU_PAGESIZE); 17650Sstevel@tonic-gate hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 17660Sstevel@tonic-gate sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 17670Sstevel@tonic-gate } 17680Sstevel@tonic-gate 17690Sstevel@tonic-gate static void 17700Sstevel@tonic-gate startup_bop_gone(void) 17710Sstevel@tonic-gate { 17720Sstevel@tonic-gate extern int bop_io_quiesced; 17730Sstevel@tonic-gate 17740Sstevel@tonic-gate /* 17751991Sheppo * Destroy the MD initialized at startup 17761991Sheppo * The startup initializes the MD framework 17771991Sheppo * using prom and BOP alloc free it now. 17781991Sheppo */ 17791991Sheppo mach_descrip_startup_fini(); 17801991Sheppo 17811991Sheppo /* 17820Sstevel@tonic-gate * Call back into boot and release boots resources. 17830Sstevel@tonic-gate */ 17840Sstevel@tonic-gate BOP_QUIESCE_IO(bootops); 17850Sstevel@tonic-gate bop_io_quiesced = 1; 17860Sstevel@tonic-gate 17870Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 17880Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 17890Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 17900Sstevel@tonic-gate /* 17910Sstevel@tonic-gate * Copy physinstalled list into kernel space. 17920Sstevel@tonic-gate */ 17930Sstevel@tonic-gate phys_install = memlist; 17940Sstevel@tonic-gate copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 17950Sstevel@tonic-gate 17960Sstevel@tonic-gate /* 17970Sstevel@tonic-gate * setup physically contiguous area twice as large as the ecache. 17980Sstevel@tonic-gate * this is used while doing displacement flush of ecaches 17990Sstevel@tonic-gate */ 18000Sstevel@tonic-gate if (&ecache_flush_address) { 18010Sstevel@tonic-gate ecache_flushaddr = ecache_flush_address(); 18020Sstevel@tonic-gate if (ecache_flushaddr == (uint64_t)-1) { 18030Sstevel@tonic-gate cmn_err(CE_PANIC, 18040Sstevel@tonic-gate "startup: no memory to set ecache_flushaddr"); 18050Sstevel@tonic-gate } 18060Sstevel@tonic-gate } 18070Sstevel@tonic-gate 18080Sstevel@tonic-gate /* 18090Sstevel@tonic-gate * Virtual available next. 18100Sstevel@tonic-gate */ 18110Sstevel@tonic-gate ASSERT(virt_avail != NULL); 18120Sstevel@tonic-gate memlist_free_list(virt_avail); 18130Sstevel@tonic-gate virt_avail = memlist; 18140Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 18150Sstevel@tonic-gate 18160Sstevel@tonic-gate /* 18170Sstevel@tonic-gate * Last chance to ask our booter questions .. 18180Sstevel@tonic-gate */ 18190Sstevel@tonic-gate } 18200Sstevel@tonic-gate 18210Sstevel@tonic-gate 18220Sstevel@tonic-gate /* 18230Sstevel@tonic-gate * startup_fixup_physavail - called from mach_sfmmu.c after the final 18240Sstevel@tonic-gate * allocations have been performed. We can't call it in startup_bop_gone 18250Sstevel@tonic-gate * since later operations can cause obp to allocate more memory. 18260Sstevel@tonic-gate */ 18270Sstevel@tonic-gate void 18280Sstevel@tonic-gate startup_fixup_physavail(void) 18290Sstevel@tonic-gate { 18300Sstevel@tonic-gate struct memlist *cur; 18310Sstevel@tonic-gate 18320Sstevel@tonic-gate /* 18330Sstevel@tonic-gate * take the most current snapshot we can by calling mem-update 18340Sstevel@tonic-gate */ 18350Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 18360Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 18370Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 18380Sstevel@tonic-gate 18390Sstevel@tonic-gate /* 18400Sstevel@tonic-gate * Copy phys_avail list, again. 18410Sstevel@tonic-gate * Both the kernel/boot and the prom have been allocating 18420Sstevel@tonic-gate * from the original list we copied earlier. 18430Sstevel@tonic-gate */ 18440Sstevel@tonic-gate cur = memlist; 18450Sstevel@tonic-gate (void) copy_physavail(boot_physavail, boot_physavail_len, 18460Sstevel@tonic-gate &memlist, 0, 0); 18470Sstevel@tonic-gate 18480Sstevel@tonic-gate /* 18490Sstevel@tonic-gate * Add any extra memory after e_text we added to the phys_avail list 18500Sstevel@tonic-gate * back to the old list. 18510Sstevel@tonic-gate */ 18520Sstevel@tonic-gate if (extra_etpg) 18530Sstevel@tonic-gate memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 18540Sstevel@tonic-gate &memlist, &cur); 18550Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE) 18560Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base), 18570Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &cur); 18580Sstevel@tonic-gate 18590Sstevel@tonic-gate /* 18600Sstevel@tonic-gate * There isn't any bounds checking on the memlist area 18610Sstevel@tonic-gate * so ensure it hasn't overgrown. 18620Sstevel@tonic-gate */ 18630Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 18640Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: memlist size exceeded"); 18650Sstevel@tonic-gate 18660Sstevel@tonic-gate /* 18670Sstevel@tonic-gate * The kernel removes the pages that were allocated for it from 18680Sstevel@tonic-gate * the freelist, but we now have to find any -extra- pages that 18690Sstevel@tonic-gate * the prom has allocated for it's own book-keeping, and remove 18700Sstevel@tonic-gate * them from the freelist too. sigh. 18710Sstevel@tonic-gate */ 18720Sstevel@tonic-gate fix_prom_pages(phys_avail, cur); 18730Sstevel@tonic-gate 18740Sstevel@tonic-gate ASSERT(phys_avail != NULL); 18750Sstevel@tonic-gate memlist_free_list(phys_avail); 18760Sstevel@tonic-gate phys_avail = cur; 18770Sstevel@tonic-gate 18780Sstevel@tonic-gate /* 18790Sstevel@tonic-gate * We're done with boot. Just after this point in time, boot 18800Sstevel@tonic-gate * gets unmapped, so we can no longer rely on its services. 18810Sstevel@tonic-gate * Zero the bootops to indicate this fact. 18820Sstevel@tonic-gate */ 18830Sstevel@tonic-gate bootops = (struct bootops *)NULL; 18840Sstevel@tonic-gate BOOTOPS_GONE(); 18850Sstevel@tonic-gate } 18860Sstevel@tonic-gate 18870Sstevel@tonic-gate static void 18880Sstevel@tonic-gate startup_vm(void) 18890Sstevel@tonic-gate { 18900Sstevel@tonic-gate size_t i; 18910Sstevel@tonic-gate struct segmap_crargs a; 18920Sstevel@tonic-gate struct segkpm_crargs b; 18930Sstevel@tonic-gate 18940Sstevel@tonic-gate uint64_t avmem; 18950Sstevel@tonic-gate caddr_t va; 18960Sstevel@tonic-gate pgcnt_t max_phys_segkp; 18970Sstevel@tonic-gate int mnode; 18980Sstevel@tonic-gate 18992991Ssusans extern int use_brk_lpg, use_stk_lpg; 1900423Sdavemq 19010Sstevel@tonic-gate /* 19020Sstevel@tonic-gate * get prom's mappings, create hments for them and switch 19030Sstevel@tonic-gate * to the kernel context. 19040Sstevel@tonic-gate */ 19050Sstevel@tonic-gate hat_kern_setup(); 19060Sstevel@tonic-gate 19070Sstevel@tonic-gate /* 19080Sstevel@tonic-gate * Take over trap table 19090Sstevel@tonic-gate */ 19100Sstevel@tonic-gate setup_trap_table(); 19110Sstevel@tonic-gate 19120Sstevel@tonic-gate /* 19130Sstevel@tonic-gate * Install the va>tte handler, so that the prom can handle 19140Sstevel@tonic-gate * misses and understand the kernel table layout in case 19150Sstevel@tonic-gate * we need call into the prom. 19160Sstevel@tonic-gate */ 19170Sstevel@tonic-gate install_va_to_tte(); 19180Sstevel@tonic-gate 19190Sstevel@tonic-gate /* 19200Sstevel@tonic-gate * Set a flag to indicate that the tba has been taken over. 19210Sstevel@tonic-gate */ 19220Sstevel@tonic-gate tba_taken_over = 1; 19230Sstevel@tonic-gate 19240Sstevel@tonic-gate /* initialize MMU primary context register */ 19250Sstevel@tonic-gate mmu_init_kcontext(); 19260Sstevel@tonic-gate 19270Sstevel@tonic-gate /* 19280Sstevel@tonic-gate * The boot cpu can now take interrupts, x-calls, x-traps 19290Sstevel@tonic-gate */ 19300Sstevel@tonic-gate CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 19310Sstevel@tonic-gate CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 19320Sstevel@tonic-gate 19330Sstevel@tonic-gate /* 19340Sstevel@tonic-gate * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 19350Sstevel@tonic-gate */ 19360Sstevel@tonic-gate tbr_wr_addr_inited = 1; 19370Sstevel@tonic-gate 19380Sstevel@tonic-gate /* 19390Sstevel@tonic-gate * Initialize VM system, and map kernel address space. 19400Sstevel@tonic-gate */ 19410Sstevel@tonic-gate kvm_init(); 19420Sstevel@tonic-gate 19430Sstevel@tonic-gate /* 19440Sstevel@tonic-gate * XXX4U: previously, we initialized and turned on 19450Sstevel@tonic-gate * the caches at this point. But of course we have 19460Sstevel@tonic-gate * nothing to do, as the prom has already done this 19470Sstevel@tonic-gate * for us -- main memory must be E$able at all times. 19480Sstevel@tonic-gate */ 19490Sstevel@tonic-gate 19500Sstevel@tonic-gate /* 19510Sstevel@tonic-gate * If the following is true, someone has patched 19520Sstevel@tonic-gate * phsymem to be less than the number of pages that 19530Sstevel@tonic-gate * the system actually has. Remove pages until system 19540Sstevel@tonic-gate * memory is limited to the requested amount. Since we 19550Sstevel@tonic-gate * have allocated page structures for all pages, we 19560Sstevel@tonic-gate * correct the amount of memory we want to remove 19570Sstevel@tonic-gate * by the size of the memory used to hold page structures 19580Sstevel@tonic-gate * for the non-used pages. 19590Sstevel@tonic-gate */ 19600Sstevel@tonic-gate if (physmem < npages) { 19610Sstevel@tonic-gate pgcnt_t diff, off; 19620Sstevel@tonic-gate struct page *pp; 19630Sstevel@tonic-gate struct seg kseg; 19640Sstevel@tonic-gate 19650Sstevel@tonic-gate cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 19660Sstevel@tonic-gate 19670Sstevel@tonic-gate off = 0; 19680Sstevel@tonic-gate diff = npages - physmem; 19690Sstevel@tonic-gate diff -= mmu_btopr(diff * sizeof (struct page)); 19700Sstevel@tonic-gate kseg.s_as = &kas; 19710Sstevel@tonic-gate while (diff--) { 19720Sstevel@tonic-gate pp = page_create_va(&unused_pages_vp, (offset_t)off, 19730Sstevel@tonic-gate MMU_PAGESIZE, PG_WAIT | PG_EXCL, 19740Sstevel@tonic-gate &kseg, (caddr_t)off); 19750Sstevel@tonic-gate if (pp == NULL) 19760Sstevel@tonic-gate cmn_err(CE_PANIC, "limited physmem too much!"); 19770Sstevel@tonic-gate page_io_unlock(pp); 19780Sstevel@tonic-gate page_downgrade(pp); 19790Sstevel@tonic-gate availrmem--; 19800Sstevel@tonic-gate off += MMU_PAGESIZE; 19810Sstevel@tonic-gate } 19820Sstevel@tonic-gate } 19830Sstevel@tonic-gate 19840Sstevel@tonic-gate /* 19850Sstevel@tonic-gate * When printing memory, show the total as physmem less 19860Sstevel@tonic-gate * that stolen by a debugger. 19870Sstevel@tonic-gate */ 19880Sstevel@tonic-gate cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 19890Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 10), 19900Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 19910Sstevel@tonic-gate 19920Sstevel@tonic-gate avmem = (uint64_t)freemem << PAGESHIFT; 19930Sstevel@tonic-gate cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 19940Sstevel@tonic-gate 19952991Ssusans /* 19962991Ssusans * For small memory systems disable automatic large pages. 19972991Ssusans */ 19982991Ssusans if (physmem < privm_lpg_min_physmem) { 1999423Sdavemq use_brk_lpg = 0; 2000423Sdavemq use_stk_lpg = 0; 2001423Sdavemq } 2002423Sdavemq 20030Sstevel@tonic-gate /* 20040Sstevel@tonic-gate * Perform platform specific freelist processing 20050Sstevel@tonic-gate */ 20060Sstevel@tonic-gate if (&plat_freelist_process) { 20070Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 20080Sstevel@tonic-gate if (mem_node_config[mnode].exists) 20090Sstevel@tonic-gate plat_freelist_process(mnode); 20100Sstevel@tonic-gate } 20110Sstevel@tonic-gate 20120Sstevel@tonic-gate /* 20130Sstevel@tonic-gate * Initialize the segkp segment type. We position it 20140Sstevel@tonic-gate * after the configured tables and buffers (whose end 20150Sstevel@tonic-gate * is given by econtig) and before V_WKBASE_ADDR. 20160Sstevel@tonic-gate * Also in this area is segkmap (size SEGMAPSIZE). 20170Sstevel@tonic-gate */ 20180Sstevel@tonic-gate 20190Sstevel@tonic-gate /* XXX - cache alignment? */ 20200Sstevel@tonic-gate va = (caddr_t)SEGKPBASE; 20210Sstevel@tonic-gate ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 20220Sstevel@tonic-gate 20230Sstevel@tonic-gate max_phys_segkp = (physmem * 2); 20240Sstevel@tonic-gate 20250Sstevel@tonic-gate if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 20260Sstevel@tonic-gate segkpsize = btop(SEGKPDEFSIZE); 20270Sstevel@tonic-gate cmn_err(CE_WARN, "Illegal value for segkpsize. " 20280Sstevel@tonic-gate "segkpsize has been reset to %ld pages", segkpsize); 20290Sstevel@tonic-gate } 20300Sstevel@tonic-gate 20310Sstevel@tonic-gate i = ptob(MIN(segkpsize, max_phys_segkp)); 20320Sstevel@tonic-gate 20330Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 20340Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkp) < 0) 20350Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: cannot attach segkp"); 20360Sstevel@tonic-gate if (segkp_create(segkp) != 0) 20370Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: segkp_create failed"); 20380Sstevel@tonic-gate rw_exit(&kas.a_lock); 20390Sstevel@tonic-gate 20400Sstevel@tonic-gate /* 20410Sstevel@tonic-gate * kpm segment 20420Sstevel@tonic-gate */ 20430Sstevel@tonic-gate segmap_kpm = kpm_enable && 20440Sstevel@tonic-gate segmap_kpm && PAGESIZE == MAXBSIZE; 20450Sstevel@tonic-gate 20460Sstevel@tonic-gate if (kpm_enable) { 20470Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 20480Sstevel@tonic-gate 20490Sstevel@tonic-gate /* 20500Sstevel@tonic-gate * The segkpm virtual range range is larger than the 20510Sstevel@tonic-gate * actual physical memory size and also covers gaps in 20520Sstevel@tonic-gate * the physical address range for the following reasons: 20530Sstevel@tonic-gate * . keep conversion between segkpm and physical addresses 20540Sstevel@tonic-gate * simple, cheap and unambiguous. 20550Sstevel@tonic-gate * . avoid extension/shrink of the the segkpm in case of DR. 20560Sstevel@tonic-gate * . avoid complexity for handling of virtual addressed 20570Sstevel@tonic-gate * caches, segkpm and the regular mapping scheme must be 20580Sstevel@tonic-gate * kept in sync wrt. the virtual color of mapped pages. 20590Sstevel@tonic-gate * Any accesses to virtual segkpm ranges not backed by 20600Sstevel@tonic-gate * physical memory will fall through the memseg pfn hash 20610Sstevel@tonic-gate * and will be handled in segkpm_fault. 20620Sstevel@tonic-gate * Additional kpm_size spaces needed for vac alias prevention. 20630Sstevel@tonic-gate */ 20640Sstevel@tonic-gate if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 20650Sstevel@tonic-gate segkpm) < 0) 20660Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkpm"); 20670Sstevel@tonic-gate 20680Sstevel@tonic-gate b.prot = PROT_READ | PROT_WRITE; 20690Sstevel@tonic-gate b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 20700Sstevel@tonic-gate 20710Sstevel@tonic-gate if (segkpm_create(segkpm, (caddr_t)&b) != 0) 20720Sstevel@tonic-gate panic("segkpm_create segkpm"); 20730Sstevel@tonic-gate 20740Sstevel@tonic-gate rw_exit(&kas.a_lock); 20752296Sae112802 20762296Sae112802 mach_kpm_init(); 20770Sstevel@tonic-gate } 20780Sstevel@tonic-gate 20793290Sjohansen if (!segzio_fromheap) { 20803290Sjohansen size_t size; 20813552Sjohansen size_t physmem_b = mmu_ptob(physmem); 20823290Sjohansen 20833290Sjohansen /* size is in bytes, segziosize is in pages */ 20843290Sjohansen if (segziosize == 0) { 20853552Sjohansen size = physmem_b; 20863290Sjohansen } else { 20873290Sjohansen size = mmu_ptob(segziosize); 20883290Sjohansen } 20893290Sjohansen 20903290Sjohansen if (size < SEGZIOMINSIZE) { 20913290Sjohansen size = SEGZIOMINSIZE; 20923552Sjohansen } else if (size > SEGZIOMAXSIZE) { 20933552Sjohansen size = SEGZIOMAXSIZE; 20943552Sjohansen /* 20953552Sjohansen * On 64-bit x86, we only have 2TB of KVA. This exists 20963552Sjohansen * for parity with x86. 20973552Sjohansen * 20983552Sjohansen * SEGZIOMAXSIZE is capped at 512gb so that segzio 20993552Sjohansen * doesn't consume all of KVA. However, if we have a 21003552Sjohansen * system that has more thant 512gb of physical memory, 21013552Sjohansen * we can actually consume about half of the difference 21023552Sjohansen * between 512gb and the rest of the available physical 21033552Sjohansen * memory. 21043552Sjohansen */ 21053552Sjohansen if (physmem_b > SEGZIOMAXSIZE) { 21063552Sjohansen size += (physmem_b - SEGZIOMAXSIZE) / 2; 21073552Sjohansen } 21083290Sjohansen } 21093290Sjohansen segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 21103290Sjohansen /* put the base of the ZIO segment after the kpm segment */ 21113290Sjohansen segzio_base = kpm_vbase + (kpm_size * vac_colors); 21123290Sjohansen PRM_DEBUG(segziosize); 21133290Sjohansen PRM_DEBUG(segzio_base); 21143290Sjohansen 21153290Sjohansen /* 21163290Sjohansen * On some platforms, kvm_init is called after the kpm 21173290Sjohansen * sizes have been determined. On SPARC, kvm_init is called 21183290Sjohansen * before, so we have to attach the kzioseg after kvm is 21193290Sjohansen * initialized, otherwise we'll try to allocate from the boot 21203290Sjohansen * area since the kernel heap hasn't yet been configured. 21213290Sjohansen */ 21223290Sjohansen rw_enter(&kas.a_lock, RW_WRITER); 21233290Sjohansen 21243290Sjohansen (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 21253290Sjohansen &kzioseg); 21263290Sjohansen (void) segkmem_zio_create(&kzioseg); 21273290Sjohansen 21283290Sjohansen /* create zio area covering new segment */ 21293290Sjohansen segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 21303290Sjohansen 21313290Sjohansen rw_exit(&kas.a_lock); 21323290Sjohansen } 21333290Sjohansen 21343290Sjohansen 21350Sstevel@tonic-gate /* 21360Sstevel@tonic-gate * Now create generic mapping segment. This mapping 21370Sstevel@tonic-gate * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 21380Sstevel@tonic-gate * virtual address is greater than the amount of free 21390Sstevel@tonic-gate * memory that is available, then we trim back the 21400Sstevel@tonic-gate * segment size to that amount 21410Sstevel@tonic-gate */ 21420Sstevel@tonic-gate va = (caddr_t)SEGMAPBASE; 21430Sstevel@tonic-gate 21440Sstevel@tonic-gate /* 21450Sstevel@tonic-gate * 1201049: segkmap base address must be MAXBSIZE aligned 21460Sstevel@tonic-gate */ 21470Sstevel@tonic-gate ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 21480Sstevel@tonic-gate 21490Sstevel@tonic-gate /* 21500Sstevel@tonic-gate * Set size of segmap to percentage of freemem at boot, 21510Sstevel@tonic-gate * but stay within the allowable range 21520Sstevel@tonic-gate * Note we take percentage before converting from pages 21530Sstevel@tonic-gate * to bytes to avoid an overflow on 32-bit kernels. 21540Sstevel@tonic-gate */ 21550Sstevel@tonic-gate i = mmu_ptob((freemem * segmap_percent) / 100); 21560Sstevel@tonic-gate 21570Sstevel@tonic-gate if (i < MINMAPSIZE) 21580Sstevel@tonic-gate i = MINMAPSIZE; 21590Sstevel@tonic-gate 21600Sstevel@tonic-gate if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 21610Sstevel@tonic-gate i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 21620Sstevel@tonic-gate 21630Sstevel@tonic-gate i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 21640Sstevel@tonic-gate 21650Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 21660Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkmap) < 0) 21670Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkmap"); 21680Sstevel@tonic-gate 21690Sstevel@tonic-gate a.prot = PROT_READ | PROT_WRITE; 21700Sstevel@tonic-gate a.shmsize = shm_alignment; 21710Sstevel@tonic-gate a.nfreelist = 0; /* use segmap driver defaults */ 21720Sstevel@tonic-gate 21730Sstevel@tonic-gate if (segmap_create(segkmap, (caddr_t)&a) != 0) 21740Sstevel@tonic-gate panic("segmap_create segkmap"); 21750Sstevel@tonic-gate rw_exit(&kas.a_lock); 21760Sstevel@tonic-gate 21770Sstevel@tonic-gate segdev_init(); 21780Sstevel@tonic-gate } 21790Sstevel@tonic-gate 21800Sstevel@tonic-gate static void 21810Sstevel@tonic-gate startup_end(void) 21820Sstevel@tonic-gate { 21830Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 21840Sstevel@tonic-gate panic("memlist overflow 2"); 21850Sstevel@tonic-gate memlist_free_block((caddr_t)memlist, 21860Sstevel@tonic-gate ((caddr_t)memlist_end - (caddr_t)memlist)); 21870Sstevel@tonic-gate memlist = NULL; 21880Sstevel@tonic-gate 21890Sstevel@tonic-gate /* enable page_relocation since OBP is now done */ 21900Sstevel@tonic-gate page_relocate_ready = 1; 21910Sstevel@tonic-gate 21920Sstevel@tonic-gate /* 21930Sstevel@tonic-gate * Perform tasks that get done after most of the VM 21940Sstevel@tonic-gate * initialization has been done but before the clock 21950Sstevel@tonic-gate * and other devices get started. 21960Sstevel@tonic-gate */ 21970Sstevel@tonic-gate kern_setup1(); 21980Sstevel@tonic-gate 21990Sstevel@tonic-gate /* 22000Sstevel@tonic-gate * Intialize the VM arenas for allocating physically 22010Sstevel@tonic-gate * contiguus memory chunk for interrupt queues snd 22020Sstevel@tonic-gate * allocate/register boot cpu's queues, if any and 22030Sstevel@tonic-gate * allocate dump buffer for sun4v systems to store 22040Sstevel@tonic-gate * extra crash information during crash dump 22050Sstevel@tonic-gate */ 22060Sstevel@tonic-gate contig_mem_init(); 22070Sstevel@tonic-gate mach_descrip_init(); 22080Sstevel@tonic-gate cpu_intrq_setup(CPU); 22090Sstevel@tonic-gate cpu_intrq_register(CPU); 22101077Ssvemuri mach_htraptrace_setup(CPU->cpu_id); 22111077Ssvemuri mach_htraptrace_configure(CPU->cpu_id); 22120Sstevel@tonic-gate mach_dump_buffer_init(); 22130Sstevel@tonic-gate 22140Sstevel@tonic-gate /* 22150Sstevel@tonic-gate * Initialize interrupt related stuff 22160Sstevel@tonic-gate */ 22171455Sandrei cpu_intr_alloc(CPU, NINTR_THREADS); 22180Sstevel@tonic-gate 22190Sstevel@tonic-gate (void) splzs(); /* allow hi clock ints but not zs */ 22200Sstevel@tonic-gate 22210Sstevel@tonic-gate /* 22220Sstevel@tonic-gate * Initialize errors. 22230Sstevel@tonic-gate */ 22240Sstevel@tonic-gate error_init(); 22250Sstevel@tonic-gate 22260Sstevel@tonic-gate /* 22270Sstevel@tonic-gate * Note that we may have already used kernel bcopy before this 22280Sstevel@tonic-gate * point - but if you really care about this, adb the use_hw_* 22290Sstevel@tonic-gate * variables to 0 before rebooting. 22300Sstevel@tonic-gate */ 22310Sstevel@tonic-gate mach_hw_copy_limit(); 22320Sstevel@tonic-gate 22330Sstevel@tonic-gate /* 22340Sstevel@tonic-gate * Install the "real" preemption guards before DDI services 22350Sstevel@tonic-gate * are available. 22360Sstevel@tonic-gate */ 22370Sstevel@tonic-gate (void) prom_set_preprom(kern_preprom); 22380Sstevel@tonic-gate (void) prom_set_postprom(kern_postprom); 22390Sstevel@tonic-gate CPU->cpu_m.mutex_ready = 1; 22400Sstevel@tonic-gate 22410Sstevel@tonic-gate /* 22420Sstevel@tonic-gate * Initialize segnf (kernel support for non-faulting loads). 22430Sstevel@tonic-gate */ 22440Sstevel@tonic-gate segnf_init(); 22450Sstevel@tonic-gate 22460Sstevel@tonic-gate /* 22470Sstevel@tonic-gate * Configure the root devinfo node. 22480Sstevel@tonic-gate */ 22490Sstevel@tonic-gate configure(); /* set up devices */ 22500Sstevel@tonic-gate mach_cpu_halt_idle(); 22510Sstevel@tonic-gate } 22520Sstevel@tonic-gate 22530Sstevel@tonic-gate 22540Sstevel@tonic-gate void 22550Sstevel@tonic-gate post_startup(void) 22560Sstevel@tonic-gate { 22570Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 22580Sstevel@tonic-gate extern void init_ptl1_thread(void); 22590Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 22600Sstevel@tonic-gate extern void abort_sequence_init(void); 22610Sstevel@tonic-gate 22620Sstevel@tonic-gate /* 22630Sstevel@tonic-gate * Set the system wide, processor-specific flags to be passed 22640Sstevel@tonic-gate * to userland via the aux vector for performance hints and 22650Sstevel@tonic-gate * instruction set extensions. 22660Sstevel@tonic-gate */ 22670Sstevel@tonic-gate bind_hwcap(); 22680Sstevel@tonic-gate 22690Sstevel@tonic-gate /* 22700Sstevel@tonic-gate * Startup memory scrubber (if any) 22710Sstevel@tonic-gate */ 22720Sstevel@tonic-gate mach_memscrub(); 22730Sstevel@tonic-gate 22740Sstevel@tonic-gate /* 22750Sstevel@tonic-gate * Allocate soft interrupt to handle abort sequence. 22760Sstevel@tonic-gate */ 22770Sstevel@tonic-gate abort_sequence_init(); 22780Sstevel@tonic-gate 22790Sstevel@tonic-gate /* 22800Sstevel@tonic-gate * Configure the rest of the system. 22810Sstevel@tonic-gate * Perform forceloading tasks for /etc/system. 22820Sstevel@tonic-gate */ 22830Sstevel@tonic-gate (void) mod_sysctl(SYS_FORCELOAD, NULL); 22840Sstevel@tonic-gate /* 22850Sstevel@tonic-gate * ON4.0: Force /proc module in until clock interrupt handle fixed 22860Sstevel@tonic-gate * ON4.0: This must be fixed or restated in /etc/systems. 22870Sstevel@tonic-gate */ 22880Sstevel@tonic-gate (void) modload("fs", "procfs"); 22890Sstevel@tonic-gate 22901991Sheppo /* load machine class specific drivers */ 22911991Sheppo load_mach_drivers(); 22921991Sheppo 22931991Sheppo /* load platform specific drivers */ 22940Sstevel@tonic-gate if (&load_platform_drivers) 22950Sstevel@tonic-gate load_platform_drivers(); 22960Sstevel@tonic-gate 22970Sstevel@tonic-gate /* load vis simulation module, if we are running w/fpu off */ 22980Sstevel@tonic-gate if (!fpu_exists) { 22990Sstevel@tonic-gate if (modload("misc", "vis") == -1) 23000Sstevel@tonic-gate halt("Can't load vis"); 23010Sstevel@tonic-gate } 23020Sstevel@tonic-gate 23030Sstevel@tonic-gate mach_fpras(); 23040Sstevel@tonic-gate 23050Sstevel@tonic-gate maxmem = freemem; 23060Sstevel@tonic-gate 23070Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 23080Sstevel@tonic-gate init_ptl1_thread(); 23090Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 23101991Sheppo 23111991Sheppo if (&cif_init) 23121991Sheppo cif_init(); 23130Sstevel@tonic-gate } 23140Sstevel@tonic-gate 23150Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 23160Sstevel@tonic-gate int ptl1_panic_test = 0; 23170Sstevel@tonic-gate int ptl1_panic_xc_one_test = 0; 23180Sstevel@tonic-gate int ptl1_panic_xc_all_test = 0; 23190Sstevel@tonic-gate int ptl1_panic_xt_one_test = 0; 23200Sstevel@tonic-gate int ptl1_panic_xt_all_test = 0; 23210Sstevel@tonic-gate kthread_id_t ptl1_thread_p = NULL; 23220Sstevel@tonic-gate kcondvar_t ptl1_cv; 23230Sstevel@tonic-gate kmutex_t ptl1_mutex; 23240Sstevel@tonic-gate int ptl1_recurse_count_threshold = 0x40; 23250Sstevel@tonic-gate int ptl1_recurse_trap_threshold = 0x3d; 23260Sstevel@tonic-gate extern void ptl1_recurse(int, int); 23270Sstevel@tonic-gate extern void ptl1_panic_xt(int, int); 23280Sstevel@tonic-gate 23290Sstevel@tonic-gate /* 23300Sstevel@tonic-gate * Called once per second by timeout() to wake up 23310Sstevel@tonic-gate * the ptl1_panic thread to see if it should cause 23320Sstevel@tonic-gate * a trap to the ptl1_panic() code. 23330Sstevel@tonic-gate */ 23340Sstevel@tonic-gate /* ARGSUSED */ 23350Sstevel@tonic-gate static void 23360Sstevel@tonic-gate ptl1_wakeup(void *arg) 23370Sstevel@tonic-gate { 23380Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 23390Sstevel@tonic-gate cv_signal(&ptl1_cv); 23400Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 23410Sstevel@tonic-gate } 23420Sstevel@tonic-gate 23430Sstevel@tonic-gate /* 23440Sstevel@tonic-gate * ptl1_panic cross call function: 23450Sstevel@tonic-gate * Needed because xc_one() and xc_some() can pass 23460Sstevel@tonic-gate * 64 bit args but ptl1_recurse() expects ints. 23470Sstevel@tonic-gate */ 23480Sstevel@tonic-gate static void 23490Sstevel@tonic-gate ptl1_panic_xc(void) 23500Sstevel@tonic-gate { 23510Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 23520Sstevel@tonic-gate ptl1_recurse_trap_threshold); 23530Sstevel@tonic-gate } 23540Sstevel@tonic-gate 23550Sstevel@tonic-gate /* 23560Sstevel@tonic-gate * The ptl1 thread waits for a global flag to be set 23570Sstevel@tonic-gate * and uses the recurse thresholds to set the stack depth 23580Sstevel@tonic-gate * to cause a ptl1_panic() directly via a call to ptl1_recurse 23590Sstevel@tonic-gate * or indirectly via the cross call and cross trap functions. 23600Sstevel@tonic-gate * 23610Sstevel@tonic-gate * This is useful testing stack overflows and normal 23620Sstevel@tonic-gate * ptl1_panic() states with a know stack frame. 23630Sstevel@tonic-gate * 23640Sstevel@tonic-gate * ptl1_recurse() is an asm function in ptl1_panic.s that 23650Sstevel@tonic-gate * sets the {In, Local, Out, and Global} registers to a 23660Sstevel@tonic-gate * know state on the stack and just prior to causing a 23670Sstevel@tonic-gate * test ptl1_panic trap. 23680Sstevel@tonic-gate */ 23690Sstevel@tonic-gate static void 23700Sstevel@tonic-gate ptl1_thread(void) 23710Sstevel@tonic-gate { 23720Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 23730Sstevel@tonic-gate while (ptl1_thread_p) { 23740Sstevel@tonic-gate cpuset_t other_cpus; 23750Sstevel@tonic-gate int cpu_id; 23760Sstevel@tonic-gate int my_cpu_id; 23770Sstevel@tonic-gate int target_cpu_id; 23780Sstevel@tonic-gate int target_found; 23790Sstevel@tonic-gate 23800Sstevel@tonic-gate if (ptl1_panic_test) { 23810Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 23820Sstevel@tonic-gate ptl1_recurse_trap_threshold); 23830Sstevel@tonic-gate } 23840Sstevel@tonic-gate 23850Sstevel@tonic-gate /* 23860Sstevel@tonic-gate * Find potential targets for x-call and x-trap, 23870Sstevel@tonic-gate * if any exist while preempt is disabled we 23880Sstevel@tonic-gate * start a ptl1_panic if requested via a 23890Sstevel@tonic-gate * globals. 23900Sstevel@tonic-gate */ 23910Sstevel@tonic-gate kpreempt_disable(); 23920Sstevel@tonic-gate my_cpu_id = CPU->cpu_id; 23930Sstevel@tonic-gate other_cpus = cpu_ready_set; 23940Sstevel@tonic-gate CPUSET_DEL(other_cpus, CPU->cpu_id); 23950Sstevel@tonic-gate target_found = 0; 23960Sstevel@tonic-gate if (!CPUSET_ISNULL(other_cpus)) { 23970Sstevel@tonic-gate /* 23980Sstevel@tonic-gate * Pick the first one 23990Sstevel@tonic-gate */ 24000Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 24010Sstevel@tonic-gate if (cpu_id == my_cpu_id) 24020Sstevel@tonic-gate continue; 24030Sstevel@tonic-gate 24040Sstevel@tonic-gate if (CPU_XCALL_READY(cpu_id)) { 24050Sstevel@tonic-gate target_cpu_id = cpu_id; 24060Sstevel@tonic-gate target_found = 1; 24070Sstevel@tonic-gate break; 24080Sstevel@tonic-gate } 24090Sstevel@tonic-gate } 24100Sstevel@tonic-gate ASSERT(target_found); 24110Sstevel@tonic-gate 24120Sstevel@tonic-gate if (ptl1_panic_xc_one_test) { 24130Sstevel@tonic-gate xc_one(target_cpu_id, 24140Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 24150Sstevel@tonic-gate } 24160Sstevel@tonic-gate if (ptl1_panic_xc_all_test) { 24170Sstevel@tonic-gate xc_some(other_cpus, 24180Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 24190Sstevel@tonic-gate } 24200Sstevel@tonic-gate if (ptl1_panic_xt_one_test) { 24210Sstevel@tonic-gate xt_one(target_cpu_id, 24220Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 24230Sstevel@tonic-gate } 24240Sstevel@tonic-gate if (ptl1_panic_xt_all_test) { 24250Sstevel@tonic-gate xt_some(other_cpus, 24260Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 24270Sstevel@tonic-gate } 24280Sstevel@tonic-gate } 24290Sstevel@tonic-gate kpreempt_enable(); 24300Sstevel@tonic-gate (void) timeout(ptl1_wakeup, NULL, hz); 24310Sstevel@tonic-gate (void) cv_wait(&ptl1_cv, &ptl1_mutex); 24320Sstevel@tonic-gate } 24330Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 24340Sstevel@tonic-gate } 24350Sstevel@tonic-gate 24360Sstevel@tonic-gate /* 24370Sstevel@tonic-gate * Called during early startup to create the ptl1_thread 24380Sstevel@tonic-gate */ 24390Sstevel@tonic-gate void 24400Sstevel@tonic-gate init_ptl1_thread(void) 24410Sstevel@tonic-gate { 24420Sstevel@tonic-gate ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 24430Sstevel@tonic-gate &p0, TS_RUN, 0); 24440Sstevel@tonic-gate } 24450Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 24460Sstevel@tonic-gate 24470Sstevel@tonic-gate 24480Sstevel@tonic-gate /* 24490Sstevel@tonic-gate * Add to a memory list. 24500Sstevel@tonic-gate * start = start of new memory segment 24510Sstevel@tonic-gate * len = length of new memory segment in bytes 24520Sstevel@tonic-gate * memlistp = pointer to array of available memory segment structures 24530Sstevel@tonic-gate * curmemlistp = memory list to which to add segment. 24540Sstevel@tonic-gate */ 24550Sstevel@tonic-gate static void 24560Sstevel@tonic-gate memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 24570Sstevel@tonic-gate struct memlist **curmemlistp) 24580Sstevel@tonic-gate { 24590Sstevel@tonic-gate struct memlist *new; 24600Sstevel@tonic-gate 24610Sstevel@tonic-gate new = *memlistp; 24620Sstevel@tonic-gate new->address = start; 24630Sstevel@tonic-gate new->size = len; 24640Sstevel@tonic-gate *memlistp = new + 1; 24650Sstevel@tonic-gate 24660Sstevel@tonic-gate memlist_insert(new, curmemlistp); 24670Sstevel@tonic-gate } 24680Sstevel@tonic-gate 24690Sstevel@tonic-gate /* 24700Sstevel@tonic-gate * In the case of architectures that support dynamic addition of 24710Sstevel@tonic-gate * memory at run-time there are two cases where memsegs need to 24720Sstevel@tonic-gate * be initialized and added to the memseg list. 24730Sstevel@tonic-gate * 1) memsegs that are constructed at startup. 24740Sstevel@tonic-gate * 2) memsegs that are constructed at run-time on 24750Sstevel@tonic-gate * hot-plug capable architectures. 24760Sstevel@tonic-gate * This code was originally part of the function kphysm_init(). 24770Sstevel@tonic-gate */ 24780Sstevel@tonic-gate 24790Sstevel@tonic-gate static void 24800Sstevel@tonic-gate memseg_list_add(struct memseg *memsegp) 24810Sstevel@tonic-gate { 24820Sstevel@tonic-gate struct memseg **prev_memsegp; 24830Sstevel@tonic-gate pgcnt_t num; 24840Sstevel@tonic-gate 24850Sstevel@tonic-gate /* insert in memseg list, decreasing number of pages order */ 24860Sstevel@tonic-gate 24870Sstevel@tonic-gate num = MSEG_NPAGES(memsegp); 24880Sstevel@tonic-gate 24890Sstevel@tonic-gate for (prev_memsegp = &memsegs; *prev_memsegp; 24900Sstevel@tonic-gate prev_memsegp = &((*prev_memsegp)->next)) { 24910Sstevel@tonic-gate if (num > MSEG_NPAGES(*prev_memsegp)) 24920Sstevel@tonic-gate break; 24930Sstevel@tonic-gate } 24940Sstevel@tonic-gate 24950Sstevel@tonic-gate memsegp->next = *prev_memsegp; 24960Sstevel@tonic-gate *prev_memsegp = memsegp; 24970Sstevel@tonic-gate 24980Sstevel@tonic-gate if (kpm_enable) { 24990Sstevel@tonic-gate memsegp->nextpa = (memsegp->next) ? 25000Sstevel@tonic-gate va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 25010Sstevel@tonic-gate 25020Sstevel@tonic-gate if (prev_memsegp != &memsegs) { 25030Sstevel@tonic-gate struct memseg *msp; 25040Sstevel@tonic-gate msp = (struct memseg *)((caddr_t)prev_memsegp - 25050Sstevel@tonic-gate offsetof(struct memseg, next)); 25060Sstevel@tonic-gate msp->nextpa = va_to_pa(memsegp); 25070Sstevel@tonic-gate } else { 25080Sstevel@tonic-gate memsegspa = va_to_pa(memsegs); 25090Sstevel@tonic-gate } 25100Sstevel@tonic-gate } 25110Sstevel@tonic-gate } 25120Sstevel@tonic-gate 25130Sstevel@tonic-gate /* 25140Sstevel@tonic-gate * PSM add_physmem_cb(). US-II and newer processors have some 25150Sstevel@tonic-gate * flavor of the prefetch capability implemented. We exploit 25160Sstevel@tonic-gate * this capability for optimum performance. 25170Sstevel@tonic-gate */ 25180Sstevel@tonic-gate #define PREFETCH_BYTES 64 25190Sstevel@tonic-gate 25200Sstevel@tonic-gate void 25210Sstevel@tonic-gate add_physmem_cb(page_t *pp, pfn_t pnum) 25220Sstevel@tonic-gate { 25230Sstevel@tonic-gate extern void prefetch_page_w(void *); 25240Sstevel@tonic-gate 25250Sstevel@tonic-gate pp->p_pagenum = pnum; 25260Sstevel@tonic-gate 25270Sstevel@tonic-gate /* 25280Sstevel@tonic-gate * Prefetch one more page_t into E$. To prevent future 25290Sstevel@tonic-gate * mishaps with the sizeof(page_t) changing on us, we 25300Sstevel@tonic-gate * catch this on debug kernels if we can't bring in the 25310Sstevel@tonic-gate * entire hpage with 2 PREFETCH_BYTES reads. See 25320Sstevel@tonic-gate * also, sun4u/cpu/cpu_module.c 25330Sstevel@tonic-gate */ 25340Sstevel@tonic-gate /*LINTED*/ 25350Sstevel@tonic-gate ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 25360Sstevel@tonic-gate prefetch_page_w((char *)pp); 25370Sstevel@tonic-gate } 25380Sstevel@tonic-gate 25390Sstevel@tonic-gate /* 25400Sstevel@tonic-gate * kphysm_init() tackles the problem of initializing physical memory. 25410Sstevel@tonic-gate * The old startup made some assumptions about the kernel living in 25420Sstevel@tonic-gate * physically contiguous space which is no longer valid. 25430Sstevel@tonic-gate */ 25440Sstevel@tonic-gate static void 25450Sstevel@tonic-gate kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 25460Sstevel@tonic-gate uintptr_t kpm_pp, pgcnt_t kpm_npages) 25470Sstevel@tonic-gate { 25480Sstevel@tonic-gate struct memlist *pmem; 25490Sstevel@tonic-gate struct memseg *msp; 25500Sstevel@tonic-gate pfn_t base; 25510Sstevel@tonic-gate pgcnt_t num; 25520Sstevel@tonic-gate pfn_t lastseg_pages_end = 0; 25530Sstevel@tonic-gate pgcnt_t nelem_used = 0; 25540Sstevel@tonic-gate 25550Sstevel@tonic-gate ASSERT(page_hash != NULL && page_hashsz != 0); 25560Sstevel@tonic-gate 25570Sstevel@tonic-gate msp = memsegp; 25580Sstevel@tonic-gate for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 25590Sstevel@tonic-gate 25600Sstevel@tonic-gate /* 25610Sstevel@tonic-gate * Build the memsegs entry 25620Sstevel@tonic-gate */ 25630Sstevel@tonic-gate num = btop(pmem->size); 25640Sstevel@tonic-gate if (num > npages) 25650Sstevel@tonic-gate num = npages; 25660Sstevel@tonic-gate npages -= num; 25670Sstevel@tonic-gate base = btop(pmem->address); 25680Sstevel@tonic-gate 25690Sstevel@tonic-gate msp->pages = pp; 25700Sstevel@tonic-gate msp->epages = pp + num; 25710Sstevel@tonic-gate msp->pages_base = base; 25720Sstevel@tonic-gate msp->pages_end = base + num; 25730Sstevel@tonic-gate 25740Sstevel@tonic-gate if (kpm_enable) { 25750Sstevel@tonic-gate pfn_t pbase_a; 25760Sstevel@tonic-gate pfn_t pend_a; 25770Sstevel@tonic-gate pfn_t prev_pend_a; 25780Sstevel@tonic-gate pgcnt_t nelem; 25790Sstevel@tonic-gate 25800Sstevel@tonic-gate msp->pagespa = va_to_pa(pp); 25810Sstevel@tonic-gate msp->epagespa = va_to_pa(pp + num); 25820Sstevel@tonic-gate pbase_a = kpmptop(ptokpmp(base)); 25830Sstevel@tonic-gate pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 25840Sstevel@tonic-gate nelem = ptokpmp(pend_a - pbase_a); 25850Sstevel@tonic-gate msp->kpm_nkpmpgs = nelem; 25860Sstevel@tonic-gate msp->kpm_pbase = pbase_a; 25870Sstevel@tonic-gate if (lastseg_pages_end) { 25880Sstevel@tonic-gate /* 25890Sstevel@tonic-gate * Assume phys_avail is in ascending order 25900Sstevel@tonic-gate * of physical addresses. 25910Sstevel@tonic-gate */ 25920Sstevel@tonic-gate ASSERT(base + num > lastseg_pages_end); 25930Sstevel@tonic-gate prev_pend_a = kpmptop( 25940Sstevel@tonic-gate ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 25950Sstevel@tonic-gate 25960Sstevel@tonic-gate if (prev_pend_a > pbase_a) { 25970Sstevel@tonic-gate /* 25980Sstevel@tonic-gate * Overlap, more than one memseg may 25990Sstevel@tonic-gate * point to the same kpm_page range. 26000Sstevel@tonic-gate */ 26010Sstevel@tonic-gate if (kpm_smallpages == 0) { 26020Sstevel@tonic-gate msp->kpm_pages = 26030Sstevel@tonic-gate (kpm_page_t *)kpm_pp - 1; 26040Sstevel@tonic-gate kpm_pp = (uintptr_t) 26050Sstevel@tonic-gate ((kpm_page_t *)kpm_pp 26060Sstevel@tonic-gate + nelem - 1); 26070Sstevel@tonic-gate } else { 26080Sstevel@tonic-gate msp->kpm_spages = 26090Sstevel@tonic-gate (kpm_spage_t *)kpm_pp - 1; 26100Sstevel@tonic-gate kpm_pp = (uintptr_t) 26110Sstevel@tonic-gate ((kpm_spage_t *)kpm_pp 26120Sstevel@tonic-gate + nelem - 1); 26130Sstevel@tonic-gate } 26140Sstevel@tonic-gate nelem_used += nelem - 1; 26150Sstevel@tonic-gate 26160Sstevel@tonic-gate } else { 26170Sstevel@tonic-gate if (kpm_smallpages == 0) { 26180Sstevel@tonic-gate msp->kpm_pages = 26190Sstevel@tonic-gate (kpm_page_t *)kpm_pp; 26200Sstevel@tonic-gate kpm_pp = (uintptr_t) 26210Sstevel@tonic-gate ((kpm_page_t *)kpm_pp 26220Sstevel@tonic-gate + nelem); 26230Sstevel@tonic-gate } else { 26240Sstevel@tonic-gate msp->kpm_spages = 26250Sstevel@tonic-gate (kpm_spage_t *)kpm_pp; 26260Sstevel@tonic-gate kpm_pp = (uintptr_t) 26270Sstevel@tonic-gate ((kpm_spage_t *) 26280Sstevel@tonic-gate kpm_pp + nelem); 26290Sstevel@tonic-gate } 26300Sstevel@tonic-gate nelem_used += nelem; 26310Sstevel@tonic-gate } 26320Sstevel@tonic-gate 26330Sstevel@tonic-gate } else { 26340Sstevel@tonic-gate if (kpm_smallpages == 0) { 26350Sstevel@tonic-gate msp->kpm_pages = (kpm_page_t *)kpm_pp; 26360Sstevel@tonic-gate kpm_pp = (uintptr_t) 26370Sstevel@tonic-gate ((kpm_page_t *)kpm_pp + nelem); 26380Sstevel@tonic-gate } else { 26390Sstevel@tonic-gate msp->kpm_spages = (kpm_spage_t *)kpm_pp; 26400Sstevel@tonic-gate kpm_pp = (uintptr_t) 26410Sstevel@tonic-gate ((kpm_spage_t *)kpm_pp + nelem); 26420Sstevel@tonic-gate } 26430Sstevel@tonic-gate nelem_used = nelem; 26440Sstevel@tonic-gate } 26450Sstevel@tonic-gate 26460Sstevel@tonic-gate if (nelem_used > kpm_npages) 26470Sstevel@tonic-gate panic("kphysm_init: kpm_pp overflow\n"); 26480Sstevel@tonic-gate 26490Sstevel@tonic-gate msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 26500Sstevel@tonic-gate lastseg_pages_end = msp->pages_end; 26510Sstevel@tonic-gate } 26520Sstevel@tonic-gate 26530Sstevel@tonic-gate memseg_list_add(msp); 26540Sstevel@tonic-gate 26550Sstevel@tonic-gate /* 26560Sstevel@tonic-gate * add_physmem() initializes the PSM part of the page 26570Sstevel@tonic-gate * struct by calling the PSM back with add_physmem_cb(). 26580Sstevel@tonic-gate * In addition it coalesces pages into larger pages as 26590Sstevel@tonic-gate * it initializes them. 26600Sstevel@tonic-gate */ 26610Sstevel@tonic-gate add_physmem(pp, num, base); 26620Sstevel@tonic-gate pp += num; 26630Sstevel@tonic-gate msp++; 26640Sstevel@tonic-gate } 26650Sstevel@tonic-gate 26660Sstevel@tonic-gate build_pfn_hash(); 26670Sstevel@tonic-gate } 26680Sstevel@tonic-gate 26690Sstevel@tonic-gate /* 26700Sstevel@tonic-gate * Kernel VM initialization. 26710Sstevel@tonic-gate * Assumptions about kernel address space ordering: 26720Sstevel@tonic-gate * (1) gap (user space) 26730Sstevel@tonic-gate * (2) kernel text 26740Sstevel@tonic-gate * (3) kernel data/bss 26750Sstevel@tonic-gate * (4) gap 26760Sstevel@tonic-gate * (5) kernel data structures 26770Sstevel@tonic-gate * (6) gap 26780Sstevel@tonic-gate * (7) debugger (optional) 26790Sstevel@tonic-gate * (8) monitor 26800Sstevel@tonic-gate * (9) gap (possibly null) 26810Sstevel@tonic-gate * (10) dvma 26820Sstevel@tonic-gate * (11) devices 26830Sstevel@tonic-gate */ 26840Sstevel@tonic-gate static void 26850Sstevel@tonic-gate kvm_init(void) 26860Sstevel@tonic-gate { 26870Sstevel@tonic-gate /* 26880Sstevel@tonic-gate * Put the kernel segments in kernel address space. 26890Sstevel@tonic-gate */ 26900Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 26910Sstevel@tonic-gate as_avlinit(&kas); 26920Sstevel@tonic-gate 26930Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)KERNELBASE, 26940Sstevel@tonic-gate (size_t)(e_moddata - KERNELBASE), &ktextseg); 26950Sstevel@tonic-gate (void) segkmem_create(&ktextseg); 26960Sstevel@tonic-gate 26970Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 26980Sstevel@tonic-gate (size_t)(MMU_PAGESIZE4M), &ktexthole); 26990Sstevel@tonic-gate (void) segkmem_create(&ktexthole); 27000Sstevel@tonic-gate 27010Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)valloc_base, 27020Sstevel@tonic-gate (size_t)(econtig32 - valloc_base), &kvalloc); 27030Sstevel@tonic-gate (void) segkmem_create(&kvalloc); 27040Sstevel@tonic-gate 27050Sstevel@tonic-gate if (kmem64_base) { 27060Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)kmem64_base, 27070Sstevel@tonic-gate (size_t)(kmem64_end - kmem64_base), &kmem64); 27080Sstevel@tonic-gate (void) segkmem_create(&kmem64); 27090Sstevel@tonic-gate } 27100Sstevel@tonic-gate 27110Sstevel@tonic-gate /* 27120Sstevel@tonic-gate * We're about to map out /boot. This is the beginning of the 27130Sstevel@tonic-gate * system resource management transition. We can no longer 27140Sstevel@tonic-gate * call into /boot for I/O or memory allocations. 27150Sstevel@tonic-gate */ 27160Sstevel@tonic-gate (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 27170Sstevel@tonic-gate (void) segkmem_create(&kvseg); 27180Sstevel@tonic-gate hblk_alloc_dynamic = 1; 27190Sstevel@tonic-gate 27200Sstevel@tonic-gate /* 27210Sstevel@tonic-gate * we need to preallocate pages for DR operations before enabling large 27220Sstevel@tonic-gate * page kernel heap because of memseg_remap_init() hat_unload() hack. 27230Sstevel@tonic-gate */ 27240Sstevel@tonic-gate memseg_remap_init(); 27250Sstevel@tonic-gate 27260Sstevel@tonic-gate /* at this point we are ready to use large page heap */ 27270Sstevel@tonic-gate segkmem_heap_lp_init(); 27280Sstevel@tonic-gate 27290Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 27300Sstevel@tonic-gate &kvseg32); 27310Sstevel@tonic-gate (void) segkmem_create(&kvseg32); 27320Sstevel@tonic-gate 27330Sstevel@tonic-gate /* 27340Sstevel@tonic-gate * Create a segment for the debugger. 27350Sstevel@tonic-gate */ 27363446Smrj (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 27370Sstevel@tonic-gate (void) segkmem_create(&kdebugseg); 27380Sstevel@tonic-gate 27390Sstevel@tonic-gate rw_exit(&kas.a_lock); 27400Sstevel@tonic-gate } 27410Sstevel@tonic-gate 27420Sstevel@tonic-gate char obp_tte_str[] = 27430Sstevel@tonic-gate "h# %x constant MMU_PAGESHIFT " 27440Sstevel@tonic-gate "h# %x constant TTE8K " 27450Sstevel@tonic-gate "h# %x constant SFHME_SIZE " 27460Sstevel@tonic-gate "h# %x constant SFHME_TTE " 27470Sstevel@tonic-gate "h# %x constant HMEBLK_TAG " 27480Sstevel@tonic-gate "h# %x constant HMEBLK_NEXT " 27490Sstevel@tonic-gate "h# %x constant HMEBLK_MISC " 27500Sstevel@tonic-gate "h# %x constant HMEBLK_HME1 " 27510Sstevel@tonic-gate "h# %x constant NHMENTS " 27520Sstevel@tonic-gate "h# %x constant HBLK_SZMASK " 27530Sstevel@tonic-gate "h# %x constant HBLK_RANGE_SHIFT " 27540Sstevel@tonic-gate "h# %x constant HMEBP_HBLK " 27550Sstevel@tonic-gate "h# %x constant HMEBUCKET_SIZE " 27560Sstevel@tonic-gate "h# %x constant HTAG_SFMMUPSZ " 27570Sstevel@tonic-gate "h# %x constant HTAG_REHASHSZ " 27580Sstevel@tonic-gate "h# %x constant mmu_hashcnt " 27590Sstevel@tonic-gate "h# %p constant uhme_hash " 27600Sstevel@tonic-gate "h# %p constant khme_hash " 27610Sstevel@tonic-gate "h# %x constant UHMEHASH_SZ " 27620Sstevel@tonic-gate "h# %x constant KHMEHASH_SZ " 27632241Shuah "h# %p constant KCONTEXT " 27640Sstevel@tonic-gate "h# %p constant KHATID " 27650Sstevel@tonic-gate "h# %x constant ASI_MEM " 27660Sstevel@tonic-gate 27670Sstevel@tonic-gate ": PHYS-X@ ( phys -- data ) " 27680Sstevel@tonic-gate " ASI_MEM spacex@ " 27690Sstevel@tonic-gate "; " 27700Sstevel@tonic-gate 27710Sstevel@tonic-gate ": PHYS-W@ ( phys -- data ) " 27720Sstevel@tonic-gate " ASI_MEM spacew@ " 27730Sstevel@tonic-gate "; " 27740Sstevel@tonic-gate 27750Sstevel@tonic-gate ": PHYS-L@ ( phys -- data ) " 27760Sstevel@tonic-gate " ASI_MEM spaceL@ " 27770Sstevel@tonic-gate "; " 27780Sstevel@tonic-gate 27790Sstevel@tonic-gate ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 27800Sstevel@tonic-gate " 3 * MMU_PAGESHIFT + " 27810Sstevel@tonic-gate "; " 27820Sstevel@tonic-gate 27830Sstevel@tonic-gate ": TTE_IS_VALID ( ttep -- flag ) " 27840Sstevel@tonic-gate " PHYS-X@ 0< " 27850Sstevel@tonic-gate "; " 27860Sstevel@tonic-gate 27870Sstevel@tonic-gate ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 27880Sstevel@tonic-gate " dup TTE8K = if " 27890Sstevel@tonic-gate " drop HBLK_RANGE_SHIFT " 27900Sstevel@tonic-gate " else " 27910Sstevel@tonic-gate " TTE_PAGE_SHIFT " 27920Sstevel@tonic-gate " then " 27930Sstevel@tonic-gate "; " 27940Sstevel@tonic-gate 27950Sstevel@tonic-gate ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 27960Sstevel@tonic-gate " tuck >> swap MMU_PAGESHIFT - << " 27970Sstevel@tonic-gate "; " 27980Sstevel@tonic-gate 27990Sstevel@tonic-gate ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 28000Sstevel@tonic-gate " >> over xor swap ( hash sfmmup ) " 28010Sstevel@tonic-gate " KHATID <> if ( hash ) " 28020Sstevel@tonic-gate " UHMEHASH_SZ and ( bucket ) " 28030Sstevel@tonic-gate " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 28040Sstevel@tonic-gate " else ( hash ) " 28050Sstevel@tonic-gate " KHMEHASH_SZ and ( bucket ) " 28060Sstevel@tonic-gate " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 28070Sstevel@tonic-gate " then ( hmebp ) " 28080Sstevel@tonic-gate "; " 28090Sstevel@tonic-gate 28100Sstevel@tonic-gate ": HME_HASH_TABLE_SEARCH " 28110Sstevel@tonic-gate " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 28120Sstevel@tonic-gate " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 28130Sstevel@tonic-gate " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 28140Sstevel@tonic-gate " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 28150Sstevel@tonic-gate " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 28160Sstevel@tonic-gate " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 28170Sstevel@tonic-gate " else " 28180Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 28190Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 28200Sstevel@tonic-gate " then " 28210Sstevel@tonic-gate " else " 28220Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 28230Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 28240Sstevel@tonic-gate " then " 28250Sstevel@tonic-gate " else " 28260Sstevel@tonic-gate " true " 28270Sstevel@tonic-gate " then " 28280Sstevel@tonic-gate " until r> drop " 28290Sstevel@tonic-gate "; " 28300Sstevel@tonic-gate 28310Sstevel@tonic-gate ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 28320Sstevel@tonic-gate " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 28330Sstevel@tonic-gate " HTAG_REHASHSZ << or nip ( hblktag ) " 28340Sstevel@tonic-gate "; " 28350Sstevel@tonic-gate 28360Sstevel@tonic-gate ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 28370Sstevel@tonic-gate " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 28380Sstevel@tonic-gate " TTE8K = if ( hmeblkp addr ) " 28390Sstevel@tonic-gate " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 28400Sstevel@tonic-gate " else ( hmeblkp addr ) " 28410Sstevel@tonic-gate " drop 0 ( hmeblkp 0 ) " 28420Sstevel@tonic-gate " then ( hmeblkp hme-index ) " 28430Sstevel@tonic-gate " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 28440Sstevel@tonic-gate " SFHME_TTE + ( ttep ) " 28450Sstevel@tonic-gate "; " 28460Sstevel@tonic-gate 28470Sstevel@tonic-gate ": unix-tte ( addr cnum -- false | tte-data true ) " 28482241Shuah " KCONTEXT = if ( addr ) " 28492241Shuah " KHATID ( addr khatid ) " 28502241Shuah " else ( addr ) " 28512241Shuah " drop false exit ( false ) " 28522241Shuah " then " 28532241Shuah " ( addr khatid ) " 28540Sstevel@tonic-gate " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 28550Sstevel@tonic-gate " 2dup swap i HME_HASH_SHIFT " 28560Sstevel@tonic-gate "( addr sfmmup sfmmup addr hmeshift ) " 28570Sstevel@tonic-gate " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 28580Sstevel@tonic-gate " over i 4 pick " 28590Sstevel@tonic-gate "( addr sfmmup hmebp sfmmup rehash addr ) " 28600Sstevel@tonic-gate " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 28610Sstevel@tonic-gate " HME_HASH_TABLE_SEARCH " 28620Sstevel@tonic-gate "( addr sfmmup { null | hmeblkp } ) " 28630Sstevel@tonic-gate " ?dup if ( addr sfmmup hmeblkp ) " 28640Sstevel@tonic-gate " nip swap HBLK_TO_TTEP ( ttep ) " 28650Sstevel@tonic-gate " dup TTE_IS_VALID if ( valid-ttep ) " 28660Sstevel@tonic-gate " PHYS-X@ true ( tte-data true ) " 28670Sstevel@tonic-gate " else ( invalid-tte ) " 28680Sstevel@tonic-gate " drop false ( false ) " 28690Sstevel@tonic-gate " then ( false | tte-data true ) " 28700Sstevel@tonic-gate " unloop exit ( false | tte-data true ) " 28710Sstevel@tonic-gate " then ( addr sfmmup ) " 28720Sstevel@tonic-gate " loop ( addr sfmmup ) " 28730Sstevel@tonic-gate " 2drop false ( false ) " 28740Sstevel@tonic-gate "; " 28750Sstevel@tonic-gate ; 28760Sstevel@tonic-gate 28770Sstevel@tonic-gate void 28780Sstevel@tonic-gate create_va_to_tte(void) 28790Sstevel@tonic-gate { 28800Sstevel@tonic-gate char *bp; 28810Sstevel@tonic-gate extern int khmehash_num, uhmehash_num; 28820Sstevel@tonic-gate extern struct hmehash_bucket *khme_hash, *uhme_hash; 28830Sstevel@tonic-gate 28840Sstevel@tonic-gate #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 28850Sstevel@tonic-gate 28860Sstevel@tonic-gate bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 28870Sstevel@tonic-gate 28880Sstevel@tonic-gate /* 28890Sstevel@tonic-gate * Teach obp how to parse our sw ttes. 28900Sstevel@tonic-gate */ 28910Sstevel@tonic-gate (void) sprintf(bp, obp_tte_str, 28920Sstevel@tonic-gate MMU_PAGESHIFT, 28930Sstevel@tonic-gate TTE8K, 28940Sstevel@tonic-gate sizeof (struct sf_hment), 28950Sstevel@tonic-gate OFFSET(struct sf_hment, hme_tte), 28960Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_tag), 28970Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_nextpa), 28980Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_misc), 28990Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_hme), 29000Sstevel@tonic-gate NHMENTS, 29010Sstevel@tonic-gate HBLK_SZMASK, 29020Sstevel@tonic-gate HBLK_RANGE_SHIFT, 29030Sstevel@tonic-gate OFFSET(struct hmehash_bucket, hmeh_nextpa), 29040Sstevel@tonic-gate sizeof (struct hmehash_bucket), 29050Sstevel@tonic-gate HTAG_SFMMUPSZ, 29060Sstevel@tonic-gate HTAG_REHASHSZ, 29070Sstevel@tonic-gate mmu_hashcnt, 29080Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)uhme_hash), 29090Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)khme_hash), 29100Sstevel@tonic-gate UHMEHASH_SZ, 29110Sstevel@tonic-gate KHMEHASH_SZ, 29122241Shuah KCONTEXT, 29130Sstevel@tonic-gate KHATID, 29140Sstevel@tonic-gate ASI_MEM); 29150Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 29160Sstevel@tonic-gate 29170Sstevel@tonic-gate kobj_free(bp, MMU_PAGESIZE); 29180Sstevel@tonic-gate } 29190Sstevel@tonic-gate 29200Sstevel@tonic-gate void 29210Sstevel@tonic-gate install_va_to_tte(void) 29220Sstevel@tonic-gate { 29230Sstevel@tonic-gate /* 29240Sstevel@tonic-gate * advise prom that he can use unix-tte 29250Sstevel@tonic-gate */ 29260Sstevel@tonic-gate prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 29270Sstevel@tonic-gate } 29280Sstevel@tonic-gate 29290Sstevel@tonic-gate 29301253Slq150181 /* 29311253Slq150181 * Because kmdb links prom_stdout_is_framebuffer into its own 29321253Slq150181 * module, we add "device-type=display" here for /os-io node, so that 29331253Slq150181 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 29341253Slq150181 * is registered into OBP. 29351253Slq150181 */ 29360Sstevel@tonic-gate static char *create_node = 29371492Slq150181 "\" /\" find-device " 29380Sstevel@tonic-gate "new-device " 29390Sstevel@tonic-gate "\" os-io\" device-name " 29401253Slq150181 "\" display\" device-type " 29410Sstevel@tonic-gate ": cb-r/w ( adr,len method$ -- #read/#written ) " 29420Sstevel@tonic-gate " 2>r swap 2 2r> ['] $callback catch if " 29430Sstevel@tonic-gate " 2drop 3drop 0 " 29440Sstevel@tonic-gate " then " 29450Sstevel@tonic-gate "; " 29460Sstevel@tonic-gate ": read ( adr,len -- #read ) " 29470Sstevel@tonic-gate " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 29480Sstevel@tonic-gate " ( retN ... ret1 N ) " 29490Sstevel@tonic-gate " ?dup if " 29500Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 29510Sstevel@tonic-gate " else " 29520Sstevel@tonic-gate " -2 " 29531492Slq150181 " then " 29540Sstevel@tonic-gate "; " 29550Sstevel@tonic-gate ": write ( adr,len -- #written ) " 29560Sstevel@tonic-gate " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 29570Sstevel@tonic-gate " ( retN ... ret1 N ) " 29580Sstevel@tonic-gate " ?dup if " 29590Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 29600Sstevel@tonic-gate " else " 29610Sstevel@tonic-gate " 0 " 29620Sstevel@tonic-gate " then " 29630Sstevel@tonic-gate "; " 29640Sstevel@tonic-gate ": poll-tty ( -- ) ; " 29650Sstevel@tonic-gate ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 29660Sstevel@tonic-gate ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 29670Sstevel@tonic-gate ": cb-give/take ( $method -- ) " 29680Sstevel@tonic-gate " 0 -rot ['] $callback catch ?dup if " 29690Sstevel@tonic-gate " >r 2drop 2drop r> throw " 29700Sstevel@tonic-gate " else " 29710Sstevel@tonic-gate " 0 ?do drop loop " 29720Sstevel@tonic-gate " then " 29730Sstevel@tonic-gate "; " 29740Sstevel@tonic-gate ": give ( -- ) \" exit-input\" cb-give/take ; " 29750Sstevel@tonic-gate ": take ( -- ) \" enter-input\" cb-give/take ; " 29760Sstevel@tonic-gate ": open ( -- ok? ) true ; " 29770Sstevel@tonic-gate ": close ( -- ) ; " 29780Sstevel@tonic-gate "finish-device " 29790Sstevel@tonic-gate "device-end "; 29800Sstevel@tonic-gate 29810Sstevel@tonic-gate /* 29821253Slq150181 * Create the OBP input/output node (FCode serial driver). 29831253Slq150181 * It is needed for both USB console keyboard and for 29841253Slq150181 * the kernel terminal emulator. It is too early to check for a 29851253Slq150181 * kernel console compatible framebuffer now, so we create this 29861253Slq150181 * so that we're ready if we need to enable kernel terminal emulation. 29870Sstevel@tonic-gate * 29881253Slq150181 * When the USB software takes over the input device at the time 29891253Slq150181 * consconfig runs, OBP's stdin is redirected to this node. 29901253Slq150181 * Whenever the FORTH user interface is used after this switch, 29911253Slq150181 * the node will call back into the kernel for console input. 29921253Slq150181 * If a serial device such as ttya or a UART with a Type 5 keyboard 29931253Slq150181 * attached is used, OBP takes over the serial device when the system 29941253Slq150181 * goes to the debugger after the system is booted. This sharing 29951253Slq150181 * of the relatively simple serial device is difficult but possible. 29961253Slq150181 * Sharing the USB host controller is impossible due its complexity. 29971253Slq150181 * 29981253Slq150181 * Similarly to USB keyboard input redirection, after consconfig_dacf 29991253Slq150181 * configures a kernel console framebuffer as the standard output 30001253Slq150181 * device, OBP's stdout is switched to to vector through the 30011253Slq150181 * /os-io node into the kernel terminal emulator. 30020Sstevel@tonic-gate */ 30030Sstevel@tonic-gate static void 30041253Slq150181 startup_create_io_node(void) 30050Sstevel@tonic-gate { 30061253Slq150181 prom_interpret(create_node, 0, 0, 0, 0, 0); 30070Sstevel@tonic-gate } 30080Sstevel@tonic-gate 30090Sstevel@tonic-gate 30100Sstevel@tonic-gate static void 30110Sstevel@tonic-gate do_prom_version_check(void) 30120Sstevel@tonic-gate { 30130Sstevel@tonic-gate int i; 3014789Sahrens pnode_t node; 30150Sstevel@tonic-gate char buf[64]; 30160Sstevel@tonic-gate static char drev[] = "Down-rev firmware detected%s\n" 30170Sstevel@tonic-gate "\tPlease upgrade to the following minimum version:\n" 30180Sstevel@tonic-gate "\t\t%s\n"; 30190Sstevel@tonic-gate 30200Sstevel@tonic-gate i = prom_version_check(buf, sizeof (buf), &node); 30210Sstevel@tonic-gate 30220Sstevel@tonic-gate if (i == PROM_VER64_OK) 30230Sstevel@tonic-gate return; 30240Sstevel@tonic-gate 30250Sstevel@tonic-gate if (i == PROM_VER64_UPGRADE) { 30260Sstevel@tonic-gate cmn_err(CE_WARN, drev, "", buf); 30270Sstevel@tonic-gate 30280Sstevel@tonic-gate #ifdef DEBUG 30290Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to continue */ 30300Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 30310Sstevel@tonic-gate return; 30320Sstevel@tonic-gate #else 30330Sstevel@tonic-gate halt(0); 30340Sstevel@tonic-gate #endif 30350Sstevel@tonic-gate } 30360Sstevel@tonic-gate 30370Sstevel@tonic-gate /* 30380Sstevel@tonic-gate * The other possibility is that this is a server running 30390Sstevel@tonic-gate * good firmware, but down-rev firmware was detected on at 30400Sstevel@tonic-gate * least one other cpu board. We just complain if we see 30410Sstevel@tonic-gate * that. 30420Sstevel@tonic-gate */ 30430Sstevel@tonic-gate cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 30440Sstevel@tonic-gate } 30450Sstevel@tonic-gate 30460Sstevel@tonic-gate static void 30470Sstevel@tonic-gate kpm_init() 30480Sstevel@tonic-gate { 30490Sstevel@tonic-gate kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 30500Sstevel@tonic-gate kpm_pgsz = 1ull << kpm_pgshft; 30510Sstevel@tonic-gate kpm_pgoff = kpm_pgsz - 1; 30520Sstevel@tonic-gate kpmp2pshft = kpm_pgshft - PAGESHIFT; 30530Sstevel@tonic-gate kpmpnpgs = 1 << kpmp2pshft; 30540Sstevel@tonic-gate ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 30550Sstevel@tonic-gate } 30560Sstevel@tonic-gate 30570Sstevel@tonic-gate void 30580Sstevel@tonic-gate kpm_npages_setup(int memblocks) 30590Sstevel@tonic-gate { 30600Sstevel@tonic-gate /* 30610Sstevel@tonic-gate * npages can be scattered in a maximum of 'memblocks' 30620Sstevel@tonic-gate */ 30630Sstevel@tonic-gate kpm_npages = ptokpmpr(npages) + memblocks; 30640Sstevel@tonic-gate } 30650Sstevel@tonic-gate 30660Sstevel@tonic-gate /* 30670Sstevel@tonic-gate * Must be defined in platform dependent code. 30680Sstevel@tonic-gate */ 30690Sstevel@tonic-gate extern caddr_t modtext; 30700Sstevel@tonic-gate extern size_t modtext_sz; 30710Sstevel@tonic-gate extern caddr_t moddata; 30720Sstevel@tonic-gate 30730Sstevel@tonic-gate #define HEAPTEXT_ARENA(addr) \ 30740Sstevel@tonic-gate ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 30750Sstevel@tonic-gate (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 30760Sstevel@tonic-gate (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 30770Sstevel@tonic-gate 30780Sstevel@tonic-gate #define HEAPTEXT_OVERSIZED(addr) \ 30790Sstevel@tonic-gate ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 30800Sstevel@tonic-gate 30810Sstevel@tonic-gate vmem_t *texthole_source[HEAPTEXT_NARENAS]; 30820Sstevel@tonic-gate vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 30830Sstevel@tonic-gate kmutex_t texthole_lock; 30840Sstevel@tonic-gate 30850Sstevel@tonic-gate char kern_bootargs[OBP_MAXPATHLEN]; 30860Sstevel@tonic-gate 30870Sstevel@tonic-gate void 30880Sstevel@tonic-gate kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 30890Sstevel@tonic-gate { 30900Sstevel@tonic-gate uintptr_t addr, limit; 30910Sstevel@tonic-gate 30920Sstevel@tonic-gate addr = HEAPTEXT_BASE; 30930Sstevel@tonic-gate limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 30940Sstevel@tonic-gate 30950Sstevel@tonic-gate /* 30960Sstevel@tonic-gate * Before we initialize the text_arena, we want to punch holes in the 30970Sstevel@tonic-gate * underlying heaptext_arena. This guarantees that for any text 30980Sstevel@tonic-gate * address we can find a text hole less than HEAPTEXT_MAPPED away. 30990Sstevel@tonic-gate */ 31000Sstevel@tonic-gate for (; addr + HEAPTEXT_UNMAPPED <= limit; 31010Sstevel@tonic-gate addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 31020Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 31030Sstevel@tonic-gate 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 31040Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 31050Sstevel@tonic-gate } 31060Sstevel@tonic-gate 31070Sstevel@tonic-gate /* 31080Sstevel@tonic-gate * Allocate one page at the oversize to break up the text region 31090Sstevel@tonic-gate * from the oversized region. 31100Sstevel@tonic-gate */ 31110Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 31120Sstevel@tonic-gate (void *)limit, (void *)(limit + PAGESIZE), 31130Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 31140Sstevel@tonic-gate 31150Sstevel@tonic-gate *text_arena = vmem_create("module_text", modtext, modtext_sz, 31160Sstevel@tonic-gate sizeof (uintptr_t), segkmem_alloc, segkmem_free, 31170Sstevel@tonic-gate heaptext_arena, 0, VM_SLEEP); 31180Sstevel@tonic-gate *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 31190Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 31200Sstevel@tonic-gate } 31210Sstevel@tonic-gate 31220Sstevel@tonic-gate caddr_t 31230Sstevel@tonic-gate kobj_text_alloc(vmem_t *arena, size_t size) 31240Sstevel@tonic-gate { 31250Sstevel@tonic-gate caddr_t rval, better; 31260Sstevel@tonic-gate 31270Sstevel@tonic-gate /* 31280Sstevel@tonic-gate * First, try a sleeping allocation. 31290Sstevel@tonic-gate */ 31300Sstevel@tonic-gate rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 31310Sstevel@tonic-gate 31320Sstevel@tonic-gate if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 31330Sstevel@tonic-gate return (rval); 31340Sstevel@tonic-gate 31350Sstevel@tonic-gate /* 31360Sstevel@tonic-gate * We didn't get the area that we wanted. We're going to try to do an 31370Sstevel@tonic-gate * allocation with explicit constraints. 31380Sstevel@tonic-gate */ 31390Sstevel@tonic-gate better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 31400Sstevel@tonic-gate (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 31410Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT); 31420Sstevel@tonic-gate 31430Sstevel@tonic-gate if (better != NULL) { 31440Sstevel@tonic-gate /* 31450Sstevel@tonic-gate * That worked. Free our first attempt and return. 31460Sstevel@tonic-gate */ 31470Sstevel@tonic-gate vmem_free(arena, rval, size); 31480Sstevel@tonic-gate return (better); 31490Sstevel@tonic-gate } 31500Sstevel@tonic-gate 31510Sstevel@tonic-gate /* 31520Sstevel@tonic-gate * That didn't work; we'll have to return our first attempt. 31530Sstevel@tonic-gate */ 31540Sstevel@tonic-gate return (rval); 31550Sstevel@tonic-gate } 31560Sstevel@tonic-gate 31570Sstevel@tonic-gate caddr_t 31580Sstevel@tonic-gate kobj_texthole_alloc(caddr_t addr, size_t size) 31590Sstevel@tonic-gate { 31600Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 31610Sstevel@tonic-gate char c[30]; 31620Sstevel@tonic-gate uintptr_t base; 31630Sstevel@tonic-gate 31640Sstevel@tonic-gate if (HEAPTEXT_OVERSIZED(addr)) { 31650Sstevel@tonic-gate /* 31660Sstevel@tonic-gate * If this is an oversized allocation, there is no text hole 31670Sstevel@tonic-gate * available for it; return NULL. 31680Sstevel@tonic-gate */ 31690Sstevel@tonic-gate return (NULL); 31700Sstevel@tonic-gate } 31710Sstevel@tonic-gate 31720Sstevel@tonic-gate mutex_enter(&texthole_lock); 31730Sstevel@tonic-gate 31740Sstevel@tonic-gate if (texthole_arena[arena] == NULL) { 31750Sstevel@tonic-gate ASSERT(texthole_source[arena] == NULL); 31760Sstevel@tonic-gate 31770Sstevel@tonic-gate if (arena == 0) { 31780Sstevel@tonic-gate texthole_source[0] = vmem_create("module_text_holesrc", 31790Sstevel@tonic-gate (void *)(KERNELBASE + MMU_PAGESIZE4M), 31800Sstevel@tonic-gate MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 31810Sstevel@tonic-gate 0, VM_SLEEP); 31820Sstevel@tonic-gate } else { 31830Sstevel@tonic-gate base = HEAPTEXT_BASE + 31840Sstevel@tonic-gate (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 31850Sstevel@tonic-gate 31860Sstevel@tonic-gate (void) snprintf(c, sizeof (c), 31870Sstevel@tonic-gate "heaptext_holesrc_%d", arena); 31880Sstevel@tonic-gate 31890Sstevel@tonic-gate texthole_source[arena] = vmem_create(c, (void *)base, 31900Sstevel@tonic-gate HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 31910Sstevel@tonic-gate 0, VM_SLEEP); 31920Sstevel@tonic-gate } 31930Sstevel@tonic-gate 31940Sstevel@tonic-gate (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 31950Sstevel@tonic-gate 31960Sstevel@tonic-gate texthole_arena[arena] = vmem_create(c, NULL, 0, 31970Sstevel@tonic-gate sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 31980Sstevel@tonic-gate texthole_source[arena], 0, VM_SLEEP); 31990Sstevel@tonic-gate } 32000Sstevel@tonic-gate 32010Sstevel@tonic-gate mutex_exit(&texthole_lock); 32020Sstevel@tonic-gate 32030Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 32040Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 32050Sstevel@tonic-gate return (vmem_alloc(texthole_arena[arena], size, 32060Sstevel@tonic-gate VM_BESTFIT | VM_NOSLEEP)); 32070Sstevel@tonic-gate } 32080Sstevel@tonic-gate 32090Sstevel@tonic-gate void 32100Sstevel@tonic-gate kobj_texthole_free(caddr_t addr, size_t size) 32110Sstevel@tonic-gate { 32120Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 32130Sstevel@tonic-gate 32140Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 32150Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 32160Sstevel@tonic-gate vmem_free(texthole_arena[arena], addr, size); 32170Sstevel@tonic-gate } 3218