10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51253Slq150181 * Common Development and Distribution License (the "License"). 61455Sandrei * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 211253Slq150181 220Sstevel@tonic-gate /* 235872Ssetje * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 240Sstevel@tonic-gate * Use is subject to license terms. 250Sstevel@tonic-gate */ 260Sstevel@tonic-gate 270Sstevel@tonic-gate #include <sys/machsystm.h> 280Sstevel@tonic-gate #include <sys/archsystm.h> 290Sstevel@tonic-gate #include <sys/vm.h> 300Sstevel@tonic-gate #include <sys/cpu.h> 310Sstevel@tonic-gate #include <sys/atomic.h> 320Sstevel@tonic-gate #include <sys/reboot.h> 330Sstevel@tonic-gate #include <sys/kdi.h> 340Sstevel@tonic-gate #include <sys/bootconf.h> 350Sstevel@tonic-gate #include <sys/memlist_plat.h> 360Sstevel@tonic-gate #include <sys/memlist_impl.h> 370Sstevel@tonic-gate #include <sys/prom_plat.h> 380Sstevel@tonic-gate #include <sys/prom_isa.h> 390Sstevel@tonic-gate #include <sys/autoconf.h> 400Sstevel@tonic-gate #include <sys/intreg.h> 410Sstevel@tonic-gate #include <sys/ivintr.h> 420Sstevel@tonic-gate #include <sys/fpu/fpusystm.h> 430Sstevel@tonic-gate #include <sys/iommutsb.h> 440Sstevel@tonic-gate #include <vm/vm_dep.h> 450Sstevel@tonic-gate #include <vm/seg_dev.h> 460Sstevel@tonic-gate #include <vm/seg_kmem.h> 470Sstevel@tonic-gate #include <vm/seg_kpm.h> 480Sstevel@tonic-gate #include <vm/seg_map.h> 490Sstevel@tonic-gate #include <vm/seg_kp.h> 500Sstevel@tonic-gate #include <sys/sysconf.h> 510Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 520Sstevel@tonic-gate #include <sys/kobj.h> 530Sstevel@tonic-gate #include <sys/sun4asi.h> 540Sstevel@tonic-gate #include <sys/clconf.h> 550Sstevel@tonic-gate #include <sys/platform_module.h> 560Sstevel@tonic-gate #include <sys/panic.h> 570Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h> 580Sstevel@tonic-gate #include <sys/clock.h> 590Sstevel@tonic-gate #include <sys/cmn_err.h> 600Sstevel@tonic-gate #include <sys/promif.h> 610Sstevel@tonic-gate #include <sys/prom_debug.h> 620Sstevel@tonic-gate #include <sys/traptrace.h> 630Sstevel@tonic-gate #include <sys/memnode.h> 640Sstevel@tonic-gate #include <sys/mem_cage.h> 652241Shuah #include <sys/mmu.h> 660Sstevel@tonic-gate 670Sstevel@tonic-gate extern void setup_trap_table(void); 684050Sjb145095 extern int cpu_intrq_setup(struct cpu *); 690Sstevel@tonic-gate extern void cpu_intrq_register(struct cpu *); 700Sstevel@tonic-gate extern void contig_mem_init(void); 714204Sha137994 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 720Sstevel@tonic-gate extern void mach_dump_buffer_init(void); 730Sstevel@tonic-gate extern void mach_descrip_init(void); 741991Sheppo extern void mach_descrip_startup_fini(void); 750Sstevel@tonic-gate extern void mach_memscrub(void); 760Sstevel@tonic-gate extern void mach_fpras(void); 770Sstevel@tonic-gate extern void mach_cpu_halt_idle(void); 780Sstevel@tonic-gate extern void mach_hw_copy_limit(void); 791991Sheppo extern void load_mach_drivers(void); 800Sstevel@tonic-gate extern void load_tod_module(void); 810Sstevel@tonic-gate #pragma weak load_tod_module 820Sstevel@tonic-gate 830Sstevel@tonic-gate extern int ndata_alloc_mmfsa(struct memlist *ndata); 840Sstevel@tonic-gate #pragma weak ndata_alloc_mmfsa 850Sstevel@tonic-gate 861991Sheppo extern void cif_init(void); 871991Sheppo #pragma weak cif_init 881991Sheppo 890Sstevel@tonic-gate extern void parse_idprom(void); 900Sstevel@tonic-gate extern void add_vx_handler(char *, int, void (*)(cell_t *)); 910Sstevel@tonic-gate extern void mem_config_init(void); 920Sstevel@tonic-gate extern void memseg_remap_init(void); 930Sstevel@tonic-gate 942296Sae112802 extern void mach_kpm_init(void); 956880Sdv142724 extern void pcf_init(); 964878Sblakej extern int size_pse_array(pgcnt_t, int); 972296Sae112802 980Sstevel@tonic-gate /* 990Sstevel@tonic-gate * External Data: 1000Sstevel@tonic-gate */ 1010Sstevel@tonic-gate extern int vac_size; /* cache size in bytes */ 1020Sstevel@tonic-gate extern uint_t vac_mask; /* VAC alignment consistency mask */ 1030Sstevel@tonic-gate extern uint_t vac_colors; 1040Sstevel@tonic-gate 1050Sstevel@tonic-gate /* 1060Sstevel@tonic-gate * Global Data Definitions: 1070Sstevel@tonic-gate */ 1080Sstevel@tonic-gate 1090Sstevel@tonic-gate /* 1100Sstevel@tonic-gate * XXX - Don't port this to new architectures 1110Sstevel@tonic-gate * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 1120Sstevel@tonic-gate * 'romp' has no use with a prom with an IEEE 1275 client interface. 1130Sstevel@tonic-gate * The driver doesn't use the value, but it depends on the symbol. 1140Sstevel@tonic-gate */ 1150Sstevel@tonic-gate void *romp; /* veritas driver won't load without romp 4154976 */ 1160Sstevel@tonic-gate /* 1170Sstevel@tonic-gate * Declare these as initialized data so we can patch them. 1180Sstevel@tonic-gate */ 1190Sstevel@tonic-gate pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 1200Sstevel@tonic-gate pgcnt_t segkpsize = 1210Sstevel@tonic-gate btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 1220Sstevel@tonic-gate uint_t segmap_percent = 12; /* Size of segmap segment */ 1230Sstevel@tonic-gate 1240Sstevel@tonic-gate int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 1250Sstevel@tonic-gate int vac_copyback = 1; 1260Sstevel@tonic-gate char *cache_mode = NULL; 1270Sstevel@tonic-gate int use_mix = 1; 1280Sstevel@tonic-gate int prom_debug = 0; 1290Sstevel@tonic-gate 1300Sstevel@tonic-gate caddr_t boot_tba; /* %tba at boot - used by kmdb */ 1310Sstevel@tonic-gate uint_t tba_taken_over = 0; 1320Sstevel@tonic-gate 1330Sstevel@tonic-gate caddr_t s_text; /* start of kernel text segment */ 1340Sstevel@tonic-gate caddr_t e_text; /* end of kernel text segment */ 1350Sstevel@tonic-gate caddr_t s_data; /* start of kernel data segment */ 1360Sstevel@tonic-gate caddr_t e_data; /* end of kernel data segment */ 1370Sstevel@tonic-gate 1380Sstevel@tonic-gate caddr_t modtext; /* beginning of module text */ 1390Sstevel@tonic-gate size_t modtext_sz; /* size of module text */ 1400Sstevel@tonic-gate caddr_t moddata; /* beginning of module data reserve */ 1410Sstevel@tonic-gate caddr_t e_moddata; /* end of module data reserve */ 1420Sstevel@tonic-gate 1430Sstevel@tonic-gate /* 1440Sstevel@tonic-gate * End of first block of contiguous kernel in 32-bit virtual address space 1450Sstevel@tonic-gate */ 1460Sstevel@tonic-gate caddr_t econtig32; /* end of first blk of contiguous kernel */ 1470Sstevel@tonic-gate 1480Sstevel@tonic-gate caddr_t ncbase; /* beginning of non-cached segment */ 1490Sstevel@tonic-gate caddr_t ncend; /* end of non-cached segment */ 1500Sstevel@tonic-gate 1510Sstevel@tonic-gate size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 1520Sstevel@tonic-gate caddr_t nalloc_base; /* beginning of nucleus allocation */ 1530Sstevel@tonic-gate caddr_t nalloc_end; /* end of nucleus allocatable memory */ 1540Sstevel@tonic-gate caddr_t valloc_base; /* beginning of kvalloc segment */ 1550Sstevel@tonic-gate 1560Sstevel@tonic-gate caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 1570Sstevel@tonic-gate caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 1585648Ssetje size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 1593764Sdp78419 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 1603764Sdp78419 int kmem64_szc; /* page size code */ 1613764Sdp78419 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 1620Sstevel@tonic-gate 1632296Sae112802 uintptr_t shm_alignment; /* VAC address consistency modulus */ 1640Sstevel@tonic-gate struct memlist *phys_install; /* Total installed physical memory */ 1650Sstevel@tonic-gate struct memlist *phys_avail; /* Available (unreserved) physical memory */ 1660Sstevel@tonic-gate struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 1675648Ssetje struct memlist *nopp_list; /* pages with no backing page structs */ 1680Sstevel@tonic-gate struct memlist ndata; /* memlist of nucleus allocatable memory */ 1690Sstevel@tonic-gate int memexp_flag; /* memory expansion card flag */ 1700Sstevel@tonic-gate uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 1710Sstevel@tonic-gate pgcnt_t obp_pages; /* Physical pages used by OBP */ 1720Sstevel@tonic-gate 1730Sstevel@tonic-gate /* 1740Sstevel@tonic-gate * VM data structures 1750Sstevel@tonic-gate */ 1760Sstevel@tonic-gate long page_hashsz; /* Size of page hash table (power of two) */ 1770Sstevel@tonic-gate struct page *pp_base; /* Base of system page struct array */ 1780Sstevel@tonic-gate size_t pp_sz; /* Size in bytes of page struct array */ 1790Sstevel@tonic-gate struct page **page_hash; /* Page hash table */ 1804878Sblakej pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 1814878Sblakej size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 1824878Sblakej int pse_shift; /* log2(pse_table_size) */ 1830Sstevel@tonic-gate struct seg ktextseg; /* Segment used for kernel executable image */ 1840Sstevel@tonic-gate struct seg kvalloc; /* Segment used for "valloc" mapping */ 1850Sstevel@tonic-gate struct seg kpseg; /* Segment used for pageable kernel virt mem */ 1860Sstevel@tonic-gate struct seg ktexthole; /* Segment used for nucleus text hole */ 1870Sstevel@tonic-gate struct seg kmapseg; /* Segment used for generic kernel mappings */ 1880Sstevel@tonic-gate struct seg kpmseg; /* Segment used for physical mapping */ 1890Sstevel@tonic-gate struct seg kdebugseg; /* Segment used for the kernel debugger */ 1900Sstevel@tonic-gate 1915648Ssetje void *kpm_pp_base; /* Base of system kpm_page array */ 1920Sstevel@tonic-gate size_t kpm_pp_sz; /* Size of system kpm_page array */ 1930Sstevel@tonic-gate pgcnt_t kpm_npages; /* How many kpm pages are managed */ 1940Sstevel@tonic-gate 1950Sstevel@tonic-gate struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 1960Sstevel@tonic-gate struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 1970Sstevel@tonic-gate struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 1980Sstevel@tonic-gate 1993290Sjohansen int segzio_fromheap = 0; /* zio allocations occur from heap */ 2003290Sjohansen caddr_t segzio_base; /* Base address of segzio */ 2013290Sjohansen pgcnt_t segziosize = 0; /* size of zio segment in pages */ 2023290Sjohansen 2030Sstevel@tonic-gate /* 2040Sstevel@tonic-gate * debugger pages (if allocated) 2050Sstevel@tonic-gate */ 2060Sstevel@tonic-gate struct vnode kdebugvp; 2070Sstevel@tonic-gate 2080Sstevel@tonic-gate /* 2093446Smrj * VA range available to the debugger 2103446Smrj */ 2113446Smrj const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 2123446Smrj const size_t kdi_segdebugsize = SEGDEBUGSIZE; 2133446Smrj 2143446Smrj /* 2150Sstevel@tonic-gate * Segment for relocated kernel structures in 64-bit large RAM kernels 2160Sstevel@tonic-gate */ 2170Sstevel@tonic-gate struct seg kmem64; 2180Sstevel@tonic-gate 2195648Ssetje struct memseg *memseg_free; 2205648Ssetje 2210Sstevel@tonic-gate struct vnode unused_pages_vp; 2220Sstevel@tonic-gate 2230Sstevel@tonic-gate /* 2240Sstevel@tonic-gate * VM data structures allocated early during boot. 2250Sstevel@tonic-gate */ 2260Sstevel@tonic-gate size_t pagehash_sz; 2270Sstevel@tonic-gate uint64_t memlist_sz; 2280Sstevel@tonic-gate 2290Sstevel@tonic-gate char tbr_wr_addr_inited = 0; 2300Sstevel@tonic-gate 2315358Sjc25722 caddr_t mpo_heap32_buf = NULL; 2325358Sjc25722 size_t mpo_heap32_bufsz = 0; 2330Sstevel@tonic-gate 2340Sstevel@tonic-gate /* 2350Sstevel@tonic-gate * Static Routines: 2360Sstevel@tonic-gate */ 2375648Ssetje static int ndata_alloc_memseg(struct memlist *, size_t); 2385648Ssetje static void memlist_new(uint64_t, uint64_t, struct memlist **); 2395648Ssetje static void memlist_add(uint64_t, uint64_t, 2405648Ssetje struct memlist **, struct memlist **); 2415648Ssetje static void kphysm_init(void); 2420Sstevel@tonic-gate static void kvm_init(void); 2435648Ssetje static void install_kmem64_tte(void); 2440Sstevel@tonic-gate 2450Sstevel@tonic-gate static void startup_init(void); 2460Sstevel@tonic-gate static void startup_memlist(void); 2470Sstevel@tonic-gate static void startup_modules(void); 2480Sstevel@tonic-gate static void startup_bop_gone(void); 2490Sstevel@tonic-gate static void startup_vm(void); 2500Sstevel@tonic-gate static void startup_end(void); 2510Sstevel@tonic-gate static void setup_cage_params(void); 2521253Slq150181 static void startup_create_io_node(void); 2530Sstevel@tonic-gate 2540Sstevel@tonic-gate static pgcnt_t npages; 2550Sstevel@tonic-gate static struct memlist *memlist; 2560Sstevel@tonic-gate void *memlist_end; 2570Sstevel@tonic-gate 2580Sstevel@tonic-gate static pgcnt_t bop_alloc_pages; 2590Sstevel@tonic-gate static caddr_t hblk_base; 2600Sstevel@tonic-gate uint_t hblk_alloc_dynamic = 0; 2610Sstevel@tonic-gate uint_t hblk1_min = H1MIN; 2620Sstevel@tonic-gate 2630Sstevel@tonic-gate 2640Sstevel@tonic-gate /* 2650Sstevel@tonic-gate * Hooks for unsupported platforms and down-rev firmware 2660Sstevel@tonic-gate */ 2670Sstevel@tonic-gate int iam_positron(void); 2680Sstevel@tonic-gate #pragma weak iam_positron 2690Sstevel@tonic-gate static void do_prom_version_check(void); 2700Sstevel@tonic-gate 2710Sstevel@tonic-gate /* 2720Sstevel@tonic-gate * After receiving a thermal interrupt, this is the number of seconds 2730Sstevel@tonic-gate * to delay before shutting off the system, assuming 2740Sstevel@tonic-gate * shutdown fails. Use /etc/system to change the delay if this isn't 2750Sstevel@tonic-gate * large enough. 2760Sstevel@tonic-gate */ 2770Sstevel@tonic-gate int thermal_powerdown_delay = 1200; 2780Sstevel@tonic-gate 2790Sstevel@tonic-gate /* 2800Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed 2810Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel. 2820Sstevel@tonic-gate */ 2830Sstevel@tonic-gate int page_relocate_ready = 0; 2840Sstevel@tonic-gate 2850Sstevel@tonic-gate /* 2867218Ssvemuri * Indicate if kmem64 allocation was done in small chunks 2877218Ssvemuri */ 2887218Ssvemuri int kmem64_smchunks = 0; 2897218Ssvemuri 2907218Ssvemuri /* 2910Sstevel@tonic-gate * Enable some debugging messages concerning memory usage... 2920Sstevel@tonic-gate */ 2930Sstevel@tonic-gate #ifdef DEBUGGING_MEM 2940Sstevel@tonic-gate static int debugging_mem; 2950Sstevel@tonic-gate static void 2960Sstevel@tonic-gate printmemlist(char *title, struct memlist *list) 2970Sstevel@tonic-gate { 2980Sstevel@tonic-gate if (!debugging_mem) 2990Sstevel@tonic-gate return; 3000Sstevel@tonic-gate 3010Sstevel@tonic-gate printf("%s\n", title); 3020Sstevel@tonic-gate 3030Sstevel@tonic-gate while (list) { 3040Sstevel@tonic-gate prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 3050Sstevel@tonic-gate (uint32_t)(list->address >> 32), (uint32_t)list->address, 3060Sstevel@tonic-gate (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 3070Sstevel@tonic-gate list = list->next; 3080Sstevel@tonic-gate } 3090Sstevel@tonic-gate } 3100Sstevel@tonic-gate 3110Sstevel@tonic-gate void 3120Sstevel@tonic-gate printmemseg(struct memseg *memseg) 3130Sstevel@tonic-gate { 3140Sstevel@tonic-gate if (!debugging_mem) 3150Sstevel@tonic-gate return; 3160Sstevel@tonic-gate 3170Sstevel@tonic-gate printf("memseg\n"); 3180Sstevel@tonic-gate 3190Sstevel@tonic-gate while (memseg) { 3200Sstevel@tonic-gate prom_printf("\tpage = 0x%p, epage = 0x%p, " 3210Sstevel@tonic-gate "pfn = 0x%x, epfn = 0x%x\n", 3220Sstevel@tonic-gate memseg->pages, memseg->epages, 3230Sstevel@tonic-gate memseg->pages_base, memseg->pages_end); 3240Sstevel@tonic-gate memseg = memseg->next; 3250Sstevel@tonic-gate } 3260Sstevel@tonic-gate } 3270Sstevel@tonic-gate 3280Sstevel@tonic-gate #define debug_pause(str) halt((str)) 3290Sstevel@tonic-gate #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 3300Sstevel@tonic-gate #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 3310Sstevel@tonic-gate #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 3320Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) \ 3330Sstevel@tonic-gate if (debugging_mem) prom_printf((str), (a), (b), (c)) 3340Sstevel@tonic-gate #else /* DEBUGGING_MEM */ 3350Sstevel@tonic-gate #define MPRINTF(str) 3360Sstevel@tonic-gate #define MPRINTF1(str, a) 3370Sstevel@tonic-gate #define MPRINTF2(str, a, b) 3380Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) 3390Sstevel@tonic-gate #endif /* DEBUGGING_MEM */ 3400Sstevel@tonic-gate 3410Sstevel@tonic-gate 3420Sstevel@tonic-gate /* 3430Sstevel@tonic-gate * 3440Sstevel@tonic-gate * Kernel's Virtual Memory Layout. 3450Sstevel@tonic-gate * /-----------------------\ 3460Sstevel@tonic-gate * 0xFFFFFFFF.FFFFFFFF -| |- 3470Sstevel@tonic-gate * | OBP's virtual page | 3480Sstevel@tonic-gate * | tables | 3490Sstevel@tonic-gate * 0xFFFFFFFC.00000000 -|-----------------------|- 3500Sstevel@tonic-gate * : : 3510Sstevel@tonic-gate * : : 3523290Sjohansen * -|-----------------------|- 3533290Sjohansen * | segzio | (base and size vary) 3540Sstevel@tonic-gate * 0xFFFFFE00.00000000 -|-----------------------|- 3550Sstevel@tonic-gate * | | Ultrasparc I/II support 3560Sstevel@tonic-gate * | segkpm segment | up to 2TB of physical 3570Sstevel@tonic-gate * | (64-bit kernel ONLY) | memory, VAC has 2 colors 3580Sstevel@tonic-gate * | | 3590Sstevel@tonic-gate * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 3600Sstevel@tonic-gate * : : 3610Sstevel@tonic-gate * : : 3620Sstevel@tonic-gate * 0xFFFFF810.00000000 -|-----------------------|- hole_end 3630Sstevel@tonic-gate * | | ^ 3640Sstevel@tonic-gate * | UltraSPARC I/II call | | 3650Sstevel@tonic-gate * | bug requires an extra | | 3660Sstevel@tonic-gate * | 4 GB of space between | | 3670Sstevel@tonic-gate * | hole and used RAM | | 3680Sstevel@tonic-gate * | | | 3690Sstevel@tonic-gate * 0xFFFFF800.00000000 -|-----------------------|- | 3700Sstevel@tonic-gate * | | | 3710Sstevel@tonic-gate * | Virtual Address Hole | UltraSPARC 3720Sstevel@tonic-gate * | on UltraSPARC I/II | I/II * ONLY * 3730Sstevel@tonic-gate * | | | 3740Sstevel@tonic-gate * 0x00000800.00000000 -|-----------------------|- | 3750Sstevel@tonic-gate * | | | 3760Sstevel@tonic-gate * | UltraSPARC I/II call | | 3770Sstevel@tonic-gate * | bug requires an extra | | 3780Sstevel@tonic-gate * | 4 GB of space between | | 3790Sstevel@tonic-gate * | hole and used RAM | | 3800Sstevel@tonic-gate * | | v 3810Sstevel@tonic-gate * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 3820Sstevel@tonic-gate * : : ^ 3830Sstevel@tonic-gate * : : | 3845648Ssetje * |-----------------------| | 3855648Ssetje * | | | 3865648Ssetje * | ecache flush area | | 3875648Ssetje * | (twice largest e$) | | 3885648Ssetje * | | | 3893764Sdp78419 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 3903764Sdp78419 * | overmapped area | alignend_end | 3913764Sdp78419 * | (kmem64_alignsize | | 3923764Sdp78419 * | boundary) | | 3930Sstevel@tonic-gate * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 3940Sstevel@tonic-gate * | | | 3950Sstevel@tonic-gate * | 64-bit kernel ONLY | | 3960Sstevel@tonic-gate * | | | 3970Sstevel@tonic-gate * | kmem64 segment | | 3980Sstevel@tonic-gate * | | | 3990Sstevel@tonic-gate * | (Relocated extra HME | Approximately 4000Sstevel@tonic-gate * | block allocations, | 1 TB of virtual 4010Sstevel@tonic-gate * | memnode freelists, | address space 4020Sstevel@tonic-gate * | HME hash buckets, | | 4030Sstevel@tonic-gate * | mml_table, kpmp_table,| | 4040Sstevel@tonic-gate * | page_t array and | | 4050Sstevel@tonic-gate * | hashblock pool to | | 4060Sstevel@tonic-gate * | avoid hard-coded | | 4070Sstevel@tonic-gate * | 32-bit vaddr | | 4080Sstevel@tonic-gate * | limitations) | | 4090Sstevel@tonic-gate * | | v 4100Sstevel@tonic-gate * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 4110Sstevel@tonic-gate * | | 4120Sstevel@tonic-gate * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 4130Sstevel@tonic-gate * | | 4140Sstevel@tonic-gate * 0x00000300.00000000 -|-----------------------|- SYSBASE 4150Sstevel@tonic-gate * : : 4160Sstevel@tonic-gate * : : 4170Sstevel@tonic-gate * -|-----------------------|- 4180Sstevel@tonic-gate * | | 4190Sstevel@tonic-gate * | segmap segment | SEGMAPSIZE (1/8th physmem, 4200Sstevel@tonic-gate * | | 256G MAX) 4210Sstevel@tonic-gate * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 4220Sstevel@tonic-gate * : : 4230Sstevel@tonic-gate * : : 4240Sstevel@tonic-gate * -|-----------------------|- 4250Sstevel@tonic-gate * | | 4260Sstevel@tonic-gate * | segkp | SEGKPSIZE (2GB) 4270Sstevel@tonic-gate * | | 4280Sstevel@tonic-gate * | | 4290Sstevel@tonic-gate * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 4300Sstevel@tonic-gate * | | 4310Sstevel@tonic-gate * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 4320Sstevel@tonic-gate * | | (SEGKPBASE - 0x400000) 4330Sstevel@tonic-gate * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 4340Sstevel@tonic-gate * | | (MEMSCRUBBASE - NCARGS) 4350Sstevel@tonic-gate * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 4360Sstevel@tonic-gate * | | (ARGSBASE - PPMAPSIZE) 4370Sstevel@tonic-gate * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 4380Sstevel@tonic-gate * | | 4390Sstevel@tonic-gate * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 4400Sstevel@tonic-gate * | | 4410Sstevel@tonic-gate * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 4420Sstevel@tonic-gate * : : 4430Sstevel@tonic-gate * : : 4440Sstevel@tonic-gate * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 4450Sstevel@tonic-gate * | | 4460Sstevel@tonic-gate * | OBP | 4470Sstevel@tonic-gate * | | 4480Sstevel@tonic-gate * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 4490Sstevel@tonic-gate * | kmdb | 4500Sstevel@tonic-gate * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 4510Sstevel@tonic-gate * : : 4520Sstevel@tonic-gate * : : 4530Sstevel@tonic-gate * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 4540Sstevel@tonic-gate * | | 4550Sstevel@tonic-gate * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 4560Sstevel@tonic-gate * | | ~64MB) 4575648Ssetje * 0x00000000.70002000 -|-----------------------| 4580Sstevel@tonic-gate * | panicbuf | 4595648Ssetje * 0x00000000.70000000 -|-----------------------|- SYSBASE32 4605648Ssetje * | boot-time | 4615648Ssetje * | temporary space | 4625648Ssetje * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 4630Sstevel@tonic-gate * : : 4640Sstevel@tonic-gate * : : 4650Sstevel@tonic-gate * | | 4660Sstevel@tonic-gate * |-----------------------|- econtig32 4670Sstevel@tonic-gate * | vm structures | 4680Sstevel@tonic-gate * 0x00000000.01C00000 |-----------------------|- nalloc_end 4690Sstevel@tonic-gate * | TSBs | 4700Sstevel@tonic-gate * |-----------------------|- end/nalloc_base 4710Sstevel@tonic-gate * | kernel data & bss | 4720Sstevel@tonic-gate * 0x00000000.01800000 -|-----------------------| 4730Sstevel@tonic-gate * : nucleus text hole : 4740Sstevel@tonic-gate * 0x00000000.01400000 -|-----------------------| 4750Sstevel@tonic-gate * : : 4760Sstevel@tonic-gate * |-----------------------| 4770Sstevel@tonic-gate * | module text | 4780Sstevel@tonic-gate * |-----------------------|- e_text/modtext 4790Sstevel@tonic-gate * | kernel text | 4800Sstevel@tonic-gate * |-----------------------| 4810Sstevel@tonic-gate * | trap table (48k) | 4820Sstevel@tonic-gate * 0x00000000.01000000 -|-----------------------|- KERNELBASE 4830Sstevel@tonic-gate * | reserved for trapstat |} TSTAT_TOTAL_SIZE 4840Sstevel@tonic-gate * |-----------------------| 4850Sstevel@tonic-gate * | | 4860Sstevel@tonic-gate * | invalid | 4870Sstevel@tonic-gate * | | 4880Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 4890Sstevel@tonic-gate * 4900Sstevel@tonic-gate * 4910Sstevel@tonic-gate * 4920Sstevel@tonic-gate * 32-bit User Virtual Memory Layout. 4930Sstevel@tonic-gate * /-----------------------\ 4940Sstevel@tonic-gate * | | 4950Sstevel@tonic-gate * | invalid | 4960Sstevel@tonic-gate * | | 4970Sstevel@tonic-gate * 0xFFC00000 -|-----------------------|- USERLIMIT 4980Sstevel@tonic-gate * | user stack | 4990Sstevel@tonic-gate * : : 5000Sstevel@tonic-gate * : : 5010Sstevel@tonic-gate * : : 5020Sstevel@tonic-gate * | user data | 5030Sstevel@tonic-gate * -|-----------------------|- 5040Sstevel@tonic-gate * | user text | 5050Sstevel@tonic-gate * 0x00002000 -|-----------------------|- 5060Sstevel@tonic-gate * | invalid | 5070Sstevel@tonic-gate * 0x00000000 _|_______________________| 5080Sstevel@tonic-gate * 5090Sstevel@tonic-gate * 5100Sstevel@tonic-gate * 5110Sstevel@tonic-gate * 64-bit User Virtual Memory Layout. 5120Sstevel@tonic-gate * /-----------------------\ 5130Sstevel@tonic-gate * | | 5140Sstevel@tonic-gate * | invalid | 5150Sstevel@tonic-gate * | | 5160Sstevel@tonic-gate * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 5170Sstevel@tonic-gate * | user stack | 5180Sstevel@tonic-gate * : : 5190Sstevel@tonic-gate * : : 5200Sstevel@tonic-gate * : : 5210Sstevel@tonic-gate * | user data | 5220Sstevel@tonic-gate * -|-----------------------|- 5230Sstevel@tonic-gate * | user text | 5245648Ssetje * 0x00000000.01000000 -|-----------------------|- 5250Sstevel@tonic-gate * | invalid | 5260Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 5270Sstevel@tonic-gate */ 5280Sstevel@tonic-gate 5290Sstevel@tonic-gate extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 5300Sstevel@tonic-gate extern uint64_t ecache_flush_address(void); 5310Sstevel@tonic-gate 5320Sstevel@tonic-gate #pragma weak load_platform_modules 5331772Sjl139090 #pragma weak plat_startup_memlist 5340Sstevel@tonic-gate #pragma weak ecache_init_scrub_flush_area 5350Sstevel@tonic-gate #pragma weak ecache_flush_address 5360Sstevel@tonic-gate 5370Sstevel@tonic-gate 5380Sstevel@tonic-gate /* 5390Sstevel@tonic-gate * By default the DR Cage is enabled for maximum OS 5400Sstevel@tonic-gate * MPSS performance. Users needing to disable the cage mechanism 5410Sstevel@tonic-gate * can set this variable to zero via /etc/system. 5420Sstevel@tonic-gate * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 5430Sstevel@tonic-gate * will result in loss of DR functionality. 5440Sstevel@tonic-gate * Platforms wishing to disable kernel Cage by default 5450Sstevel@tonic-gate * should do so in their set_platform_defaults() routine. 5460Sstevel@tonic-gate */ 5470Sstevel@tonic-gate int kernel_cage_enable = 1; 5480Sstevel@tonic-gate 5490Sstevel@tonic-gate static void 5500Sstevel@tonic-gate setup_cage_params(void) 5510Sstevel@tonic-gate { 5520Sstevel@tonic-gate void (*func)(void); 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 5550Sstevel@tonic-gate if (func != NULL) { 5560Sstevel@tonic-gate (*func)(); 5570Sstevel@tonic-gate return; 5580Sstevel@tonic-gate } 5590Sstevel@tonic-gate 5600Sstevel@tonic-gate if (kernel_cage_enable == 0) { 5610Sstevel@tonic-gate return; 5620Sstevel@tonic-gate } 5634266Sdp78419 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 5640Sstevel@tonic-gate 5650Sstevel@tonic-gate if (kcage_on) { 5660Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 5670Sstevel@tonic-gate } else { 5680Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 5690Sstevel@tonic-gate } 5700Sstevel@tonic-gate 5710Sstevel@tonic-gate } 5720Sstevel@tonic-gate 5730Sstevel@tonic-gate /* 5740Sstevel@tonic-gate * Machine-dependent startup code 5750Sstevel@tonic-gate */ 5760Sstevel@tonic-gate void 5770Sstevel@tonic-gate startup(void) 5780Sstevel@tonic-gate { 5790Sstevel@tonic-gate startup_init(); 5800Sstevel@tonic-gate if (&startup_platform) 5810Sstevel@tonic-gate startup_platform(); 5820Sstevel@tonic-gate startup_memlist(); 5830Sstevel@tonic-gate startup_modules(); 5840Sstevel@tonic-gate setup_cage_params(); 5850Sstevel@tonic-gate startup_bop_gone(); 5860Sstevel@tonic-gate startup_vm(); 5870Sstevel@tonic-gate startup_end(); 5880Sstevel@tonic-gate } 5890Sstevel@tonic-gate 5900Sstevel@tonic-gate struct regs sync_reg_buf; 5910Sstevel@tonic-gate uint64_t sync_tt; 5920Sstevel@tonic-gate 5930Sstevel@tonic-gate void 5940Sstevel@tonic-gate sync_handler(void) 5950Sstevel@tonic-gate { 5965084Sjohnlev struct panic_trap_info ti; 5970Sstevel@tonic-gate int i; 5980Sstevel@tonic-gate 5990Sstevel@tonic-gate /* 6000Sstevel@tonic-gate * Prevent trying to talk to the other CPUs since they are 6010Sstevel@tonic-gate * sitting in the prom and won't reply. 6020Sstevel@tonic-gate */ 6030Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 6040Sstevel@tonic-gate if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 6050Sstevel@tonic-gate cpu[i]->cpu_flags &= ~CPU_READY; 6060Sstevel@tonic-gate cpu[i]->cpu_flags |= CPU_QUIESCED; 6070Sstevel@tonic-gate CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 6080Sstevel@tonic-gate } 6090Sstevel@tonic-gate } 6100Sstevel@tonic-gate 6110Sstevel@tonic-gate /* 6120Sstevel@tonic-gate * We've managed to get here without going through the 6130Sstevel@tonic-gate * normal panic code path. Try and save some useful 6140Sstevel@tonic-gate * information. 6150Sstevel@tonic-gate */ 6160Sstevel@tonic-gate if (!panicstr && (curthread->t_panic_trap == NULL)) { 6170Sstevel@tonic-gate ti.trap_type = sync_tt; 6180Sstevel@tonic-gate ti.trap_regs = &sync_reg_buf; 6190Sstevel@tonic-gate ti.trap_addr = NULL; 6200Sstevel@tonic-gate ti.trap_mmu_fsr = 0x0; 6210Sstevel@tonic-gate 6220Sstevel@tonic-gate curthread->t_panic_trap = &ti; 6230Sstevel@tonic-gate } 6240Sstevel@tonic-gate 6250Sstevel@tonic-gate /* 6260Sstevel@tonic-gate * If we're re-entering the panic path, update the signature 6270Sstevel@tonic-gate * block so that the SC knows we're in the second part of panic. 6280Sstevel@tonic-gate */ 6290Sstevel@tonic-gate if (panicstr) 6300Sstevel@tonic-gate CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 6310Sstevel@tonic-gate 6320Sstevel@tonic-gate nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 6330Sstevel@tonic-gate panic("sync initiated"); 6340Sstevel@tonic-gate } 6350Sstevel@tonic-gate 6360Sstevel@tonic-gate 6370Sstevel@tonic-gate static void 6380Sstevel@tonic-gate startup_init(void) 6390Sstevel@tonic-gate { 6400Sstevel@tonic-gate /* 6410Sstevel@tonic-gate * We want to save the registers while we're still in OBP 6420Sstevel@tonic-gate * so that we know they haven't been fiddled with since. 6430Sstevel@tonic-gate * (In principle, OBP can't change them just because it 6440Sstevel@tonic-gate * makes a callback, but we'd rather not depend on that 6450Sstevel@tonic-gate * behavior.) 6460Sstevel@tonic-gate */ 6470Sstevel@tonic-gate char sync_str[] = 6484528Spaulsan "warning @ warning off : sync " 6494528Spaulsan "%%tl-c %%tstate h# %p x! " 6504528Spaulsan "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 6514528Spaulsan "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 6524528Spaulsan "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 6534528Spaulsan "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 6544528Spaulsan "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 6554528Spaulsan "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 6564528Spaulsan "%%y h# %p l! %%tl-c %%tt h# %p x! " 6574528Spaulsan "sync ; warning !"; 6580Sstevel@tonic-gate 6590Sstevel@tonic-gate /* 6600Sstevel@tonic-gate * 20 == num of %p substrings 6610Sstevel@tonic-gate * 16 == max num of chars %p will expand to. 6620Sstevel@tonic-gate */ 6630Sstevel@tonic-gate char bp[sizeof (sync_str) + 16 * 20]; 6640Sstevel@tonic-gate 6650Sstevel@tonic-gate /* 6660Sstevel@tonic-gate * Initialize ptl1 stack for the 1st CPU. 6670Sstevel@tonic-gate */ 6680Sstevel@tonic-gate ptl1_init_cpu(&cpu0); 6690Sstevel@tonic-gate 6700Sstevel@tonic-gate /* 6710Sstevel@tonic-gate * Initialize the address map for cache consistent mappings 6720Sstevel@tonic-gate * to random pages; must be done after vac_size is set. 6730Sstevel@tonic-gate */ 6740Sstevel@tonic-gate ppmapinit(); 6750Sstevel@tonic-gate 6760Sstevel@tonic-gate /* 6770Sstevel@tonic-gate * Initialize the PROM callback handler. 6780Sstevel@tonic-gate */ 6790Sstevel@tonic-gate init_vx_handler(); 6800Sstevel@tonic-gate 6810Sstevel@tonic-gate /* 6820Sstevel@tonic-gate * have prom call sync_callback() to handle the sync and 6830Sstevel@tonic-gate * save some useful information which will be stored in the 6840Sstevel@tonic-gate * core file later. 6850Sstevel@tonic-gate */ 6860Sstevel@tonic-gate (void) sprintf((char *)bp, sync_str, 6874528Spaulsan (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 6884528Spaulsan (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 6894528Spaulsan (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 6904528Spaulsan (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 6914528Spaulsan (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 6924528Spaulsan (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 6934528Spaulsan (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 6944528Spaulsan (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 6954528Spaulsan (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 6964528Spaulsan (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 6970Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 6980Sstevel@tonic-gate add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 6990Sstevel@tonic-gate } 7000Sstevel@tonic-gate 7015648Ssetje 7025648Ssetje size_t 7035648Ssetje calc_pp_sz(pgcnt_t npages) 7045648Ssetje { 7055648Ssetje 7065648Ssetje return (npages * sizeof (struct page)); 7075648Ssetje } 7085648Ssetje 7095648Ssetje size_t 7105648Ssetje calc_kpmpp_sz(pgcnt_t npages) 7115648Ssetje { 7125648Ssetje 7135648Ssetje kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 7145648Ssetje kpm_pgsz = 1ull << kpm_pgshft; 7155648Ssetje kpm_pgoff = kpm_pgsz - 1; 7165648Ssetje kpmp2pshft = kpm_pgshft - PAGESHIFT; 7175648Ssetje kpmpnpgs = 1 << kpmp2pshft; 7185648Ssetje 7195648Ssetje if (kpm_smallpages == 0) { 7205648Ssetje /* 7215648Ssetje * Avoid fragmentation problems in kphysm_init() 7225648Ssetje * by allocating for all of physical memory 7235648Ssetje */ 7245648Ssetje kpm_npages = ptokpmpr(physinstalled); 7255648Ssetje return (kpm_npages * sizeof (kpm_page_t)); 7265648Ssetje } else { 7275648Ssetje kpm_npages = npages; 7285648Ssetje return (kpm_npages * sizeof (kpm_spage_t)); 7295648Ssetje } 7305648Ssetje } 7315648Ssetje 7325648Ssetje size_t 7335648Ssetje calc_pagehash_sz(pgcnt_t npages) 7345648Ssetje { 7355648Ssetje 7365648Ssetje /* 7375648Ssetje * The page structure hash table size is a power of 2 7385648Ssetje * such that the average hash chain length is PAGE_HASHAVELEN. 7395648Ssetje */ 7405648Ssetje page_hashsz = npages / PAGE_HASHAVELEN; 7415648Ssetje page_hashsz = 1 << highbit(page_hashsz); 7425648Ssetje return (page_hashsz * sizeof (struct page *)); 7435648Ssetje } 7445648Ssetje 7457218Ssvemuri int testkmem64_smchunks = 0; 7467218Ssvemuri 7477218Ssvemuri int 7485648Ssetje alloc_kmem64(caddr_t base, caddr_t end) 7495648Ssetje { 7505648Ssetje int i; 7515648Ssetje caddr_t aligned_end = NULL; 7525648Ssetje 7537218Ssvemuri if (testkmem64_smchunks) 7547218Ssvemuri return (1); 7557218Ssvemuri 7565648Ssetje /* 7575648Ssetje * Make one large memory alloc after figuring out the 64-bit size. This 7585648Ssetje * will enable use of the largest page size appropriate for the system 7595648Ssetje * architecture. 7605648Ssetje */ 7615648Ssetje ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 7625648Ssetje ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 7635648Ssetje for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 7645648Ssetje size_t alloc_size, alignsize; 7655648Ssetje #if !defined(C_OBP) 7665648Ssetje unsigned long long pa; 7675648Ssetje #endif /* !C_OBP */ 7685648Ssetje 7695648Ssetje if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 7705648Ssetje continue; 7715648Ssetje alignsize = TTEBYTES(i); 7725648Ssetje kmem64_szc = i; 7735648Ssetje 7745648Ssetje /* limit page size for small memory */ 7755648Ssetje if (mmu_btop(alignsize) > (npages >> 2)) 7765648Ssetje continue; 7775648Ssetje 7785648Ssetje aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 7795648Ssetje alloc_size = aligned_end - base; 7805648Ssetje #if !defined(C_OBP) 7815648Ssetje if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 7825648Ssetje if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 7835648Ssetje kmem64_pabase = pa; 7845648Ssetje kmem64_aligned_end = aligned_end; 7855648Ssetje install_kmem64_tte(); 7865648Ssetje break; 7875648Ssetje } else { 7885648Ssetje prom_free_phys(alloc_size, pa); 7895648Ssetje } 7905648Ssetje } 7915648Ssetje #else /* !C_OBP */ 7925648Ssetje if (prom_alloc(base, alloc_size, alignsize) == base) { 7935648Ssetje kmem64_pabase = va_to_pa(kmem64_base); 7945648Ssetje kmem64_aligned_end = aligned_end; 7955648Ssetje break; 7965648Ssetje } 7975648Ssetje #endif /* !C_OBP */ 7985648Ssetje if (i == TTE8K) { 7997218Ssvemuri #ifdef sun4v 8007218Ssvemuri /* return failure to try small allocations */ 8017218Ssvemuri return (1); 8027218Ssvemuri #else 8035648Ssetje prom_panic("kmem64 allocation failure"); 8047218Ssvemuri #endif 8055648Ssetje } 8065648Ssetje } 8075648Ssetje ASSERT(aligned_end != NULL); 8087218Ssvemuri return (0); 8095648Ssetje } 8105648Ssetje 8115648Ssetje static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 8120Sstevel@tonic-gate static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 8130Sstevel@tonic-gate 8145648Ssetje #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 8155648Ssetje (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 8165648Ssetje (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 8170Sstevel@tonic-gate 8183764Sdp78419 #if !defined(C_OBP) 8193764Sdp78419 /* 8203764Sdp78419 * Install a temporary tte handler in OBP for kmem64 area. 8213764Sdp78419 * 8223764Sdp78419 * We map kmem64 area with large pages before the trap table is taken 8233764Sdp78419 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 8243764Sdp78419 * the same area. Duplicate tlb entries with different page sizes 8253764Sdp78419 * cause unpredicatble behavior. To avoid this, we don't create 8263764Sdp78419 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 8273764Sdp78419 * OBP). Instead, we manage translations with a temporary va>tte-data 8283764Sdp78419 * handler (kmem64-tte). This handler is replaced by unix-tte when 8293764Sdp78419 * the trap table is taken over. 8303764Sdp78419 * 8313764Sdp78419 * The temporary handler knows the physical address of the kmem64 8323764Sdp78419 * area. It uses the prom's pgmap@ Forth word for other addresses. 8333764Sdp78419 * 8343764Sdp78419 * We have to use BOP_ALLOC() method for C-OBP platforms because 8353764Sdp78419 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 8363764Sdp78419 * sun4u platforms. On sun4u we flush tlb after trap table is taken 8373764Sdp78419 * over if we use large pages for kernel heap and kmem64. Since sun4u 8383764Sdp78419 * prom (unlike sun4v) calls va>tte-data first for client address 8393764Sdp78419 * translation prom's ttes for kmem64 can't get into TLB even if we 8403764Sdp78419 * later switch to prom's trap table again. C-OBP uses 4M pages for 8413764Sdp78419 * client mappings when possible so on all platforms we get the 8423764Sdp78419 * benefit from large mappings for kmem64 area immediately during 8433764Sdp78419 * boot. 8443764Sdp78419 * 8453764Sdp78419 * pseudo code: 8463764Sdp78419 * if (context != 0) { 8473764Sdp78419 * return false 8483764Sdp78419 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 8493764Sdp78419 * tte = tte_template + 8503764Sdp78419 * (((miss_va & pagemask) - kmem64_base)); 8513764Sdp78419 * return tte, true 8523764Sdp78419 * } else { 8533764Sdp78419 * return pgmap@ result 8543764Sdp78419 * } 8553764Sdp78419 */ 8563764Sdp78419 char kmem64_obp_str[] = 8575648Ssetje "h# %lx constant kmem64-base " 8585648Ssetje "h# %lx constant kmem64-end " 8595648Ssetje "h# %lx constant kmem64-pagemask " 8605648Ssetje "h# %lx constant kmem64-template " 8613764Sdp78419 8623764Sdp78419 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 8633764Sdp78419 " if ( addr ) " 8643764Sdp78419 " drop false exit then ( false ) " 8655648Ssetje " dup kmem64-base kmem64-end within if ( addr ) " 8665648Ssetje " kmem64-pagemask and ( addr' ) " 8675648Ssetje " kmem64-base - ( addr' ) " 8685648Ssetje " kmem64-template + ( tte ) " 8693764Sdp78419 " true ( tte true ) " 8703764Sdp78419 " else ( addr ) " 8713764Sdp78419 " pgmap@ ( tte ) " 8723764Sdp78419 " dup 0< if true else drop false then ( tte true | false ) " 8733764Sdp78419 " then ( tte true | false ) " 8743764Sdp78419 "; " 8753764Sdp78419 8763764Sdp78419 "' kmem64-tte is va>tte-data " 8773764Sdp78419 ; 8783764Sdp78419 8795648Ssetje static void 8803764Sdp78419 install_kmem64_tte() 8813764Sdp78419 { 8823764Sdp78419 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 8833764Sdp78419 tte_t tte; 8843764Sdp78419 8853764Sdp78419 PRM_DEBUG(kmem64_pabase); 8863764Sdp78419 PRM_DEBUG(kmem64_szc); 8873764Sdp78419 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 8883764Sdp78419 PROC_DATA | HAT_NOSYNC, kmem64_szc); 8893764Sdp78419 PRM_DEBUG(tte.ll); 8903764Sdp78419 (void) sprintf(b, kmem64_obp_str, 8913764Sdp78419 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 8923764Sdp78419 ASSERT(strlen(b) < sizeof (b)); 8933764Sdp78419 prom_interpret(b, 0, 0, 0, 0, 0); 8943764Sdp78419 } 8953764Sdp78419 #endif /* !C_OBP */ 8963764Sdp78419 8970Sstevel@tonic-gate /* 8980Sstevel@tonic-gate * As OBP takes up some RAM when the system boots, pages will already be "lost" 8990Sstevel@tonic-gate * to the system and reflected in npages by the time we see it. 9000Sstevel@tonic-gate * 9010Sstevel@tonic-gate * We only want to allocate kernel structures in the 64-bit virtual address 9020Sstevel@tonic-gate * space on systems with enough RAM to make the overhead of keeping track of 9030Sstevel@tonic-gate * an extra kernel memory segment worthwhile. 9040Sstevel@tonic-gate * 9050Sstevel@tonic-gate * Since OBP has already performed its memory allocations by this point, if we 9060Sstevel@tonic-gate * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 9070Sstevel@tonic-gate * memory in the 64-bit virtual address space; otherwise keep allocations 9080Sstevel@tonic-gate * contiguous with we've mapped so far in the 32-bit virtual address space. 9090Sstevel@tonic-gate */ 9100Sstevel@tonic-gate #define MINMOVE_RAM_MB ((size_t)1900) 9110Sstevel@tonic-gate #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 9125872Ssetje #define BYTES_TO_MB(b) ((b) / 1048576ul) 9130Sstevel@tonic-gate 9140Sstevel@tonic-gate pgcnt_t tune_npages = (pgcnt_t) 9150Sstevel@tonic-gate (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 9160Sstevel@tonic-gate 9173733Sdp78419 #pragma weak page_set_colorequiv_arr_cpu 9183733Sdp78419 extern void page_set_colorequiv_arr_cpu(void); 9195648Ssetje extern void page_set_colorequiv_arr(void); 9205648Ssetje 9215872Ssetje static pgcnt_t ramdisk_npages; 9225872Ssetje static struct memlist *old_phys_avail; 9235872Ssetje 9245872Ssetje kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 9253733Sdp78419 9260Sstevel@tonic-gate static void 9270Sstevel@tonic-gate startup_memlist(void) 9280Sstevel@tonic-gate { 9295648Ssetje size_t hmehash_sz, pagelist_sz, tt_sz; 9305648Ssetje size_t psetable_sz; 9310Sstevel@tonic-gate caddr_t alloc_base; 9320Sstevel@tonic-gate caddr_t memspace; 9330Sstevel@tonic-gate struct memlist *cur; 9340Sstevel@tonic-gate size_t syslimit = (size_t)SYSLIMIT; 9350Sstevel@tonic-gate size_t sysbase = (size_t)SYSBASE; 9360Sstevel@tonic-gate 9370Sstevel@tonic-gate /* 9380Sstevel@tonic-gate * Initialize enough of the system to allow kmem_alloc to work by 9390Sstevel@tonic-gate * calling boot to allocate its memory until the time that 9400Sstevel@tonic-gate * kvm_init is completed. The page structs are allocated after 9410Sstevel@tonic-gate * rounding up end to the nearest page boundary; the memsegs are 9420Sstevel@tonic-gate * initialized and the space they use comes from the kernel heap. 9430Sstevel@tonic-gate * With appropriate initialization, they can be reallocated later 9440Sstevel@tonic-gate * to a size appropriate for the machine's configuration. 9450Sstevel@tonic-gate * 9460Sstevel@tonic-gate * At this point, memory is allocated for things that will never 9470Sstevel@tonic-gate * need to be freed, this used to be "valloced". This allows a 9480Sstevel@tonic-gate * savings as the pages don't need page structures to describe 9490Sstevel@tonic-gate * them because them will not be managed by the vm system. 9500Sstevel@tonic-gate */ 9510Sstevel@tonic-gate 9520Sstevel@tonic-gate /* 9530Sstevel@tonic-gate * We're loaded by boot with the following configuration (as 9540Sstevel@tonic-gate * specified in the sun4u/conf/Mapfile): 9550Sstevel@tonic-gate * 9560Sstevel@tonic-gate * text: 4 MB chunk aligned on a 4MB boundary 9570Sstevel@tonic-gate * data & bss: 4 MB chunk aligned on a 4MB boundary 9580Sstevel@tonic-gate * 9590Sstevel@tonic-gate * These two chunks will eventually be mapped by 2 locked 4MB 9600Sstevel@tonic-gate * ttes and will represent the nucleus of the kernel. This gives 9610Sstevel@tonic-gate * us some free space that is already allocated, some or all of 9620Sstevel@tonic-gate * which is made available to kernel module text. 9630Sstevel@tonic-gate * 9640Sstevel@tonic-gate * The free space in the data-bss chunk is used for nucleus 9650Sstevel@tonic-gate * allocatable data structures and we reserve it using the 9660Sstevel@tonic-gate * nalloc_base and nalloc_end variables. This space is currently 9670Sstevel@tonic-gate * being used for hat data structures required for tlb miss 9680Sstevel@tonic-gate * handling operations. We align nalloc_base to a l2 cache 9690Sstevel@tonic-gate * linesize because this is the line size the hardware uses to 9700Sstevel@tonic-gate * maintain cache coherency. 9715648Ssetje * 512K is carved out for module data. 9720Sstevel@tonic-gate */ 9730Sstevel@tonic-gate 9745648Ssetje moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 9755648Ssetje e_moddata = moddata + MODDATA; 9760Sstevel@tonic-gate nalloc_base = e_moddata; 9770Sstevel@tonic-gate 9780Sstevel@tonic-gate nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 9790Sstevel@tonic-gate valloc_base = nalloc_base; 9800Sstevel@tonic-gate 9810Sstevel@tonic-gate /* 9820Sstevel@tonic-gate * Calculate the start of the data segment. 9830Sstevel@tonic-gate */ 9845648Ssetje if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 9855648Ssetje prom_panic("nucleus data overflow"); 9860Sstevel@tonic-gate 9870Sstevel@tonic-gate PRM_DEBUG(moddata); 9880Sstevel@tonic-gate PRM_DEBUG(nalloc_base); 9890Sstevel@tonic-gate PRM_DEBUG(nalloc_end); 9900Sstevel@tonic-gate 9910Sstevel@tonic-gate /* 9920Sstevel@tonic-gate * Remember any slop after e_text so we can give it to the modules. 9930Sstevel@tonic-gate */ 9940Sstevel@tonic-gate PRM_DEBUG(e_text); 9950Sstevel@tonic-gate modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 9963791Skchow if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 9973764Sdp78419 prom_panic("nucleus text overflow"); 9980Sstevel@tonic-gate modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 9990Sstevel@tonic-gate modtext; 10000Sstevel@tonic-gate PRM_DEBUG(modtext); 10010Sstevel@tonic-gate PRM_DEBUG(modtext_sz); 10020Sstevel@tonic-gate 10035648Ssetje init_boot_memlists(); 10040Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 10050Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 10060Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 10075648Ssetje 10080Sstevel@tonic-gate /* 10090Sstevel@tonic-gate * Remember what the physically available highest page is 10100Sstevel@tonic-gate * so that dumpsys works properly, and find out how much 10110Sstevel@tonic-gate * memory is installed. 10120Sstevel@tonic-gate */ 10130Sstevel@tonic-gate installed_top_size_memlist_array(boot_physinstalled, 10140Sstevel@tonic-gate boot_physinstalled_len, &physmax, &physinstalled); 10150Sstevel@tonic-gate PRM_DEBUG(physinstalled); 10160Sstevel@tonic-gate PRM_DEBUG(physmax); 10170Sstevel@tonic-gate 10180Sstevel@tonic-gate /* Fill out memory nodes config structure */ 10190Sstevel@tonic-gate startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 10200Sstevel@tonic-gate 10210Sstevel@tonic-gate /* 10220Sstevel@tonic-gate * npages is the maximum of available physical memory possible. 10230Sstevel@tonic-gate * (ie. it will never be more than this) 10245648Ssetje * 10255648Ssetje * When we boot from a ramdisk, the ramdisk memory isn't free, so 10265648Ssetje * using phys_avail will underestimate what will end up being freed. 10275648Ssetje * A better initial guess is just total memory minus the kernel text 10280Sstevel@tonic-gate */ 10295648Ssetje npages = physinstalled - btop(MMU_PAGESIZE4M); 10300Sstevel@tonic-gate 10310Sstevel@tonic-gate /* 10325648Ssetje * First allocate things that can go in the nucleus data page 10335648Ssetje * (fault status, TSBs, dmv, CPUs) 10340Sstevel@tonic-gate */ 10355648Ssetje ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 10365648Ssetje 10370Sstevel@tonic-gate if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 10380Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 10390Sstevel@tonic-gate 10400Sstevel@tonic-gate if (ndata_alloc_tsbs(&ndata, npages) != 0) 10410Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 10420Sstevel@tonic-gate 10430Sstevel@tonic-gate if (ndata_alloc_dmv(&ndata) != 0) 10440Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 10450Sstevel@tonic-gate 10465648Ssetje if (ndata_alloc_page_mutexs(&ndata) != 0) 10470Sstevel@tonic-gate cmn_err(CE_PANIC, 10480Sstevel@tonic-gate "no more nucleus memory after page free lists alloc"); 10490Sstevel@tonic-gate 10505648Ssetje if (ndata_alloc_hat(&ndata, npages) != 0) 10510Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 10520Sstevel@tonic-gate 10535648Ssetje if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 10545648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 10550Sstevel@tonic-gate 10560Sstevel@tonic-gate /* 10570Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 10580Sstevel@tonic-gate * 10590Sstevel@tonic-gate * There are comments all over the SFMMU code warning of dire 10600Sstevel@tonic-gate * consequences if the TSBs are moved out of 32-bit space. This 10610Sstevel@tonic-gate * is largely because the asm code uses "sethi %hi(addr)"-type 10620Sstevel@tonic-gate * instructions which will not provide the expected result if the 10630Sstevel@tonic-gate * address is a 64-bit one. 10640Sstevel@tonic-gate * 10650Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 10660Sstevel@tonic-gate */ 10670Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 10685648Ssetje PRM_DEBUG(alloc_base); 10695648Ssetje 10700Sstevel@tonic-gate alloc_base = sfmmu_ktsb_alloc(alloc_base); 10710Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 10720Sstevel@tonic-gate PRM_DEBUG(alloc_base); 10730Sstevel@tonic-gate 10740Sstevel@tonic-gate /* 10750Sstevel@tonic-gate * Allocate IOMMU TSB array. We do this here so that the physical 10760Sstevel@tonic-gate * memory gets deducted from the PROM's physical memory list. 10770Sstevel@tonic-gate */ 10780Sstevel@tonic-gate alloc_base = iommu_tsb_init(alloc_base); 10795648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 10800Sstevel@tonic-gate PRM_DEBUG(alloc_base); 10810Sstevel@tonic-gate 10820Sstevel@tonic-gate /* 10835648Ssetje * Allow for an early allocation of physically contiguous memory. 10845648Ssetje */ 10855648Ssetje alloc_base = contig_mem_prealloc(alloc_base, npages); 10865648Ssetje 10875648Ssetje /* 10881772Sjl139090 * Platforms like Starcat and OPL need special structures assigned in 10891772Sjl139090 * 32-bit virtual address space because their probing routines execute 10901772Sjl139090 * FCode, and FCode can't handle 64-bit virtual addresses... 10910Sstevel@tonic-gate */ 10921772Sjl139090 if (&plat_startup_memlist) { 10931772Sjl139090 alloc_base = plat_startup_memlist(alloc_base); 10940Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 10950Sstevel@tonic-gate ecache_alignsize); 10960Sstevel@tonic-gate PRM_DEBUG(alloc_base); 10970Sstevel@tonic-gate } 10980Sstevel@tonic-gate 10990Sstevel@tonic-gate /* 11000Sstevel@tonic-gate * Save off where the contiguous allocations to date have ended 11010Sstevel@tonic-gate * in econtig32. 11020Sstevel@tonic-gate */ 11030Sstevel@tonic-gate econtig32 = alloc_base; 11040Sstevel@tonic-gate PRM_DEBUG(econtig32); 11050Sstevel@tonic-gate if (econtig32 > (caddr_t)KERNEL_LIMIT32) 11060Sstevel@tonic-gate cmn_err(CE_PANIC, "econtig32 too big"); 11070Sstevel@tonic-gate 11085648Ssetje pp_sz = calc_pp_sz(npages); 11095648Ssetje PRM_DEBUG(pp_sz); 11105648Ssetje if (kpm_enable) { 11115648Ssetje kpm_pp_sz = calc_kpmpp_sz(npages); 11125648Ssetje PRM_DEBUG(kpm_pp_sz); 11135648Ssetje } 11145648Ssetje 11155648Ssetje hmehash_sz = calc_hmehash_sz(npages); 11165648Ssetje PRM_DEBUG(hmehash_sz); 11175648Ssetje 11185648Ssetje pagehash_sz = calc_pagehash_sz(npages); 11195648Ssetje PRM_DEBUG(pagehash_sz); 11205648Ssetje 11215648Ssetje pagelist_sz = calc_free_pagelist_sz(); 11225648Ssetje PRM_DEBUG(pagelist_sz); 11235648Ssetje 11245648Ssetje #ifdef TRAPTRACE 11255648Ssetje tt_sz = calc_traptrace_sz(); 11265648Ssetje PRM_DEBUG(tt_sz); 11275648Ssetje #else 11285648Ssetje tt_sz = 0; 11295648Ssetje #endif /* TRAPTRACE */ 11300Sstevel@tonic-gate 11313764Sdp78419 /* 11325648Ssetje * Place the array that protects pp->p_selock in the kmem64 wad. 11333764Sdp78419 */ 11346582Ssetje pse_shift = size_pse_array(npages, max_ncpus); 11355648Ssetje PRM_DEBUG(pse_shift); 11365648Ssetje pse_table_size = 1 << pse_shift; 11375648Ssetje PRM_DEBUG(pse_table_size); 11385648Ssetje psetable_sz = roundup( 11395648Ssetje pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 11405648Ssetje PRM_DEBUG(psetable_sz); 11410Sstevel@tonic-gate 11420Sstevel@tonic-gate /* 11435648Ssetje * Now allocate the whole wad 11440Sstevel@tonic-gate */ 11455648Ssetje kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 11465648Ssetje pagelist_sz + tt_sz + psetable_sz; 11475648Ssetje kmem64_sz = roundup(kmem64_sz, PAGESIZE); 11485648Ssetje kmem64_base = (caddr_t)syslimit; 11495648Ssetje kmem64_end = kmem64_base + kmem64_sz; 11507218Ssvemuri if (alloc_kmem64(kmem64_base, kmem64_end)) { 11517218Ssvemuri /* 11527218Ssvemuri * Attempt for kmem64 to allocate one big 11537218Ssvemuri * contiguous chunk of memory failed. 11547218Ssvemuri * We get here because we are sun4v. 11557218Ssvemuri * We will proceed by breaking up 11567218Ssvemuri * the allocation into two attempts. 11577218Ssvemuri * First, we allocate kpm_pp_sz, hmehash_sz, 11587218Ssvemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 11597218Ssvemuri * one contiguous chunk. This is a much smaller 11607218Ssvemuri * chunk and we should get it, if not we panic. 11617218Ssvemuri * Note that hmehash and tt need to be physically 11627218Ssvemuri * (in the real address sense) contiguous. 11637218Ssvemuri * Next, we use bop_alloc_chunk() to 11647218Ssvemuri * to allocate the page_t structures. 11657218Ssvemuri * This will allow the page_t to be allocated 11667218Ssvemuri * in multiple smaller chunks. 11677218Ssvemuri * In doing so, the assumption that page_t is 11687218Ssvemuri * physically contiguous no longer hold, this is ok 11697218Ssvemuri * for sun4v but not for sun4u. 11707218Ssvemuri */ 11717218Ssvemuri size_t tmp_size; 11727218Ssvemuri caddr_t tmp_base; 11737218Ssvemuri 11747218Ssvemuri pp_sz = roundup(pp_sz, PAGESIZE); 11757218Ssvemuri 11767218Ssvemuri /* 11777218Ssvemuri * Allocate kpm_pp_sz, hmehash_sz, 11787218Ssvemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 11797218Ssvemuri */ 11807218Ssvemuri tmp_base = kmem64_base + pp_sz; 11817218Ssvemuri tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 11827218Ssvemuri pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 11837218Ssvemuri if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 11847218Ssvemuri prom_panic("kmem64 prom_alloc contig failed"); 11857218Ssvemuri PRM_DEBUG(tmp_base); 11867218Ssvemuri PRM_DEBUG(tmp_size); 11877218Ssvemuri 11887218Ssvemuri /* 11897218Ssvemuri * Allocate the page_ts 11907218Ssvemuri */ 11917218Ssvemuri if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 11927218Ssvemuri prom_panic("kmem64 bop_alloc_chunk page_t failed"); 11937218Ssvemuri PRM_DEBUG(kmem64_base); 11947218Ssvemuri PRM_DEBUG(pp_sz); 11957218Ssvemuri 11967218Ssvemuri kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 11977218Ssvemuri ASSERT(kmem64_aligned_end >= kmem64_end); 11987218Ssvemuri 11997218Ssvemuri kmem64_smchunks = 1; 12007218Ssvemuri } else { 12017218Ssvemuri 12027218Ssvemuri /* 12037218Ssvemuri * We need to adjust pp_sz for the normal 12047218Ssvemuri * case where kmem64 can allocate one large chunk 12057218Ssvemuri */ 12067218Ssvemuri if (kpm_smallpages == 0) { 12077218Ssvemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 12087218Ssvemuri } else { 12097218Ssvemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 12107218Ssvemuri sizeof (kpm_spage_t)); 12117218Ssvemuri } 12127218Ssvemuri pp_sz = npages * sizeof (struct page); 12137218Ssvemuri } 12147218Ssvemuri 12155648Ssetje if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 12165648Ssetje cmn_err(CE_PANIC, "not enough kmem64 space"); 12175648Ssetje PRM_DEBUG(kmem64_base); 12185648Ssetje PRM_DEBUG(kmem64_end); 12195648Ssetje PRM_DEBUG(kmem64_aligned_end); 12204204Sha137994 12214204Sha137994 /* 12225648Ssetje * ... and divy it up 12230Sstevel@tonic-gate */ 12245648Ssetje alloc_base = kmem64_base; 12255872Ssetje 12265648Ssetje pp_base = (page_t *)alloc_base; 12275648Ssetje alloc_base += pp_sz; 12285648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12295648Ssetje PRM_DEBUG(pp_base); 12305648Ssetje PRM_DEBUG(npages); 12315648Ssetje 12325648Ssetje if (kpm_enable) { 12335648Ssetje kpm_pp_base = alloc_base; 12345648Ssetje if (kpm_smallpages == 0) { 12355648Ssetje /* kpm_npages based on physinstalled, don't reset */ 12365648Ssetje kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 12375648Ssetje } else { 12385648Ssetje kpm_npages = ptokpmpr(npages); 12395648Ssetje kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 12400Sstevel@tonic-gate } 12415648Ssetje alloc_base += kpm_pp_sz; 12425648Ssetje alloc_base = 12435648Ssetje (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12445648Ssetje PRM_DEBUG(kpm_pp_base); 12450Sstevel@tonic-gate } 12460Sstevel@tonic-gate 12475648Ssetje alloc_base = alloc_hmehash(alloc_base); 12485648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12495648Ssetje PRM_DEBUG(alloc_base); 12505648Ssetje 12515648Ssetje page_hash = (page_t **)alloc_base; 12525648Ssetje alloc_base += pagehash_sz; 12535648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12545648Ssetje PRM_DEBUG(page_hash); 12555648Ssetje 12565648Ssetje alloc_base = alloc_page_freelists(alloc_base); 12575648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12585648Ssetje PRM_DEBUG(alloc_base); 12595648Ssetje 12605648Ssetje #ifdef TRAPTRACE 12615648Ssetje ttrace_buf = alloc_base; 12625648Ssetje alloc_base += tt_sz; 12635648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12645648Ssetje PRM_DEBUG(alloc_base); 12655648Ssetje #endif /* TRAPTRACE */ 12665648Ssetje 12675648Ssetje pse_mutex = (pad_mutex_t *)alloc_base; 12685648Ssetje alloc_base += psetable_sz; 12695648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 12705648Ssetje PRM_DEBUG(alloc_base); 12715648Ssetje 12727218Ssvemuri /* 12737218Ssvemuri * Note that if we use small chunk allocations for 12747218Ssvemuri * kmem64, we need to ensure kmem64_end is the same as 12757218Ssvemuri * kmem64_aligned_end to prevent subsequent logic from 12767218Ssvemuri * trying to reuse the overmapping. 12777218Ssvemuri * Otherwise we adjust kmem64_end to what we really allocated. 12787218Ssvemuri */ 12797218Ssvemuri if (kmem64_smchunks) { 12807218Ssvemuri kmem64_end = kmem64_aligned_end; 12817218Ssvemuri } else { 12827218Ssvemuri kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 12837218Ssvemuri } 12845648Ssetje kmem64_sz = kmem64_end - kmem64_base; 12850Sstevel@tonic-gate 12860Sstevel@tonic-gate if (&ecache_init_scrub_flush_area) { 12875648Ssetje alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 12885648Ssetje ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 12890Sstevel@tonic-gate } 12900Sstevel@tonic-gate 12910Sstevel@tonic-gate /* 12920Sstevel@tonic-gate * If physmem is patched to be non-zero, use it instead of 12930Sstevel@tonic-gate * the monitor value unless physmem is larger than the total 12940Sstevel@tonic-gate * amount of memory on hand. 12950Sstevel@tonic-gate */ 12960Sstevel@tonic-gate if (physmem == 0 || physmem > npages) 12970Sstevel@tonic-gate physmem = npages; 12980Sstevel@tonic-gate 12990Sstevel@tonic-gate /* 13005648Ssetje * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 13015648Ssetje * is mounted as root. This memory is held down by OBP and unlike 13025648Ssetje * the stub boot_archive is never released. 13030Sstevel@tonic-gate * 13045648Ssetje * In order to get things sized correctly on lower memory 13055648Ssetje * machines (where the memory used by the ramdisk represents 13065648Ssetje * a significant portion of memory), physmem is adjusted. 13075648Ssetje * 13085648Ssetje * This is done by subtracting the ramdisk_size which is set 13095648Ssetje * to the size of the ramdisk (in Kb) in /etc/system at the 13105648Ssetje * time the miniroot archive is constructed. 13110Sstevel@tonic-gate */ 13125872Ssetje if (root_is_ramdisk == B_TRUE) { 13135872Ssetje ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 13145872Ssetje physmem -= ramdisk_npages; 13155872Ssetje } 13165648Ssetje 13175648Ssetje if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 13185648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 13190Sstevel@tonic-gate 13204878Sblakej /* 13215648Ssetje * Allocate space for the interrupt vector table. 13223764Sdp78419 */ 13235648Ssetje memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 13242973Sgovinda if (memspace != (caddr_t)intr_vec_table) 13253764Sdp78419 prom_panic("interrupt vector table allocation failure"); 13260Sstevel@tonic-gate 13270Sstevel@tonic-gate /* 13280Sstevel@tonic-gate * Between now and when we finish copying in the memory lists, 13290Sstevel@tonic-gate * allocations happen so the space gets fragmented and the 13305648Ssetje * lists longer. Leave enough space for lists twice as 13315648Ssetje * long as we have now; then roundup to a pagesize. 13320Sstevel@tonic-gate */ 13335648Ssetje memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 13345648Ssetje prom_phys_avail_len() + prom_virt_avail_len()); 13355648Ssetje memlist_sz *= 2; 13365648Ssetje memlist_sz = roundup(memlist_sz, PAGESIZE); 13375648Ssetje memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 13380Sstevel@tonic-gate if (memspace == NULL) 13395648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 13400Sstevel@tonic-gate 13410Sstevel@tonic-gate memlist = (struct memlist *)memspace; 13420Sstevel@tonic-gate memlist_end = (char *)memspace + memlist_sz; 13430Sstevel@tonic-gate PRM_DEBUG(memlist); 13440Sstevel@tonic-gate PRM_DEBUG(memlist_end); 13455648Ssetje 13460Sstevel@tonic-gate PRM_DEBUG(sysbase); 13470Sstevel@tonic-gate PRM_DEBUG(syslimit); 13480Sstevel@tonic-gate kernelheap_init((void *)sysbase, (void *)syslimit, 13490Sstevel@tonic-gate (caddr_t)sysbase + PAGESIZE, NULL, NULL); 13500Sstevel@tonic-gate 13510Sstevel@tonic-gate /* 13520Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 13530Sstevel@tonic-gate */ 13540Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 13550Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 13560Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 13570Sstevel@tonic-gate 13580Sstevel@tonic-gate /* 13595648Ssetje * Remove the space used by prom_alloc from the kernel heap 13600Sstevel@tonic-gate * plus the area actually used by the OBP (if any) 13610Sstevel@tonic-gate * ignoring virtual addresses in virt_avail, above syslimit. 13620Sstevel@tonic-gate */ 13630Sstevel@tonic-gate virt_avail = memlist; 13640Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 13650Sstevel@tonic-gate 13660Sstevel@tonic-gate for (cur = virt_avail; cur->next; cur = cur->next) { 13670Sstevel@tonic-gate uint64_t range_base, range_size; 13680Sstevel@tonic-gate 13690Sstevel@tonic-gate if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 13700Sstevel@tonic-gate continue; 13710Sstevel@tonic-gate if (range_base >= (uint64_t)syslimit) 13720Sstevel@tonic-gate break; 13730Sstevel@tonic-gate /* 13740Sstevel@tonic-gate * Limit the range to end at syslimit. 13750Sstevel@tonic-gate */ 13760Sstevel@tonic-gate range_size = MIN(cur->next->address, 13770Sstevel@tonic-gate (uint64_t)syslimit) - range_base; 13780Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 13790Sstevel@tonic-gate 0, 0, (void *)range_base, (void *)(range_base + range_size), 13800Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 13810Sstevel@tonic-gate } 13820Sstevel@tonic-gate 13830Sstevel@tonic-gate phys_avail = memlist; 13845648Ssetje copy_memlist(boot_physavail, boot_physavail_len, &memlist); 13850Sstevel@tonic-gate 13860Sstevel@tonic-gate /* 13874104Sblakej * Add any extra memory at the end of the ndata region if there's at 13884104Sblakej * least a page to add. There might be a few more pages available in 13894104Sblakej * the middle of the ndata region, but for now they are ignored. 13900Sstevel@tonic-gate */ 13914104Sblakej nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 13924104Sblakej if (nalloc_base == NULL) 13930Sstevel@tonic-gate nalloc_base = nalloc_end; 13940Sstevel@tonic-gate ndata_remain_sz = nalloc_end - nalloc_base; 13950Sstevel@tonic-gate 13965648Ssetje /* 13975648Ssetje * Copy physinstalled list into kernel space. 13985648Ssetje */ 13995648Ssetje phys_install = memlist; 14005648Ssetje copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 14015648Ssetje 14025648Ssetje /* 14035648Ssetje * Create list of physical addrs we don't need pp's for: 14045648Ssetje * kernel text 4M page 14055648Ssetje * kernel data 4M page - ndata_remain_sz 14065648Ssetje * kmem64 pages 14075648Ssetje * 14085648Ssetje * NB if adding any pages here, make sure no kpm page 14095648Ssetje * overlaps can occur (see ASSERTs in kphysm_memsegs) 14105648Ssetje */ 14115648Ssetje nopp_list = memlist; 14125648Ssetje memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 14135648Ssetje memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 14145648Ssetje &memlist, &nopp_list); 14157218Ssvemuri 14167218Ssvemuri /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 14177218Ssvemuri if (!kmem64_smchunks) 14187218Ssvemuri memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 14190Sstevel@tonic-gate 14200Sstevel@tonic-gate if ((caddr_t)memlist > (memspace + memlist_sz)) 14213764Sdp78419 prom_panic("memlist overflow"); 14220Sstevel@tonic-gate 14230Sstevel@tonic-gate /* 14246880Sdv142724 * Size the pcf array based on the number of cpus in the box at 14256880Sdv142724 * boot time. 14266880Sdv142724 */ 14276880Sdv142724 pcf_init(); 14286880Sdv142724 14296880Sdv142724 /* 14300Sstevel@tonic-gate * Initialize the page structures from the memory lists. 14310Sstevel@tonic-gate */ 14325648Ssetje kphysm_init(); 14330Sstevel@tonic-gate 14340Sstevel@tonic-gate availrmem_initial = availrmem = freemem; 14350Sstevel@tonic-gate PRM_DEBUG(availrmem); 14360Sstevel@tonic-gate 14370Sstevel@tonic-gate /* 14380Sstevel@tonic-gate * Some of the locks depend on page_hashsz being set! 14390Sstevel@tonic-gate * kmem_init() depends on this; so, keep it here. 14400Sstevel@tonic-gate */ 14410Sstevel@tonic-gate page_lock_init(); 14420Sstevel@tonic-gate 14430Sstevel@tonic-gate /* 14440Sstevel@tonic-gate * Initialize kernel memory allocator. 14450Sstevel@tonic-gate */ 14460Sstevel@tonic-gate kmem_init(); 14470Sstevel@tonic-gate 14480Sstevel@tonic-gate /* 14493717Sdp78419 * Factor in colorequiv to check additional 'equivalent' bins 14503717Sdp78419 */ 14513733Sdp78419 if (&page_set_colorequiv_arr_cpu != NULL) 14523733Sdp78419 page_set_colorequiv_arr_cpu(); 14533733Sdp78419 else 14543733Sdp78419 page_set_colorequiv_arr(); 14553717Sdp78419 14563717Sdp78419 /* 14570Sstevel@tonic-gate * Initialize bp_mapin(). 14580Sstevel@tonic-gate */ 14590Sstevel@tonic-gate bp_init(shm_alignment, HAT_STRICTORDER); 14600Sstevel@tonic-gate 14610Sstevel@tonic-gate /* 14625358Sjc25722 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 14635358Sjc25722 * vector data structures and MPO mblock structs from the 32-bit heap. 14640Sstevel@tonic-gate */ 14650Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 14660Sstevel@tonic-gate panicbuf, panicbuf + PANICBUFSIZE, 14670Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 14680Sstevel@tonic-gate 14690Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 14702973Sgovinda intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 14710Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 14720Sstevel@tonic-gate 14735358Sjc25722 if (mpo_heap32_bufsz > (size_t)0) { 14745358Sjc25722 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 14755358Sjc25722 PAGESIZE, 0, 0, mpo_heap32_buf, 14765358Sjc25722 mpo_heap32_buf + mpo_heap32_bufsz, 14775358Sjc25722 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 14785358Sjc25722 } 14790Sstevel@tonic-gate mem_config_init(); 14800Sstevel@tonic-gate } 14810Sstevel@tonic-gate 14820Sstevel@tonic-gate static void 14830Sstevel@tonic-gate startup_modules(void) 14840Sstevel@tonic-gate { 14855648Ssetje int nhblk1, nhblk8; 14860Sstevel@tonic-gate size_t nhblksz; 14873764Sdp78419 pgcnt_t pages_per_hblk; 14880Sstevel@tonic-gate size_t hme8blk_sz, hme1blk_sz; 14890Sstevel@tonic-gate 14900Sstevel@tonic-gate /* 14910Sstevel@tonic-gate * Let the platforms have a chance to change default 14920Sstevel@tonic-gate * values before reading system file. 14930Sstevel@tonic-gate */ 14940Sstevel@tonic-gate if (&set_platform_defaults) 14950Sstevel@tonic-gate set_platform_defaults(); 14960Sstevel@tonic-gate 14970Sstevel@tonic-gate /* 14980Sstevel@tonic-gate * Calculate default settings of system parameters based upon 14990Sstevel@tonic-gate * maxusers, yet allow to be overridden via the /etc/system file. 15000Sstevel@tonic-gate */ 15010Sstevel@tonic-gate param_calc(0); 15020Sstevel@tonic-gate 15030Sstevel@tonic-gate mod_setup(); 15040Sstevel@tonic-gate 15050Sstevel@tonic-gate /* 15060Sstevel@tonic-gate * If this is a positron, complain and halt. 15070Sstevel@tonic-gate */ 15080Sstevel@tonic-gate if (&iam_positron && iam_positron()) { 15090Sstevel@tonic-gate cmn_err(CE_WARN, "This hardware platform is not supported" 15100Sstevel@tonic-gate " by this release of Solaris.\n"); 15110Sstevel@tonic-gate #ifdef DEBUG 15120Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to resume */ 15130Sstevel@tonic-gate cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 15140Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 15150Sstevel@tonic-gate 15160Sstevel@tonic-gate #else /* DEBUG */ 15170Sstevel@tonic-gate halt(0); 15180Sstevel@tonic-gate #endif /* DEBUG */ 15190Sstevel@tonic-gate } 15200Sstevel@tonic-gate 15210Sstevel@tonic-gate /* 15220Sstevel@tonic-gate * If we are running firmware that isn't 64-bit ready 15230Sstevel@tonic-gate * then complain and halt. 15240Sstevel@tonic-gate */ 15250Sstevel@tonic-gate do_prom_version_check(); 15260Sstevel@tonic-gate 15270Sstevel@tonic-gate /* 15280Sstevel@tonic-gate * Initialize system parameters 15290Sstevel@tonic-gate */ 15300Sstevel@tonic-gate param_init(); 15310Sstevel@tonic-gate 15320Sstevel@tonic-gate /* 15330Sstevel@tonic-gate * maxmem is the amount of physical memory we're playing with. 15340Sstevel@tonic-gate */ 15350Sstevel@tonic-gate maxmem = physmem; 15360Sstevel@tonic-gate 15370Sstevel@tonic-gate /* Set segkp limits. */ 15383446Smrj ncbase = kdi_segdebugbase; 15393446Smrj ncend = kdi_segdebugbase; 15400Sstevel@tonic-gate 15410Sstevel@tonic-gate /* 15420Sstevel@tonic-gate * Initialize the hat layer. 15430Sstevel@tonic-gate */ 15440Sstevel@tonic-gate hat_init(); 15450Sstevel@tonic-gate 15460Sstevel@tonic-gate /* 15470Sstevel@tonic-gate * Initialize segment management stuff. 15480Sstevel@tonic-gate */ 15490Sstevel@tonic-gate seg_init(); 15500Sstevel@tonic-gate 15510Sstevel@tonic-gate /* 15520Sstevel@tonic-gate * Create the va>tte handler, so the prom can understand 15530Sstevel@tonic-gate * kernel translations. The handler is installed later, just 15540Sstevel@tonic-gate * as we are about to take over the trap table from the prom. 15550Sstevel@tonic-gate */ 15560Sstevel@tonic-gate create_va_to_tte(); 15570Sstevel@tonic-gate 15580Sstevel@tonic-gate /* 15590Sstevel@tonic-gate * Load the forthdebugger (optional) 15600Sstevel@tonic-gate */ 15610Sstevel@tonic-gate forthdebug_init(); 15620Sstevel@tonic-gate 15630Sstevel@tonic-gate /* 15640Sstevel@tonic-gate * Create OBP node for console input callbacks 15650Sstevel@tonic-gate * if it is needed. 15660Sstevel@tonic-gate */ 15671253Slq150181 startup_create_io_node(); 15680Sstevel@tonic-gate 15690Sstevel@tonic-gate if (modloadonly("fs", "specfs") == -1) 15700Sstevel@tonic-gate halt("Can't load specfs"); 15710Sstevel@tonic-gate 15720Sstevel@tonic-gate if (modloadonly("fs", "devfs") == -1) 15730Sstevel@tonic-gate halt("Can't load devfs"); 15740Sstevel@tonic-gate 15750Sstevel@tonic-gate if (modloadonly("misc", "swapgeneric") == -1) 15760Sstevel@tonic-gate halt("Can't load swapgeneric"); 15770Sstevel@tonic-gate 15781676Sjpk (void) modloadonly("sys", "lbl_edition"); 15791676Sjpk 15800Sstevel@tonic-gate dispinit(); 15810Sstevel@tonic-gate 15820Sstevel@tonic-gate /* 15830Sstevel@tonic-gate * Infer meanings to the members of the idprom buffer. 15840Sstevel@tonic-gate */ 15850Sstevel@tonic-gate parse_idprom(); 15860Sstevel@tonic-gate 15870Sstevel@tonic-gate /* Read cluster configuration data. */ 15880Sstevel@tonic-gate clconf_init(); 15890Sstevel@tonic-gate 15900Sstevel@tonic-gate setup_ddi(); 15910Sstevel@tonic-gate 15920Sstevel@tonic-gate /* 15930Sstevel@tonic-gate * Lets take this opportunity to load the root device. 15940Sstevel@tonic-gate */ 15950Sstevel@tonic-gate if (loadrootmodules() != 0) 15960Sstevel@tonic-gate debug_enter("Can't load the root filesystem"); 15970Sstevel@tonic-gate 15980Sstevel@tonic-gate /* 15990Sstevel@tonic-gate * Load tod driver module for the tod part found on this system. 16000Sstevel@tonic-gate * Recompute the cpu frequency/delays based on tod as tod part 16010Sstevel@tonic-gate * tends to keep time more accurately. 16020Sstevel@tonic-gate */ 16030Sstevel@tonic-gate if (&load_tod_module) 16040Sstevel@tonic-gate load_tod_module(); 16050Sstevel@tonic-gate 16060Sstevel@tonic-gate /* 16070Sstevel@tonic-gate * Allow platforms to load modules which might 16080Sstevel@tonic-gate * be needed after bootops are gone. 16090Sstevel@tonic-gate */ 16100Sstevel@tonic-gate if (&load_platform_modules) 16110Sstevel@tonic-gate load_platform_modules(); 16120Sstevel@tonic-gate 16130Sstevel@tonic-gate setcpudelay(); 16140Sstevel@tonic-gate 16150Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 16160Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 16170Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 16180Sstevel@tonic-gate 16190Sstevel@tonic-gate /* 16200Sstevel@tonic-gate * Calculation and allocation of hmeblks needed to remap 16213764Sdp78419 * the memory allocated by PROM till now. 16223764Sdp78419 * Overestimate the number of hblk1 elements by assuming 16233764Sdp78419 * worst case of TTE64K mappings. 16243764Sdp78419 * sfmmu_hblk_alloc will panic if this calculation is wrong. 16250Sstevel@tonic-gate */ 16263764Sdp78419 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 16273764Sdp78419 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 16283764Sdp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 16293764Sdp78419 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 16303764Sdp78419 16313764Sdp78419 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 16323764Sdp78419 16333764Sdp78419 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 16340Sstevel@tonic-gate hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 16350Sstevel@tonic-gate hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 16360Sstevel@tonic-gate 16373764Sdp78419 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 16383764Sdp78419 16390Sstevel@tonic-gate pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 16403764Sdp78419 nhblk8 = 0; 16413764Sdp78419 while (bop_alloc_pages > 1) { 16423764Sdp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 16433764Sdp78419 nhblk8 += bop_alloc_pages /= pages_per_hblk; 16443764Sdp78419 bop_alloc_pages *= hme8blk_sz; 16453764Sdp78419 bop_alloc_pages = btopr(bop_alloc_pages); 16463764Sdp78419 } 16473764Sdp78419 nhblk8 += 2; 16480Sstevel@tonic-gate 16490Sstevel@tonic-gate /* 16500Sstevel@tonic-gate * Since hblk8's can hold up to 64k of mappings aligned on a 64k 16510Sstevel@tonic-gate * boundary, the number of hblk8's needed to map the entries in the 16520Sstevel@tonic-gate * boot_virtavail list needs to be adjusted to take this into 16530Sstevel@tonic-gate * consideration. Thus, we need to add additional hblk8's since it 16540Sstevel@tonic-gate * is possible that an hblk8 will not have all 8 slots used due to 16550Sstevel@tonic-gate * alignment constraints. Since there were boot_virtavail_len entries 16560Sstevel@tonic-gate * in that list, we need to add that many hblk8's to the number 16570Sstevel@tonic-gate * already calculated to make sure we don't underestimate. 16580Sstevel@tonic-gate */ 16590Sstevel@tonic-gate nhblk8 += boot_virtavail_len; 16600Sstevel@tonic-gate nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 16610Sstevel@tonic-gate 16620Sstevel@tonic-gate /* Allocate in pagesize chunks */ 16630Sstevel@tonic-gate nhblksz = roundup(nhblksz, MMU_PAGESIZE); 16640Sstevel@tonic-gate hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 16650Sstevel@tonic-gate sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 16660Sstevel@tonic-gate } 16670Sstevel@tonic-gate 16680Sstevel@tonic-gate static void 16690Sstevel@tonic-gate startup_bop_gone(void) 16700Sstevel@tonic-gate { 16710Sstevel@tonic-gate 16720Sstevel@tonic-gate /* 16731991Sheppo * Destroy the MD initialized at startup 16741991Sheppo * The startup initializes the MD framework 16751991Sheppo * using prom and BOP alloc free it now. 16761991Sheppo */ 16771991Sheppo mach_descrip_startup_fini(); 16781991Sheppo 16791991Sheppo /* 16805648Ssetje * We're done with prom allocations. 16810Sstevel@tonic-gate */ 16825648Ssetje bop_fini(); 16830Sstevel@tonic-gate 16840Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 16850Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 16860Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 16870Sstevel@tonic-gate 16880Sstevel@tonic-gate /* 16890Sstevel@tonic-gate * setup physically contiguous area twice as large as the ecache. 16900Sstevel@tonic-gate * this is used while doing displacement flush of ecaches 16910Sstevel@tonic-gate */ 16920Sstevel@tonic-gate if (&ecache_flush_address) { 16930Sstevel@tonic-gate ecache_flushaddr = ecache_flush_address(); 16940Sstevel@tonic-gate if (ecache_flushaddr == (uint64_t)-1) { 16950Sstevel@tonic-gate cmn_err(CE_PANIC, 16960Sstevel@tonic-gate "startup: no memory to set ecache_flushaddr"); 16970Sstevel@tonic-gate } 16980Sstevel@tonic-gate } 16990Sstevel@tonic-gate 17000Sstevel@tonic-gate /* 17010Sstevel@tonic-gate * Virtual available next. 17020Sstevel@tonic-gate */ 17030Sstevel@tonic-gate ASSERT(virt_avail != NULL); 17040Sstevel@tonic-gate memlist_free_list(virt_avail); 17050Sstevel@tonic-gate virt_avail = memlist; 17060Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 17070Sstevel@tonic-gate 17080Sstevel@tonic-gate } 17090Sstevel@tonic-gate 17100Sstevel@tonic-gate 17110Sstevel@tonic-gate /* 17120Sstevel@tonic-gate * startup_fixup_physavail - called from mach_sfmmu.c after the final 17130Sstevel@tonic-gate * allocations have been performed. We can't call it in startup_bop_gone 17140Sstevel@tonic-gate * since later operations can cause obp to allocate more memory. 17150Sstevel@tonic-gate */ 17160Sstevel@tonic-gate void 17170Sstevel@tonic-gate startup_fixup_physavail(void) 17180Sstevel@tonic-gate { 17190Sstevel@tonic-gate struct memlist *cur; 17203764Sdp78419 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 17213764Sdp78419 17223764Sdp78419 PRM_DEBUG(kmem64_overmap_size); 17230Sstevel@tonic-gate 17240Sstevel@tonic-gate /* 17250Sstevel@tonic-gate * take the most current snapshot we can by calling mem-update 17260Sstevel@tonic-gate */ 17270Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 17280Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 17290Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 17300Sstevel@tonic-gate 17310Sstevel@tonic-gate /* 17320Sstevel@tonic-gate * Copy phys_avail list, again. 17330Sstevel@tonic-gate * Both the kernel/boot and the prom have been allocating 17340Sstevel@tonic-gate * from the original list we copied earlier. 17350Sstevel@tonic-gate */ 17360Sstevel@tonic-gate cur = memlist; 17375648Ssetje copy_memlist(boot_physavail, boot_physavail_len, &memlist); 17380Sstevel@tonic-gate 17390Sstevel@tonic-gate /* 17403764Sdp78419 * Add any unused kmem64 memory from overmapped page 17413764Sdp78419 * (Note: va_to_pa does not work for kmem64_end) 17423764Sdp78419 */ 17433764Sdp78419 if (kmem64_overmap_size) { 17443764Sdp78419 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 17455648Ssetje kmem64_overmap_size, &memlist, &cur); 17463764Sdp78419 } 17473764Sdp78419 17483764Sdp78419 /* 17495648Ssetje * Add any extra memory after e_data we added to the phys_avail list 17500Sstevel@tonic-gate * back to the old list. 17510Sstevel@tonic-gate */ 17520Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE) 17530Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base), 17540Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &cur); 17550Sstevel@tonic-gate 17560Sstevel@tonic-gate /* 17570Sstevel@tonic-gate * There isn't any bounds checking on the memlist area 17580Sstevel@tonic-gate * so ensure it hasn't overgrown. 17590Sstevel@tonic-gate */ 17600Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 17610Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: memlist size exceeded"); 17620Sstevel@tonic-gate 17630Sstevel@tonic-gate /* 17640Sstevel@tonic-gate * The kernel removes the pages that were allocated for it from 17650Sstevel@tonic-gate * the freelist, but we now have to find any -extra- pages that 17660Sstevel@tonic-gate * the prom has allocated for it's own book-keeping, and remove 17670Sstevel@tonic-gate * them from the freelist too. sigh. 17680Sstevel@tonic-gate */ 17695648Ssetje sync_memlists(phys_avail, cur); 17700Sstevel@tonic-gate 17710Sstevel@tonic-gate ASSERT(phys_avail != NULL); 17725872Ssetje 17735872Ssetje old_phys_avail = phys_avail; 17740Sstevel@tonic-gate phys_avail = cur; 17755872Ssetje } 17765872Ssetje 17775872Ssetje void 17785872Ssetje update_kcage_ranges(uint64_t addr, uint64_t len) 17795872Ssetje { 17805872Ssetje pfn_t base = btop(addr); 17815872Ssetje pgcnt_t num = btop(len); 17825872Ssetje int rv; 17835872Ssetje 17845872Ssetje rv = kcage_range_add(base, num, kcage_startup_dir); 17855872Ssetje 17865872Ssetje if (rv == ENOMEM) { 17875872Ssetje cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 17885872Ssetje (len == 0 ? 0 : BYTES_TO_MB(len))); 17895872Ssetje } else if (rv != 0) { 17905872Ssetje /* catch this in debug kernels */ 17915872Ssetje ASSERT(0); 17925872Ssetje 17935872Ssetje cmn_err(CE_WARN, "unexpected kcage_range_add" 17945872Ssetje " return value %d", rv); 17955872Ssetje } 17960Sstevel@tonic-gate } 17970Sstevel@tonic-gate 17980Sstevel@tonic-gate static void 17990Sstevel@tonic-gate startup_vm(void) 18000Sstevel@tonic-gate { 18010Sstevel@tonic-gate size_t i; 18020Sstevel@tonic-gate struct segmap_crargs a; 18030Sstevel@tonic-gate struct segkpm_crargs b; 18040Sstevel@tonic-gate 18050Sstevel@tonic-gate uint64_t avmem; 18060Sstevel@tonic-gate caddr_t va; 18070Sstevel@tonic-gate pgcnt_t max_phys_segkp; 18080Sstevel@tonic-gate int mnode; 18090Sstevel@tonic-gate 18102991Ssusans extern int use_brk_lpg, use_stk_lpg; 1811423Sdavemq 18120Sstevel@tonic-gate /* 18130Sstevel@tonic-gate * get prom's mappings, create hments for them and switch 18140Sstevel@tonic-gate * to the kernel context. 18150Sstevel@tonic-gate */ 18160Sstevel@tonic-gate hat_kern_setup(); 18170Sstevel@tonic-gate 18180Sstevel@tonic-gate /* 18190Sstevel@tonic-gate * Take over trap table 18200Sstevel@tonic-gate */ 18210Sstevel@tonic-gate setup_trap_table(); 18220Sstevel@tonic-gate 18230Sstevel@tonic-gate /* 18240Sstevel@tonic-gate * Install the va>tte handler, so that the prom can handle 18250Sstevel@tonic-gate * misses and understand the kernel table layout in case 18260Sstevel@tonic-gate * we need call into the prom. 18270Sstevel@tonic-gate */ 18280Sstevel@tonic-gate install_va_to_tte(); 18290Sstevel@tonic-gate 18300Sstevel@tonic-gate /* 18310Sstevel@tonic-gate * Set a flag to indicate that the tba has been taken over. 18320Sstevel@tonic-gate */ 18330Sstevel@tonic-gate tba_taken_over = 1; 18340Sstevel@tonic-gate 18350Sstevel@tonic-gate /* initialize MMU primary context register */ 18360Sstevel@tonic-gate mmu_init_kcontext(); 18370Sstevel@tonic-gate 18380Sstevel@tonic-gate /* 18390Sstevel@tonic-gate * The boot cpu can now take interrupts, x-calls, x-traps 18400Sstevel@tonic-gate */ 18410Sstevel@tonic-gate CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 18420Sstevel@tonic-gate CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 18430Sstevel@tonic-gate 18440Sstevel@tonic-gate /* 18450Sstevel@tonic-gate * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 18460Sstevel@tonic-gate */ 18470Sstevel@tonic-gate tbr_wr_addr_inited = 1; 18480Sstevel@tonic-gate 18490Sstevel@tonic-gate /* 18500Sstevel@tonic-gate * Initialize VM system, and map kernel address space. 18510Sstevel@tonic-gate */ 18520Sstevel@tonic-gate kvm_init(); 18530Sstevel@tonic-gate 18545872Ssetje ASSERT(old_phys_avail != NULL && phys_avail != NULL); 18555872Ssetje if (kernel_cage_enable) { 18565872Ssetje diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 18575872Ssetje } 18585872Ssetje memlist_free_list(old_phys_avail); 18595872Ssetje 18600Sstevel@tonic-gate /* 18610Sstevel@tonic-gate * If the following is true, someone has patched 18620Sstevel@tonic-gate * phsymem to be less than the number of pages that 18630Sstevel@tonic-gate * the system actually has. Remove pages until system 18640Sstevel@tonic-gate * memory is limited to the requested amount. Since we 18650Sstevel@tonic-gate * have allocated page structures for all pages, we 18660Sstevel@tonic-gate * correct the amount of memory we want to remove 18670Sstevel@tonic-gate * by the size of the memory used to hold page structures 18680Sstevel@tonic-gate * for the non-used pages. 18690Sstevel@tonic-gate */ 18705872Ssetje if (physmem + ramdisk_npages < npages) { 18710Sstevel@tonic-gate pgcnt_t diff, off; 18720Sstevel@tonic-gate struct page *pp; 18730Sstevel@tonic-gate struct seg kseg; 18740Sstevel@tonic-gate 18750Sstevel@tonic-gate cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 18760Sstevel@tonic-gate 18770Sstevel@tonic-gate off = 0; 18785872Ssetje diff = npages - (physmem + ramdisk_npages); 18790Sstevel@tonic-gate diff -= mmu_btopr(diff * sizeof (struct page)); 18800Sstevel@tonic-gate kseg.s_as = &kas; 18810Sstevel@tonic-gate while (diff--) { 18820Sstevel@tonic-gate pp = page_create_va(&unused_pages_vp, (offset_t)off, 18830Sstevel@tonic-gate MMU_PAGESIZE, PG_WAIT | PG_EXCL, 18840Sstevel@tonic-gate &kseg, (caddr_t)off); 18850Sstevel@tonic-gate if (pp == NULL) 18860Sstevel@tonic-gate cmn_err(CE_PANIC, "limited physmem too much!"); 18870Sstevel@tonic-gate page_io_unlock(pp); 18880Sstevel@tonic-gate page_downgrade(pp); 18890Sstevel@tonic-gate availrmem--; 18900Sstevel@tonic-gate off += MMU_PAGESIZE; 18910Sstevel@tonic-gate } 18920Sstevel@tonic-gate } 18930Sstevel@tonic-gate 18940Sstevel@tonic-gate /* 18950Sstevel@tonic-gate * When printing memory, show the total as physmem less 18960Sstevel@tonic-gate * that stolen by a debugger. 18970Sstevel@tonic-gate */ 18980Sstevel@tonic-gate cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 18990Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 10), 19000Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 19010Sstevel@tonic-gate 19020Sstevel@tonic-gate avmem = (uint64_t)freemem << PAGESHIFT; 19030Sstevel@tonic-gate cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 19040Sstevel@tonic-gate 19052991Ssusans /* 19062991Ssusans * For small memory systems disable automatic large pages. 19072991Ssusans */ 19082991Ssusans if (physmem < privm_lpg_min_physmem) { 1909423Sdavemq use_brk_lpg = 0; 1910423Sdavemq use_stk_lpg = 0; 1911423Sdavemq } 1912423Sdavemq 19130Sstevel@tonic-gate /* 19140Sstevel@tonic-gate * Perform platform specific freelist processing 19150Sstevel@tonic-gate */ 19160Sstevel@tonic-gate if (&plat_freelist_process) { 19170Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 19180Sstevel@tonic-gate if (mem_node_config[mnode].exists) 19190Sstevel@tonic-gate plat_freelist_process(mnode); 19200Sstevel@tonic-gate } 19210Sstevel@tonic-gate 19220Sstevel@tonic-gate /* 19230Sstevel@tonic-gate * Initialize the segkp segment type. We position it 19240Sstevel@tonic-gate * after the configured tables and buffers (whose end 19250Sstevel@tonic-gate * is given by econtig) and before V_WKBASE_ADDR. 19260Sstevel@tonic-gate * Also in this area is segkmap (size SEGMAPSIZE). 19270Sstevel@tonic-gate */ 19280Sstevel@tonic-gate 19290Sstevel@tonic-gate /* XXX - cache alignment? */ 19300Sstevel@tonic-gate va = (caddr_t)SEGKPBASE; 19310Sstevel@tonic-gate ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 19320Sstevel@tonic-gate 19330Sstevel@tonic-gate max_phys_segkp = (physmem * 2); 19340Sstevel@tonic-gate 19350Sstevel@tonic-gate if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 19360Sstevel@tonic-gate segkpsize = btop(SEGKPDEFSIZE); 19370Sstevel@tonic-gate cmn_err(CE_WARN, "Illegal value for segkpsize. " 19380Sstevel@tonic-gate "segkpsize has been reset to %ld pages", segkpsize); 19390Sstevel@tonic-gate } 19400Sstevel@tonic-gate 19410Sstevel@tonic-gate i = ptob(MIN(segkpsize, max_phys_segkp)); 19420Sstevel@tonic-gate 19430Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 19440Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkp) < 0) 19450Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: cannot attach segkp"); 19460Sstevel@tonic-gate if (segkp_create(segkp) != 0) 19470Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: segkp_create failed"); 19480Sstevel@tonic-gate rw_exit(&kas.a_lock); 19490Sstevel@tonic-gate 19500Sstevel@tonic-gate /* 19510Sstevel@tonic-gate * kpm segment 19520Sstevel@tonic-gate */ 19530Sstevel@tonic-gate segmap_kpm = kpm_enable && 19544528Spaulsan segmap_kpm && PAGESIZE == MAXBSIZE; 19550Sstevel@tonic-gate 19560Sstevel@tonic-gate if (kpm_enable) { 19570Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 19580Sstevel@tonic-gate 19590Sstevel@tonic-gate /* 19600Sstevel@tonic-gate * The segkpm virtual range range is larger than the 19610Sstevel@tonic-gate * actual physical memory size and also covers gaps in 19620Sstevel@tonic-gate * the physical address range for the following reasons: 19630Sstevel@tonic-gate * . keep conversion between segkpm and physical addresses 19640Sstevel@tonic-gate * simple, cheap and unambiguous. 19650Sstevel@tonic-gate * . avoid extension/shrink of the the segkpm in case of DR. 19660Sstevel@tonic-gate * . avoid complexity for handling of virtual addressed 19670Sstevel@tonic-gate * caches, segkpm and the regular mapping scheme must be 19680Sstevel@tonic-gate * kept in sync wrt. the virtual color of mapped pages. 19690Sstevel@tonic-gate * Any accesses to virtual segkpm ranges not backed by 19700Sstevel@tonic-gate * physical memory will fall through the memseg pfn hash 19710Sstevel@tonic-gate * and will be handled in segkpm_fault. 19720Sstevel@tonic-gate * Additional kpm_size spaces needed for vac alias prevention. 19730Sstevel@tonic-gate */ 19740Sstevel@tonic-gate if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 19750Sstevel@tonic-gate segkpm) < 0) 19760Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkpm"); 19770Sstevel@tonic-gate 19780Sstevel@tonic-gate b.prot = PROT_READ | PROT_WRITE; 19790Sstevel@tonic-gate b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 19800Sstevel@tonic-gate 19810Sstevel@tonic-gate if (segkpm_create(segkpm, (caddr_t)&b) != 0) 19820Sstevel@tonic-gate panic("segkpm_create segkpm"); 19830Sstevel@tonic-gate 19840Sstevel@tonic-gate rw_exit(&kas.a_lock); 19852296Sae112802 19862296Sae112802 mach_kpm_init(); 19870Sstevel@tonic-gate } 19880Sstevel@tonic-gate 19893290Sjohansen if (!segzio_fromheap) { 19903290Sjohansen size_t size; 19913552Sjohansen size_t physmem_b = mmu_ptob(physmem); 19923290Sjohansen 19933290Sjohansen /* size is in bytes, segziosize is in pages */ 19943290Sjohansen if (segziosize == 0) { 19953552Sjohansen size = physmem_b; 19963290Sjohansen } else { 19973290Sjohansen size = mmu_ptob(segziosize); 19983290Sjohansen } 19993290Sjohansen 20003290Sjohansen if (size < SEGZIOMINSIZE) { 20013290Sjohansen size = SEGZIOMINSIZE; 20023552Sjohansen } else if (size > SEGZIOMAXSIZE) { 20033552Sjohansen size = SEGZIOMAXSIZE; 20043552Sjohansen /* 20053552Sjohansen * On 64-bit x86, we only have 2TB of KVA. This exists 20063552Sjohansen * for parity with x86. 20073552Sjohansen * 20083552Sjohansen * SEGZIOMAXSIZE is capped at 512gb so that segzio 20093552Sjohansen * doesn't consume all of KVA. However, if we have a 20103552Sjohansen * system that has more thant 512gb of physical memory, 20113552Sjohansen * we can actually consume about half of the difference 20123552Sjohansen * between 512gb and the rest of the available physical 20133552Sjohansen * memory. 20143552Sjohansen */ 20153552Sjohansen if (physmem_b > SEGZIOMAXSIZE) { 20163552Sjohansen size += (physmem_b - SEGZIOMAXSIZE) / 2; 20173552Sjohansen } 20183290Sjohansen } 20193290Sjohansen segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 20203290Sjohansen /* put the base of the ZIO segment after the kpm segment */ 20213290Sjohansen segzio_base = kpm_vbase + (kpm_size * vac_colors); 20223290Sjohansen PRM_DEBUG(segziosize); 20233290Sjohansen PRM_DEBUG(segzio_base); 20243290Sjohansen 20253290Sjohansen /* 20263290Sjohansen * On some platforms, kvm_init is called after the kpm 20273290Sjohansen * sizes have been determined. On SPARC, kvm_init is called 20283290Sjohansen * before, so we have to attach the kzioseg after kvm is 20293290Sjohansen * initialized, otherwise we'll try to allocate from the boot 20303290Sjohansen * area since the kernel heap hasn't yet been configured. 20313290Sjohansen */ 20323290Sjohansen rw_enter(&kas.a_lock, RW_WRITER); 20333290Sjohansen 20343290Sjohansen (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 20353290Sjohansen &kzioseg); 20363290Sjohansen (void) segkmem_zio_create(&kzioseg); 20373290Sjohansen 20383290Sjohansen /* create zio area covering new segment */ 20393290Sjohansen segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 20403290Sjohansen 20413290Sjohansen rw_exit(&kas.a_lock); 20423290Sjohansen } 20433290Sjohansen 20443290Sjohansen 20450Sstevel@tonic-gate /* 20460Sstevel@tonic-gate * Now create generic mapping segment. This mapping 20470Sstevel@tonic-gate * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 20480Sstevel@tonic-gate * virtual address is greater than the amount of free 20490Sstevel@tonic-gate * memory that is available, then we trim back the 20500Sstevel@tonic-gate * segment size to that amount 20510Sstevel@tonic-gate */ 20520Sstevel@tonic-gate va = (caddr_t)SEGMAPBASE; 20530Sstevel@tonic-gate 20540Sstevel@tonic-gate /* 20550Sstevel@tonic-gate * 1201049: segkmap base address must be MAXBSIZE aligned 20560Sstevel@tonic-gate */ 20570Sstevel@tonic-gate ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 20580Sstevel@tonic-gate 20590Sstevel@tonic-gate /* 20600Sstevel@tonic-gate * Set size of segmap to percentage of freemem at boot, 20610Sstevel@tonic-gate * but stay within the allowable range 20620Sstevel@tonic-gate * Note we take percentage before converting from pages 20630Sstevel@tonic-gate * to bytes to avoid an overflow on 32-bit kernels. 20640Sstevel@tonic-gate */ 20650Sstevel@tonic-gate i = mmu_ptob((freemem * segmap_percent) / 100); 20660Sstevel@tonic-gate 20670Sstevel@tonic-gate if (i < MINMAPSIZE) 20680Sstevel@tonic-gate i = MINMAPSIZE; 20690Sstevel@tonic-gate 20700Sstevel@tonic-gate if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 20710Sstevel@tonic-gate i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 20720Sstevel@tonic-gate 20730Sstevel@tonic-gate i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 20740Sstevel@tonic-gate 20750Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 20760Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkmap) < 0) 20770Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkmap"); 20780Sstevel@tonic-gate 20790Sstevel@tonic-gate a.prot = PROT_READ | PROT_WRITE; 20800Sstevel@tonic-gate a.shmsize = shm_alignment; 20810Sstevel@tonic-gate a.nfreelist = 0; /* use segmap driver defaults */ 20820Sstevel@tonic-gate 20830Sstevel@tonic-gate if (segmap_create(segkmap, (caddr_t)&a) != 0) 20840Sstevel@tonic-gate panic("segmap_create segkmap"); 20850Sstevel@tonic-gate rw_exit(&kas.a_lock); 20860Sstevel@tonic-gate 20870Sstevel@tonic-gate segdev_init(); 20880Sstevel@tonic-gate } 20890Sstevel@tonic-gate 20900Sstevel@tonic-gate static void 20910Sstevel@tonic-gate startup_end(void) 20920Sstevel@tonic-gate { 20930Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 20940Sstevel@tonic-gate panic("memlist overflow 2"); 20950Sstevel@tonic-gate memlist_free_block((caddr_t)memlist, 20960Sstevel@tonic-gate ((caddr_t)memlist_end - (caddr_t)memlist)); 20970Sstevel@tonic-gate memlist = NULL; 20980Sstevel@tonic-gate 20990Sstevel@tonic-gate /* enable page_relocation since OBP is now done */ 21000Sstevel@tonic-gate page_relocate_ready = 1; 21010Sstevel@tonic-gate 21020Sstevel@tonic-gate /* 21030Sstevel@tonic-gate * Perform tasks that get done after most of the VM 21040Sstevel@tonic-gate * initialization has been done but before the clock 21050Sstevel@tonic-gate * and other devices get started. 21060Sstevel@tonic-gate */ 21070Sstevel@tonic-gate kern_setup1(); 21080Sstevel@tonic-gate 21090Sstevel@tonic-gate /* 21100Sstevel@tonic-gate * Intialize the VM arenas for allocating physically 21110Sstevel@tonic-gate * contiguus memory chunk for interrupt queues snd 21120Sstevel@tonic-gate * allocate/register boot cpu's queues, if any and 21130Sstevel@tonic-gate * allocate dump buffer for sun4v systems to store 21140Sstevel@tonic-gate * extra crash information during crash dump 21150Sstevel@tonic-gate */ 21160Sstevel@tonic-gate contig_mem_init(); 21170Sstevel@tonic-gate mach_descrip_init(); 21184050Sjb145095 21194050Sjb145095 if (cpu_intrq_setup(CPU)) { 21204050Sjb145095 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 21214050Sjb145095 } 21220Sstevel@tonic-gate cpu_intrq_register(CPU); 21231077Ssvemuri mach_htraptrace_setup(CPU->cpu_id); 21241077Ssvemuri mach_htraptrace_configure(CPU->cpu_id); 21250Sstevel@tonic-gate mach_dump_buffer_init(); 21260Sstevel@tonic-gate 21270Sstevel@tonic-gate /* 21280Sstevel@tonic-gate * Initialize interrupt related stuff 21290Sstevel@tonic-gate */ 21301455Sandrei cpu_intr_alloc(CPU, NINTR_THREADS); 21310Sstevel@tonic-gate 21320Sstevel@tonic-gate (void) splzs(); /* allow hi clock ints but not zs */ 21330Sstevel@tonic-gate 21340Sstevel@tonic-gate /* 21350Sstevel@tonic-gate * Initialize errors. 21360Sstevel@tonic-gate */ 21370Sstevel@tonic-gate error_init(); 21380Sstevel@tonic-gate 21390Sstevel@tonic-gate /* 21400Sstevel@tonic-gate * Note that we may have already used kernel bcopy before this 21410Sstevel@tonic-gate * point - but if you really care about this, adb the use_hw_* 21420Sstevel@tonic-gate * variables to 0 before rebooting. 21430Sstevel@tonic-gate */ 21440Sstevel@tonic-gate mach_hw_copy_limit(); 21450Sstevel@tonic-gate 21460Sstevel@tonic-gate /* 21470Sstevel@tonic-gate * Install the "real" preemption guards before DDI services 21480Sstevel@tonic-gate * are available. 21490Sstevel@tonic-gate */ 21500Sstevel@tonic-gate (void) prom_set_preprom(kern_preprom); 21510Sstevel@tonic-gate (void) prom_set_postprom(kern_postprom); 21520Sstevel@tonic-gate CPU->cpu_m.mutex_ready = 1; 21530Sstevel@tonic-gate 21540Sstevel@tonic-gate /* 21550Sstevel@tonic-gate * Initialize segnf (kernel support for non-faulting loads). 21560Sstevel@tonic-gate */ 21570Sstevel@tonic-gate segnf_init(); 21580Sstevel@tonic-gate 21590Sstevel@tonic-gate /* 21600Sstevel@tonic-gate * Configure the root devinfo node. 21610Sstevel@tonic-gate */ 21620Sstevel@tonic-gate configure(); /* set up devices */ 21630Sstevel@tonic-gate mach_cpu_halt_idle(); 21640Sstevel@tonic-gate } 21650Sstevel@tonic-gate 21660Sstevel@tonic-gate 21670Sstevel@tonic-gate void 21680Sstevel@tonic-gate post_startup(void) 21690Sstevel@tonic-gate { 21700Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 21710Sstevel@tonic-gate extern void init_ptl1_thread(void); 21720Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 21730Sstevel@tonic-gate extern void abort_sequence_init(void); 21740Sstevel@tonic-gate 21750Sstevel@tonic-gate /* 21760Sstevel@tonic-gate * Set the system wide, processor-specific flags to be passed 21770Sstevel@tonic-gate * to userland via the aux vector for performance hints and 21780Sstevel@tonic-gate * instruction set extensions. 21790Sstevel@tonic-gate */ 21800Sstevel@tonic-gate bind_hwcap(); 21810Sstevel@tonic-gate 21820Sstevel@tonic-gate /* 21830Sstevel@tonic-gate * Startup memory scrubber (if any) 21840Sstevel@tonic-gate */ 21850Sstevel@tonic-gate mach_memscrub(); 21860Sstevel@tonic-gate 21870Sstevel@tonic-gate /* 21880Sstevel@tonic-gate * Allocate soft interrupt to handle abort sequence. 21890Sstevel@tonic-gate */ 21900Sstevel@tonic-gate abort_sequence_init(); 21910Sstevel@tonic-gate 21920Sstevel@tonic-gate /* 21930Sstevel@tonic-gate * Configure the rest of the system. 21940Sstevel@tonic-gate * Perform forceloading tasks for /etc/system. 21950Sstevel@tonic-gate */ 21960Sstevel@tonic-gate (void) mod_sysctl(SYS_FORCELOAD, NULL); 21970Sstevel@tonic-gate /* 21980Sstevel@tonic-gate * ON4.0: Force /proc module in until clock interrupt handle fixed 21990Sstevel@tonic-gate * ON4.0: This must be fixed or restated in /etc/systems. 22000Sstevel@tonic-gate */ 22010Sstevel@tonic-gate (void) modload("fs", "procfs"); 22020Sstevel@tonic-gate 22031991Sheppo /* load machine class specific drivers */ 22041991Sheppo load_mach_drivers(); 22051991Sheppo 22061991Sheppo /* load platform specific drivers */ 22070Sstevel@tonic-gate if (&load_platform_drivers) 22080Sstevel@tonic-gate load_platform_drivers(); 22090Sstevel@tonic-gate 22100Sstevel@tonic-gate /* load vis simulation module, if we are running w/fpu off */ 22110Sstevel@tonic-gate if (!fpu_exists) { 22120Sstevel@tonic-gate if (modload("misc", "vis") == -1) 22130Sstevel@tonic-gate halt("Can't load vis"); 22140Sstevel@tonic-gate } 22150Sstevel@tonic-gate 22160Sstevel@tonic-gate mach_fpras(); 22170Sstevel@tonic-gate 22180Sstevel@tonic-gate maxmem = freemem; 22190Sstevel@tonic-gate 22200Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 22210Sstevel@tonic-gate init_ptl1_thread(); 22220Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 22230Sstevel@tonic-gate } 22240Sstevel@tonic-gate 22250Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 22260Sstevel@tonic-gate int ptl1_panic_test = 0; 22270Sstevel@tonic-gate int ptl1_panic_xc_one_test = 0; 22280Sstevel@tonic-gate int ptl1_panic_xc_all_test = 0; 22290Sstevel@tonic-gate int ptl1_panic_xt_one_test = 0; 22300Sstevel@tonic-gate int ptl1_panic_xt_all_test = 0; 22310Sstevel@tonic-gate kthread_id_t ptl1_thread_p = NULL; 22320Sstevel@tonic-gate kcondvar_t ptl1_cv; 22330Sstevel@tonic-gate kmutex_t ptl1_mutex; 22340Sstevel@tonic-gate int ptl1_recurse_count_threshold = 0x40; 22350Sstevel@tonic-gate int ptl1_recurse_trap_threshold = 0x3d; 22360Sstevel@tonic-gate extern void ptl1_recurse(int, int); 22370Sstevel@tonic-gate extern void ptl1_panic_xt(int, int); 22380Sstevel@tonic-gate 22390Sstevel@tonic-gate /* 22400Sstevel@tonic-gate * Called once per second by timeout() to wake up 22410Sstevel@tonic-gate * the ptl1_panic thread to see if it should cause 22420Sstevel@tonic-gate * a trap to the ptl1_panic() code. 22430Sstevel@tonic-gate */ 22440Sstevel@tonic-gate /* ARGSUSED */ 22450Sstevel@tonic-gate static void 22460Sstevel@tonic-gate ptl1_wakeup(void *arg) 22470Sstevel@tonic-gate { 22480Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 22490Sstevel@tonic-gate cv_signal(&ptl1_cv); 22500Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 22510Sstevel@tonic-gate } 22520Sstevel@tonic-gate 22530Sstevel@tonic-gate /* 22540Sstevel@tonic-gate * ptl1_panic cross call function: 22550Sstevel@tonic-gate * Needed because xc_one() and xc_some() can pass 22560Sstevel@tonic-gate * 64 bit args but ptl1_recurse() expects ints. 22570Sstevel@tonic-gate */ 22580Sstevel@tonic-gate static void 22590Sstevel@tonic-gate ptl1_panic_xc(void) 22600Sstevel@tonic-gate { 22610Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 22620Sstevel@tonic-gate ptl1_recurse_trap_threshold); 22630Sstevel@tonic-gate } 22640Sstevel@tonic-gate 22650Sstevel@tonic-gate /* 22660Sstevel@tonic-gate * The ptl1 thread waits for a global flag to be set 22670Sstevel@tonic-gate * and uses the recurse thresholds to set the stack depth 22680Sstevel@tonic-gate * to cause a ptl1_panic() directly via a call to ptl1_recurse 22690Sstevel@tonic-gate * or indirectly via the cross call and cross trap functions. 22700Sstevel@tonic-gate * 22710Sstevel@tonic-gate * This is useful testing stack overflows and normal 22720Sstevel@tonic-gate * ptl1_panic() states with a know stack frame. 22730Sstevel@tonic-gate * 22740Sstevel@tonic-gate * ptl1_recurse() is an asm function in ptl1_panic.s that 22750Sstevel@tonic-gate * sets the {In, Local, Out, and Global} registers to a 22760Sstevel@tonic-gate * know state on the stack and just prior to causing a 22770Sstevel@tonic-gate * test ptl1_panic trap. 22780Sstevel@tonic-gate */ 22790Sstevel@tonic-gate static void 22800Sstevel@tonic-gate ptl1_thread(void) 22810Sstevel@tonic-gate { 22820Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 22830Sstevel@tonic-gate while (ptl1_thread_p) { 22840Sstevel@tonic-gate cpuset_t other_cpus; 22850Sstevel@tonic-gate int cpu_id; 22860Sstevel@tonic-gate int my_cpu_id; 22870Sstevel@tonic-gate int target_cpu_id; 22880Sstevel@tonic-gate int target_found; 22890Sstevel@tonic-gate 22900Sstevel@tonic-gate if (ptl1_panic_test) { 22910Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 22920Sstevel@tonic-gate ptl1_recurse_trap_threshold); 22930Sstevel@tonic-gate } 22940Sstevel@tonic-gate 22950Sstevel@tonic-gate /* 22960Sstevel@tonic-gate * Find potential targets for x-call and x-trap, 22970Sstevel@tonic-gate * if any exist while preempt is disabled we 22980Sstevel@tonic-gate * start a ptl1_panic if requested via a 22990Sstevel@tonic-gate * globals. 23000Sstevel@tonic-gate */ 23010Sstevel@tonic-gate kpreempt_disable(); 23020Sstevel@tonic-gate my_cpu_id = CPU->cpu_id; 23030Sstevel@tonic-gate other_cpus = cpu_ready_set; 23040Sstevel@tonic-gate CPUSET_DEL(other_cpus, CPU->cpu_id); 23050Sstevel@tonic-gate target_found = 0; 23060Sstevel@tonic-gate if (!CPUSET_ISNULL(other_cpus)) { 23070Sstevel@tonic-gate /* 23080Sstevel@tonic-gate * Pick the first one 23090Sstevel@tonic-gate */ 23100Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 23110Sstevel@tonic-gate if (cpu_id == my_cpu_id) 23120Sstevel@tonic-gate continue; 23130Sstevel@tonic-gate 23140Sstevel@tonic-gate if (CPU_XCALL_READY(cpu_id)) { 23150Sstevel@tonic-gate target_cpu_id = cpu_id; 23160Sstevel@tonic-gate target_found = 1; 23170Sstevel@tonic-gate break; 23180Sstevel@tonic-gate } 23190Sstevel@tonic-gate } 23200Sstevel@tonic-gate ASSERT(target_found); 23210Sstevel@tonic-gate 23220Sstevel@tonic-gate if (ptl1_panic_xc_one_test) { 23230Sstevel@tonic-gate xc_one(target_cpu_id, 23240Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 23250Sstevel@tonic-gate } 23260Sstevel@tonic-gate if (ptl1_panic_xc_all_test) { 23270Sstevel@tonic-gate xc_some(other_cpus, 23280Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 23290Sstevel@tonic-gate } 23300Sstevel@tonic-gate if (ptl1_panic_xt_one_test) { 23310Sstevel@tonic-gate xt_one(target_cpu_id, 23320Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 23330Sstevel@tonic-gate } 23340Sstevel@tonic-gate if (ptl1_panic_xt_all_test) { 23350Sstevel@tonic-gate xt_some(other_cpus, 23360Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 23370Sstevel@tonic-gate } 23380Sstevel@tonic-gate } 23390Sstevel@tonic-gate kpreempt_enable(); 23400Sstevel@tonic-gate (void) timeout(ptl1_wakeup, NULL, hz); 23410Sstevel@tonic-gate (void) cv_wait(&ptl1_cv, &ptl1_mutex); 23420Sstevel@tonic-gate } 23430Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 23440Sstevel@tonic-gate } 23450Sstevel@tonic-gate 23460Sstevel@tonic-gate /* 23470Sstevel@tonic-gate * Called during early startup to create the ptl1_thread 23480Sstevel@tonic-gate */ 23490Sstevel@tonic-gate void 23500Sstevel@tonic-gate init_ptl1_thread(void) 23510Sstevel@tonic-gate { 23520Sstevel@tonic-gate ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 23530Sstevel@tonic-gate &p0, TS_RUN, 0); 23540Sstevel@tonic-gate } 23550Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 23560Sstevel@tonic-gate 23570Sstevel@tonic-gate 23585648Ssetje static void 23595648Ssetje memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 23605648Ssetje { 23615648Ssetje struct memlist *new; 23625648Ssetje 23635648Ssetje new = *memlistp; 23645648Ssetje new->address = start; 23655648Ssetje new->size = len; 23665648Ssetje *memlistp = new + 1; 23675648Ssetje } 23685648Ssetje 23690Sstevel@tonic-gate /* 23700Sstevel@tonic-gate * Add to a memory list. 23710Sstevel@tonic-gate * start = start of new memory segment 23720Sstevel@tonic-gate * len = length of new memory segment in bytes 23730Sstevel@tonic-gate * memlistp = pointer to array of available memory segment structures 23740Sstevel@tonic-gate * curmemlistp = memory list to which to add segment. 23750Sstevel@tonic-gate */ 23760Sstevel@tonic-gate static void 23770Sstevel@tonic-gate memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 23780Sstevel@tonic-gate struct memlist **curmemlistp) 23790Sstevel@tonic-gate { 23805648Ssetje struct memlist *new = *memlistp; 23815648Ssetje 23825648Ssetje memlist_new(start, len, memlistp); 23830Sstevel@tonic-gate memlist_insert(new, curmemlistp); 23840Sstevel@tonic-gate } 23850Sstevel@tonic-gate 23865648Ssetje static int 23875648Ssetje ndata_alloc_memseg(struct memlist *ndata, size_t avail) 23885648Ssetje { 23895648Ssetje int nseg; 23905648Ssetje size_t memseg_sz; 23915648Ssetje struct memseg *msp; 23925648Ssetje 23935648Ssetje /* 23945648Ssetje * The memseg list is for the chunks of physical memory that 23955648Ssetje * will be managed by the vm system. The number calculated is 23965648Ssetje * a guess as boot may fragment it more when memory allocations 23975648Ssetje * are made before kphysm_init(). 23985648Ssetje */ 23995648Ssetje memseg_sz = (avail + 10) * sizeof (struct memseg); 24005648Ssetje memseg_sz = roundup(memseg_sz, PAGESIZE); 24015648Ssetje nseg = memseg_sz / sizeof (struct memseg); 24025648Ssetje msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 24035648Ssetje if (msp == NULL) 24045648Ssetje return (1); 24055648Ssetje PRM_DEBUG(memseg_free); 24065648Ssetje 24075648Ssetje while (nseg--) { 24085648Ssetje msp->next = memseg_free; 24095648Ssetje memseg_free = msp; 24105648Ssetje msp++; 24115648Ssetje } 24125648Ssetje return (0); 24135648Ssetje } 24145648Ssetje 24150Sstevel@tonic-gate /* 24160Sstevel@tonic-gate * In the case of architectures that support dynamic addition of 24170Sstevel@tonic-gate * memory at run-time there are two cases where memsegs need to 24180Sstevel@tonic-gate * be initialized and added to the memseg list. 24190Sstevel@tonic-gate * 1) memsegs that are constructed at startup. 24200Sstevel@tonic-gate * 2) memsegs that are constructed at run-time on 24210Sstevel@tonic-gate * hot-plug capable architectures. 24220Sstevel@tonic-gate * This code was originally part of the function kphysm_init(). 24230Sstevel@tonic-gate */ 24240Sstevel@tonic-gate 24250Sstevel@tonic-gate static void 24260Sstevel@tonic-gate memseg_list_add(struct memseg *memsegp) 24270Sstevel@tonic-gate { 24280Sstevel@tonic-gate struct memseg **prev_memsegp; 24290Sstevel@tonic-gate pgcnt_t num; 24300Sstevel@tonic-gate 24310Sstevel@tonic-gate /* insert in memseg list, decreasing number of pages order */ 24320Sstevel@tonic-gate 24330Sstevel@tonic-gate num = MSEG_NPAGES(memsegp); 24340Sstevel@tonic-gate 24350Sstevel@tonic-gate for (prev_memsegp = &memsegs; *prev_memsegp; 24360Sstevel@tonic-gate prev_memsegp = &((*prev_memsegp)->next)) { 24370Sstevel@tonic-gate if (num > MSEG_NPAGES(*prev_memsegp)) 24380Sstevel@tonic-gate break; 24390Sstevel@tonic-gate } 24400Sstevel@tonic-gate 24410Sstevel@tonic-gate memsegp->next = *prev_memsegp; 24420Sstevel@tonic-gate *prev_memsegp = memsegp; 24430Sstevel@tonic-gate 24440Sstevel@tonic-gate if (kpm_enable) { 24450Sstevel@tonic-gate memsegp->nextpa = (memsegp->next) ? 24464528Spaulsan va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 24470Sstevel@tonic-gate 24480Sstevel@tonic-gate if (prev_memsegp != &memsegs) { 24490Sstevel@tonic-gate struct memseg *msp; 24500Sstevel@tonic-gate msp = (struct memseg *)((caddr_t)prev_memsegp - 24514528Spaulsan offsetof(struct memseg, next)); 24520Sstevel@tonic-gate msp->nextpa = va_to_pa(memsegp); 24530Sstevel@tonic-gate } else { 24540Sstevel@tonic-gate memsegspa = va_to_pa(memsegs); 24550Sstevel@tonic-gate } 24560Sstevel@tonic-gate } 24570Sstevel@tonic-gate } 24580Sstevel@tonic-gate 24590Sstevel@tonic-gate /* 24600Sstevel@tonic-gate * PSM add_physmem_cb(). US-II and newer processors have some 24610Sstevel@tonic-gate * flavor of the prefetch capability implemented. We exploit 24620Sstevel@tonic-gate * this capability for optimum performance. 24630Sstevel@tonic-gate */ 24640Sstevel@tonic-gate #define PREFETCH_BYTES 64 24650Sstevel@tonic-gate 24660Sstevel@tonic-gate void 24670Sstevel@tonic-gate add_physmem_cb(page_t *pp, pfn_t pnum) 24680Sstevel@tonic-gate { 24690Sstevel@tonic-gate extern void prefetch_page_w(void *); 24700Sstevel@tonic-gate 24710Sstevel@tonic-gate pp->p_pagenum = pnum; 24720Sstevel@tonic-gate 24730Sstevel@tonic-gate /* 24740Sstevel@tonic-gate * Prefetch one more page_t into E$. To prevent future 24750Sstevel@tonic-gate * mishaps with the sizeof(page_t) changing on us, we 24760Sstevel@tonic-gate * catch this on debug kernels if we can't bring in the 24770Sstevel@tonic-gate * entire hpage with 2 PREFETCH_BYTES reads. See 24780Sstevel@tonic-gate * also, sun4u/cpu/cpu_module.c 24790Sstevel@tonic-gate */ 24800Sstevel@tonic-gate /*LINTED*/ 24810Sstevel@tonic-gate ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 24820Sstevel@tonic-gate prefetch_page_w((char *)pp); 24830Sstevel@tonic-gate } 24840Sstevel@tonic-gate 24850Sstevel@tonic-gate /* 24865648Ssetje * Find memseg with given pfn 24875648Ssetje */ 24885648Ssetje static struct memseg * 24895648Ssetje memseg_find(pfn_t base, pfn_t *next) 24905648Ssetje { 24915648Ssetje struct memseg *seg; 24925648Ssetje 24935648Ssetje if (next != NULL) 24945648Ssetje *next = LONG_MAX; 24955648Ssetje for (seg = memsegs; seg != NULL; seg = seg->next) { 24965648Ssetje if (base >= seg->pages_base && base < seg->pages_end) 24975648Ssetje return (seg); 24985648Ssetje if (next != NULL && seg->pages_base > base && 24995648Ssetje seg->pages_base < *next) 25005648Ssetje *next = seg->pages_base; 25015648Ssetje } 25025648Ssetje return (NULL); 25035648Ssetje } 25045648Ssetje 25055648Ssetje extern struct vnode prom_ppages; 25065648Ssetje 25075648Ssetje /* 25085648Ssetje * Put page allocated by OBP on prom_ppages 25090Sstevel@tonic-gate */ 25100Sstevel@tonic-gate static void 25115648Ssetje kphysm_erase(uint64_t addr, uint64_t len) 25125648Ssetje { 25135648Ssetje struct page *pp; 25145648Ssetje struct memseg *seg; 25155648Ssetje pfn_t base = btop(addr), next; 25165648Ssetje pgcnt_t num = btop(len); 25175648Ssetje 25185648Ssetje while (num != 0) { 25195648Ssetje pgcnt_t off, left; 25205648Ssetje 25215648Ssetje seg = memseg_find(base, &next); 25225648Ssetje if (seg == NULL) { 25235648Ssetje if (next == LONG_MAX) 25245648Ssetje break; 25255648Ssetje left = MIN(next - base, num); 25265648Ssetje base += left, num -= left; 25275648Ssetje continue; 25285648Ssetje } 25295648Ssetje off = base - seg->pages_base; 25305648Ssetje pp = seg->pages + off; 25315648Ssetje left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 25325648Ssetje while (num != left) { 25335648Ssetje /* 25345648Ssetje * init it, lock it, and hashin on prom_pages vp. 25355648Ssetje * 25365648Ssetje * XXX vnode offsets on the prom_ppages vnode 25375648Ssetje * are page numbers (gack) for >32 bit 25385648Ssetje * physical memory machines. 25395648Ssetje */ 25405648Ssetje add_physmem_cb(pp, base); 25415648Ssetje if (page_trylock(pp, SE_EXCL) == 0) 25425648Ssetje cmn_err(CE_PANIC, "prom page locked"); 25435648Ssetje (void) page_hashin(pp, &prom_ppages, 25445648Ssetje (offset_t)base, NULL); 25455648Ssetje (void) page_pp_lock(pp, 0, 1); 25465648Ssetje pp++, base++, num--; 25475648Ssetje } 25485648Ssetje } 25495648Ssetje } 25505648Ssetje 25515648Ssetje static page_t *ppnext; 25525648Ssetje static pgcnt_t ppleft; 25535648Ssetje 25545648Ssetje static void *kpm_ppnext; 25555648Ssetje static pgcnt_t kpm_ppleft; 25565648Ssetje 25575648Ssetje /* 25585648Ssetje * Create a memseg 25595648Ssetje */ 25605648Ssetje static void 25615648Ssetje kphysm_memseg(uint64_t addr, uint64_t len) 25620Sstevel@tonic-gate { 25635648Ssetje pfn_t base = btop(addr); 25645648Ssetje pgcnt_t num = btop(len); 25655648Ssetje struct memseg *seg; 25665648Ssetje 25675648Ssetje seg = memseg_free; 25685648Ssetje memseg_free = seg->next; 25695648Ssetje ASSERT(seg != NULL); 25705648Ssetje 25715648Ssetje seg->pages = ppnext; 25725648Ssetje seg->epages = ppnext + num; 25735648Ssetje seg->pages_base = base; 25745648Ssetje seg->pages_end = base + num; 25755648Ssetje ppnext += num; 25765648Ssetje ppleft -= num; 25775648Ssetje 25785648Ssetje if (kpm_enable) { 25795648Ssetje pgcnt_t kpnum = ptokpmpr(num); 25805648Ssetje 25815648Ssetje if (kpnum > kpm_ppleft) 25825648Ssetje panic("kphysm_memseg: kpm_pp overflow"); 25835648Ssetje seg->pagespa = va_to_pa(seg->pages); 25845648Ssetje seg->epagespa = va_to_pa(seg->epages); 25855648Ssetje seg->kpm_pbase = kpmptop(ptokpmp(base)); 25865648Ssetje seg->kpm_nkpmpgs = kpnum; 25875648Ssetje /* 25885648Ssetje * In the kpm_smallpage case, the kpm array 25895648Ssetje * is 1-1 wrt the page array 25905648Ssetje */ 25915648Ssetje if (kpm_smallpages) { 25925648Ssetje kpm_spage_t *kpm_pp = kpm_ppnext; 25935648Ssetje 25945648Ssetje kpm_ppnext = kpm_pp + kpnum; 25955648Ssetje seg->kpm_spages = kpm_pp; 25965648Ssetje seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 25975648Ssetje } else { 25985648Ssetje kpm_page_t *kpm_pp = kpm_ppnext; 25995648Ssetje 26005648Ssetje kpm_ppnext = kpm_pp + kpnum; 26015648Ssetje seg->kpm_pages = kpm_pp; 26025648Ssetje seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 26035648Ssetje /* ASSERT no kpm overlaps */ 26045648Ssetje ASSERT( 26055648Ssetje memseg_find(base - pmodkpmp(base), NULL) == NULL); 26065648Ssetje ASSERT(memseg_find( 26075648Ssetje roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 26085648Ssetje } 26095648Ssetje kpm_ppleft -= num; 26105648Ssetje } 26115648Ssetje 26125648Ssetje memseg_list_add(seg); 26135648Ssetje } 26145648Ssetje 26155648Ssetje /* 26165648Ssetje * Add range to free list 26175648Ssetje */ 26185648Ssetje void 26195648Ssetje kphysm_add(uint64_t addr, uint64_t len, int reclaim) 26205648Ssetje { 26215648Ssetje struct page *pp; 26225648Ssetje struct memseg *seg; 26235648Ssetje pfn_t base = btop(addr); 26245648Ssetje pgcnt_t num = btop(len); 26255648Ssetje 26265648Ssetje seg = memseg_find(base, NULL); 26275648Ssetje ASSERT(seg != NULL); 26285648Ssetje pp = seg->pages + (base - seg->pages_base); 26295648Ssetje 26305648Ssetje if (reclaim) { 26315648Ssetje struct page *rpp = pp; 26325648Ssetje struct page *lpp = pp + num; 26330Sstevel@tonic-gate 26340Sstevel@tonic-gate /* 26355648Ssetje * page should be locked on prom_ppages 26365648Ssetje * unhash and unlock it 26370Sstevel@tonic-gate */ 26385648Ssetje while (rpp < lpp) { 26395648Ssetje ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages); 26405648Ssetje page_pp_unlock(rpp, 0, 1); 26415648Ssetje page_hashout(rpp, NULL); 26425648Ssetje page_unlock(rpp); 26435648Ssetje rpp++; 26440Sstevel@tonic-gate } 26450Sstevel@tonic-gate } 26460Sstevel@tonic-gate 26475648Ssetje /* 26485648Ssetje * add_physmem() initializes the PSM part of the page 26495648Ssetje * struct by calling the PSM back with add_physmem_cb(). 26505648Ssetje * In addition it coalesces pages into larger pages as 26515648Ssetje * it initializes them. 26525648Ssetje */ 26535648Ssetje add_physmem(pp, num, base); 26545648Ssetje } 26555648Ssetje 26565648Ssetje /* 26575648Ssetje * kphysm_init() tackles the problem of initializing physical memory. 26585648Ssetje */ 26595648Ssetje static void 26605648Ssetje kphysm_init(void) 26615648Ssetje { 26625648Ssetje struct memlist *pmem; 26635648Ssetje 26645648Ssetje ASSERT(page_hash != NULL && page_hashsz != 0); 26655648Ssetje 26665648Ssetje ppnext = pp_base; 26675648Ssetje ppleft = npages; 26685648Ssetje kpm_ppnext = kpm_pp_base; 26695648Ssetje kpm_ppleft = kpm_npages; 26705648Ssetje 26715648Ssetje /* 26725648Ssetje * installed pages not on nopp_memlist go in memseg list 26735648Ssetje */ 26745648Ssetje diff_memlists(phys_install, nopp_list, kphysm_memseg); 26755648Ssetje 26765648Ssetje /* 26775648Ssetje * Free the avail list 26785648Ssetje */ 26795648Ssetje for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 26805648Ssetje kphysm_add(pmem->address, pmem->size, 0); 26815648Ssetje 26825648Ssetje /* 26835648Ssetje * Erase pages that aren't available 26845648Ssetje */ 26855648Ssetje diff_memlists(phys_install, phys_avail, kphysm_erase); 26865648Ssetje 26870Sstevel@tonic-gate build_pfn_hash(); 26880Sstevel@tonic-gate } 26890Sstevel@tonic-gate 26900Sstevel@tonic-gate /* 26910Sstevel@tonic-gate * Kernel VM initialization. 26920Sstevel@tonic-gate * Assumptions about kernel address space ordering: 26930Sstevel@tonic-gate * (1) gap (user space) 26940Sstevel@tonic-gate * (2) kernel text 26950Sstevel@tonic-gate * (3) kernel data/bss 26960Sstevel@tonic-gate * (4) gap 26970Sstevel@tonic-gate * (5) kernel data structures 26980Sstevel@tonic-gate * (6) gap 26990Sstevel@tonic-gate * (7) debugger (optional) 27000Sstevel@tonic-gate * (8) monitor 27010Sstevel@tonic-gate * (9) gap (possibly null) 27020Sstevel@tonic-gate * (10) dvma 27030Sstevel@tonic-gate * (11) devices 27040Sstevel@tonic-gate */ 27050Sstevel@tonic-gate static void 27060Sstevel@tonic-gate kvm_init(void) 27070Sstevel@tonic-gate { 27080Sstevel@tonic-gate /* 27090Sstevel@tonic-gate * Put the kernel segments in kernel address space. 27100Sstevel@tonic-gate */ 27110Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 27120Sstevel@tonic-gate as_avlinit(&kas); 27130Sstevel@tonic-gate 27140Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)KERNELBASE, 27150Sstevel@tonic-gate (size_t)(e_moddata - KERNELBASE), &ktextseg); 27160Sstevel@tonic-gate (void) segkmem_create(&ktextseg); 27170Sstevel@tonic-gate 27180Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 27190Sstevel@tonic-gate (size_t)(MMU_PAGESIZE4M), &ktexthole); 27200Sstevel@tonic-gate (void) segkmem_create(&ktexthole); 27210Sstevel@tonic-gate 27220Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)valloc_base, 27230Sstevel@tonic-gate (size_t)(econtig32 - valloc_base), &kvalloc); 27240Sstevel@tonic-gate (void) segkmem_create(&kvalloc); 27250Sstevel@tonic-gate 27260Sstevel@tonic-gate if (kmem64_base) { 27274528Spaulsan (void) seg_attach(&kas, (caddr_t)kmem64_base, 27284528Spaulsan (size_t)(kmem64_end - kmem64_base), &kmem64); 27294528Spaulsan (void) segkmem_create(&kmem64); 27300Sstevel@tonic-gate } 27310Sstevel@tonic-gate 27320Sstevel@tonic-gate /* 27330Sstevel@tonic-gate * We're about to map out /boot. This is the beginning of the 27340Sstevel@tonic-gate * system resource management transition. We can no longer 27350Sstevel@tonic-gate * call into /boot for I/O or memory allocations. 27360Sstevel@tonic-gate */ 27370Sstevel@tonic-gate (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 27380Sstevel@tonic-gate (void) segkmem_create(&kvseg); 27390Sstevel@tonic-gate hblk_alloc_dynamic = 1; 27400Sstevel@tonic-gate 27410Sstevel@tonic-gate /* 27420Sstevel@tonic-gate * we need to preallocate pages for DR operations before enabling large 27430Sstevel@tonic-gate * page kernel heap because of memseg_remap_init() hat_unload() hack. 27440Sstevel@tonic-gate */ 27450Sstevel@tonic-gate memseg_remap_init(); 27460Sstevel@tonic-gate 27470Sstevel@tonic-gate /* at this point we are ready to use large page heap */ 27480Sstevel@tonic-gate segkmem_heap_lp_init(); 27490Sstevel@tonic-gate 27500Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 27510Sstevel@tonic-gate &kvseg32); 27520Sstevel@tonic-gate (void) segkmem_create(&kvseg32); 27530Sstevel@tonic-gate 27540Sstevel@tonic-gate /* 27550Sstevel@tonic-gate * Create a segment for the debugger. 27560Sstevel@tonic-gate */ 27573446Smrj (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 27580Sstevel@tonic-gate (void) segkmem_create(&kdebugseg); 27590Sstevel@tonic-gate 27600Sstevel@tonic-gate rw_exit(&kas.a_lock); 27610Sstevel@tonic-gate } 27620Sstevel@tonic-gate 27630Sstevel@tonic-gate char obp_tte_str[] = 27640Sstevel@tonic-gate "h# %x constant MMU_PAGESHIFT " 27650Sstevel@tonic-gate "h# %x constant TTE8K " 27660Sstevel@tonic-gate "h# %x constant SFHME_SIZE " 27670Sstevel@tonic-gate "h# %x constant SFHME_TTE " 27680Sstevel@tonic-gate "h# %x constant HMEBLK_TAG " 27690Sstevel@tonic-gate "h# %x constant HMEBLK_NEXT " 27700Sstevel@tonic-gate "h# %x constant HMEBLK_MISC " 27710Sstevel@tonic-gate "h# %x constant HMEBLK_HME1 " 27720Sstevel@tonic-gate "h# %x constant NHMENTS " 27730Sstevel@tonic-gate "h# %x constant HBLK_SZMASK " 27740Sstevel@tonic-gate "h# %x constant HBLK_RANGE_SHIFT " 27750Sstevel@tonic-gate "h# %x constant HMEBP_HBLK " 27760Sstevel@tonic-gate "h# %x constant HMEBUCKET_SIZE " 27770Sstevel@tonic-gate "h# %x constant HTAG_SFMMUPSZ " 27784528Spaulsan "h# %x constant HTAG_BSPAGE_SHIFT " 27794528Spaulsan "h# %x constant HTAG_REHASH_SHIFT " 27804528Spaulsan "h# %x constant SFMMU_INVALID_SHMERID " 27810Sstevel@tonic-gate "h# %x constant mmu_hashcnt " 27820Sstevel@tonic-gate "h# %p constant uhme_hash " 27830Sstevel@tonic-gate "h# %p constant khme_hash " 27840Sstevel@tonic-gate "h# %x constant UHMEHASH_SZ " 27850Sstevel@tonic-gate "h# %x constant KHMEHASH_SZ " 27862241Shuah "h# %p constant KCONTEXT " 27870Sstevel@tonic-gate "h# %p constant KHATID " 27880Sstevel@tonic-gate "h# %x constant ASI_MEM " 27890Sstevel@tonic-gate 27900Sstevel@tonic-gate ": PHYS-X@ ( phys -- data ) " 27910Sstevel@tonic-gate " ASI_MEM spacex@ " 27920Sstevel@tonic-gate "; " 27930Sstevel@tonic-gate 27940Sstevel@tonic-gate ": PHYS-W@ ( phys -- data ) " 27950Sstevel@tonic-gate " ASI_MEM spacew@ " 27960Sstevel@tonic-gate "; " 27970Sstevel@tonic-gate 27980Sstevel@tonic-gate ": PHYS-L@ ( phys -- data ) " 27990Sstevel@tonic-gate " ASI_MEM spaceL@ " 28000Sstevel@tonic-gate "; " 28010Sstevel@tonic-gate 28020Sstevel@tonic-gate ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 28030Sstevel@tonic-gate " 3 * MMU_PAGESHIFT + " 28040Sstevel@tonic-gate "; " 28050Sstevel@tonic-gate 28060Sstevel@tonic-gate ": TTE_IS_VALID ( ttep -- flag ) " 28070Sstevel@tonic-gate " PHYS-X@ 0< " 28080Sstevel@tonic-gate "; " 28090Sstevel@tonic-gate 28100Sstevel@tonic-gate ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 28110Sstevel@tonic-gate " dup TTE8K = if " 28120Sstevel@tonic-gate " drop HBLK_RANGE_SHIFT " 28130Sstevel@tonic-gate " else " 28140Sstevel@tonic-gate " TTE_PAGE_SHIFT " 28150Sstevel@tonic-gate " then " 28160Sstevel@tonic-gate "; " 28170Sstevel@tonic-gate 28180Sstevel@tonic-gate ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 28190Sstevel@tonic-gate " tuck >> swap MMU_PAGESHIFT - << " 28200Sstevel@tonic-gate "; " 28210Sstevel@tonic-gate 28220Sstevel@tonic-gate ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 28230Sstevel@tonic-gate " >> over xor swap ( hash sfmmup ) " 28240Sstevel@tonic-gate " KHATID <> if ( hash ) " 28250Sstevel@tonic-gate " UHMEHASH_SZ and ( bucket ) " 28260Sstevel@tonic-gate " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 28270Sstevel@tonic-gate " else ( hash ) " 28280Sstevel@tonic-gate " KHMEHASH_SZ and ( bucket ) " 28290Sstevel@tonic-gate " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 28300Sstevel@tonic-gate " then ( hmebp ) " 28310Sstevel@tonic-gate "; " 28320Sstevel@tonic-gate 28330Sstevel@tonic-gate ": HME_HASH_TABLE_SEARCH " 28340Sstevel@tonic-gate " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 28350Sstevel@tonic-gate " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 28360Sstevel@tonic-gate " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 28370Sstevel@tonic-gate " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 28380Sstevel@tonic-gate " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 28390Sstevel@tonic-gate " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 28400Sstevel@tonic-gate " else " 28410Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 28420Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 28430Sstevel@tonic-gate " then " 28440Sstevel@tonic-gate " else " 28450Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 28460Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 28470Sstevel@tonic-gate " then " 28480Sstevel@tonic-gate " else " 28490Sstevel@tonic-gate " true " 28500Sstevel@tonic-gate " then " 28510Sstevel@tonic-gate " until r> drop " 28520Sstevel@tonic-gate "; " 28530Sstevel@tonic-gate 28540Sstevel@tonic-gate ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 28554528Spaulsan " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 28564528Spaulsan " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 28574528Spaulsan " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 28584528Spaulsan " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 28590Sstevel@tonic-gate "; " 28600Sstevel@tonic-gate 28610Sstevel@tonic-gate ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 28620Sstevel@tonic-gate " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 28630Sstevel@tonic-gate " TTE8K = if ( hmeblkp addr ) " 28640Sstevel@tonic-gate " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 28650Sstevel@tonic-gate " else ( hmeblkp addr ) " 28660Sstevel@tonic-gate " drop 0 ( hmeblkp 0 ) " 28670Sstevel@tonic-gate " then ( hmeblkp hme-index ) " 28680Sstevel@tonic-gate " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 28690Sstevel@tonic-gate " SFHME_TTE + ( ttep ) " 28700Sstevel@tonic-gate "; " 28710Sstevel@tonic-gate 28720Sstevel@tonic-gate ": unix-tte ( addr cnum -- false | tte-data true ) " 28732241Shuah " KCONTEXT = if ( addr ) " 28742241Shuah " KHATID ( addr khatid ) " 28752241Shuah " else ( addr ) " 28762241Shuah " drop false exit ( false ) " 28772241Shuah " then " 28782241Shuah " ( addr khatid ) " 28790Sstevel@tonic-gate " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 28800Sstevel@tonic-gate " 2dup swap i HME_HASH_SHIFT " 28810Sstevel@tonic-gate "( addr sfmmup sfmmup addr hmeshift ) " 28820Sstevel@tonic-gate " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 28830Sstevel@tonic-gate " over i 4 pick " 28840Sstevel@tonic-gate "( addr sfmmup hmebp sfmmup rehash addr ) " 28850Sstevel@tonic-gate " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 28860Sstevel@tonic-gate " HME_HASH_TABLE_SEARCH " 28870Sstevel@tonic-gate "( addr sfmmup { null | hmeblkp } ) " 28880Sstevel@tonic-gate " ?dup if ( addr sfmmup hmeblkp ) " 28890Sstevel@tonic-gate " nip swap HBLK_TO_TTEP ( ttep ) " 28900Sstevel@tonic-gate " dup TTE_IS_VALID if ( valid-ttep ) " 28910Sstevel@tonic-gate " PHYS-X@ true ( tte-data true ) " 28920Sstevel@tonic-gate " else ( invalid-tte ) " 28930Sstevel@tonic-gate " drop false ( false ) " 28940Sstevel@tonic-gate " then ( false | tte-data true ) " 28950Sstevel@tonic-gate " unloop exit ( false | tte-data true ) " 28960Sstevel@tonic-gate " then ( addr sfmmup ) " 28970Sstevel@tonic-gate " loop ( addr sfmmup ) " 28980Sstevel@tonic-gate " 2drop false ( false ) " 28990Sstevel@tonic-gate "; " 29000Sstevel@tonic-gate ; 29010Sstevel@tonic-gate 29020Sstevel@tonic-gate void 29030Sstevel@tonic-gate create_va_to_tte(void) 29040Sstevel@tonic-gate { 29050Sstevel@tonic-gate char *bp; 29060Sstevel@tonic-gate extern int khmehash_num, uhmehash_num; 29070Sstevel@tonic-gate extern struct hmehash_bucket *khme_hash, *uhme_hash; 29080Sstevel@tonic-gate 29090Sstevel@tonic-gate #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 29100Sstevel@tonic-gate 29110Sstevel@tonic-gate bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 29120Sstevel@tonic-gate 29130Sstevel@tonic-gate /* 29140Sstevel@tonic-gate * Teach obp how to parse our sw ttes. 29150Sstevel@tonic-gate */ 29160Sstevel@tonic-gate (void) sprintf(bp, obp_tte_str, 29170Sstevel@tonic-gate MMU_PAGESHIFT, 29180Sstevel@tonic-gate TTE8K, 29190Sstevel@tonic-gate sizeof (struct sf_hment), 29200Sstevel@tonic-gate OFFSET(struct sf_hment, hme_tte), 29210Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_tag), 29220Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_nextpa), 29230Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_misc), 29240Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_hme), 29250Sstevel@tonic-gate NHMENTS, 29260Sstevel@tonic-gate HBLK_SZMASK, 29270Sstevel@tonic-gate HBLK_RANGE_SHIFT, 29280Sstevel@tonic-gate OFFSET(struct hmehash_bucket, hmeh_nextpa), 29290Sstevel@tonic-gate sizeof (struct hmehash_bucket), 29300Sstevel@tonic-gate HTAG_SFMMUPSZ, 29314528Spaulsan HTAG_BSPAGE_SHIFT, 29324528Spaulsan HTAG_REHASH_SHIFT, 29334528Spaulsan SFMMU_INVALID_SHMERID, 29340Sstevel@tonic-gate mmu_hashcnt, 29350Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)uhme_hash), 29360Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)khme_hash), 29370Sstevel@tonic-gate UHMEHASH_SZ, 29380Sstevel@tonic-gate KHMEHASH_SZ, 29392241Shuah KCONTEXT, 29400Sstevel@tonic-gate KHATID, 29410Sstevel@tonic-gate ASI_MEM); 29420Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 29430Sstevel@tonic-gate 29440Sstevel@tonic-gate kobj_free(bp, MMU_PAGESIZE); 29450Sstevel@tonic-gate } 29460Sstevel@tonic-gate 29470Sstevel@tonic-gate void 29480Sstevel@tonic-gate install_va_to_tte(void) 29490Sstevel@tonic-gate { 29500Sstevel@tonic-gate /* 29510Sstevel@tonic-gate * advise prom that he can use unix-tte 29520Sstevel@tonic-gate */ 29530Sstevel@tonic-gate prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 29540Sstevel@tonic-gate } 29550Sstevel@tonic-gate 29561253Slq150181 /* 29573928Slq150181 * Here we add "device-type=console" for /os-io node, for currently 29583928Slq150181 * our kernel console output only supports displaying text and 29593928Slq150181 * performing cursor-positioning operations (through kernel framebuffer 29603928Slq150181 * driver) and it doesn't support other functionalities required for a 29613928Slq150181 * standard "display" device as specified in 1275 spec. The main missing 29623928Slq150181 * interface defined by the 1275 spec is "draw-logo". 29633928Slq150181 * also see the comments above prom_stdout_is_framebuffer(). 29641253Slq150181 */ 29650Sstevel@tonic-gate static char *create_node = 29661492Slq150181 "\" /\" find-device " 29670Sstevel@tonic-gate "new-device " 29680Sstevel@tonic-gate "\" os-io\" device-name " 29693928Slq150181 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 29700Sstevel@tonic-gate ": cb-r/w ( adr,len method$ -- #read/#written ) " 29710Sstevel@tonic-gate " 2>r swap 2 2r> ['] $callback catch if " 29720Sstevel@tonic-gate " 2drop 3drop 0 " 29730Sstevel@tonic-gate " then " 29740Sstevel@tonic-gate "; " 29750Sstevel@tonic-gate ": read ( adr,len -- #read ) " 29760Sstevel@tonic-gate " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 29770Sstevel@tonic-gate " ( retN ... ret1 N ) " 29780Sstevel@tonic-gate " ?dup if " 29790Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 29800Sstevel@tonic-gate " else " 29810Sstevel@tonic-gate " -2 " 29821492Slq150181 " then " 29830Sstevel@tonic-gate "; " 29840Sstevel@tonic-gate ": write ( adr,len -- #written ) " 29850Sstevel@tonic-gate " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 29860Sstevel@tonic-gate " ( retN ... ret1 N ) " 29870Sstevel@tonic-gate " ?dup if " 29880Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 29890Sstevel@tonic-gate " else " 29900Sstevel@tonic-gate " 0 " 29910Sstevel@tonic-gate " then " 29920Sstevel@tonic-gate "; " 29930Sstevel@tonic-gate ": poll-tty ( -- ) ; " 29940Sstevel@tonic-gate ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 29950Sstevel@tonic-gate ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 29960Sstevel@tonic-gate ": cb-give/take ( $method -- ) " 29970Sstevel@tonic-gate " 0 -rot ['] $callback catch ?dup if " 29980Sstevel@tonic-gate " >r 2drop 2drop r> throw " 29990Sstevel@tonic-gate " else " 30000Sstevel@tonic-gate " 0 ?do drop loop " 30010Sstevel@tonic-gate " then " 30020Sstevel@tonic-gate "; " 30030Sstevel@tonic-gate ": give ( -- ) \" exit-input\" cb-give/take ; " 30040Sstevel@tonic-gate ": take ( -- ) \" enter-input\" cb-give/take ; " 30050Sstevel@tonic-gate ": open ( -- ok? ) true ; " 30060Sstevel@tonic-gate ": close ( -- ) ; " 30070Sstevel@tonic-gate "finish-device " 30080Sstevel@tonic-gate "device-end "; 30090Sstevel@tonic-gate 30100Sstevel@tonic-gate /* 30111253Slq150181 * Create the OBP input/output node (FCode serial driver). 30121253Slq150181 * It is needed for both USB console keyboard and for 30131253Slq150181 * the kernel terminal emulator. It is too early to check for a 30141253Slq150181 * kernel console compatible framebuffer now, so we create this 30151253Slq150181 * so that we're ready if we need to enable kernel terminal emulation. 30160Sstevel@tonic-gate * 30171253Slq150181 * When the USB software takes over the input device at the time 30181253Slq150181 * consconfig runs, OBP's stdin is redirected to this node. 30191253Slq150181 * Whenever the FORTH user interface is used after this switch, 30201253Slq150181 * the node will call back into the kernel for console input. 30211253Slq150181 * If a serial device such as ttya or a UART with a Type 5 keyboard 30221253Slq150181 * attached is used, OBP takes over the serial device when the system 30231253Slq150181 * goes to the debugger after the system is booted. This sharing 30241253Slq150181 * of the relatively simple serial device is difficult but possible. 30251253Slq150181 * Sharing the USB host controller is impossible due its complexity. 30261253Slq150181 * 30271253Slq150181 * Similarly to USB keyboard input redirection, after consconfig_dacf 30281253Slq150181 * configures a kernel console framebuffer as the standard output 30291253Slq150181 * device, OBP's stdout is switched to to vector through the 30301253Slq150181 * /os-io node into the kernel terminal emulator. 30310Sstevel@tonic-gate */ 30320Sstevel@tonic-gate static void 30331253Slq150181 startup_create_io_node(void) 30340Sstevel@tonic-gate { 30351253Slq150181 prom_interpret(create_node, 0, 0, 0, 0, 0); 30360Sstevel@tonic-gate } 30370Sstevel@tonic-gate 30380Sstevel@tonic-gate 30390Sstevel@tonic-gate static void 30400Sstevel@tonic-gate do_prom_version_check(void) 30410Sstevel@tonic-gate { 30420Sstevel@tonic-gate int i; 3043789Sahrens pnode_t node; 30440Sstevel@tonic-gate char buf[64]; 30450Sstevel@tonic-gate static char drev[] = "Down-rev firmware detected%s\n" 30464528Spaulsan "\tPlease upgrade to the following minimum version:\n" 30474528Spaulsan "\t\t%s\n"; 30480Sstevel@tonic-gate 30490Sstevel@tonic-gate i = prom_version_check(buf, sizeof (buf), &node); 30500Sstevel@tonic-gate 30510Sstevel@tonic-gate if (i == PROM_VER64_OK) 30520Sstevel@tonic-gate return; 30530Sstevel@tonic-gate 30540Sstevel@tonic-gate if (i == PROM_VER64_UPGRADE) { 30550Sstevel@tonic-gate cmn_err(CE_WARN, drev, "", buf); 30560Sstevel@tonic-gate 30570Sstevel@tonic-gate #ifdef DEBUG 30580Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to continue */ 30590Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 30600Sstevel@tonic-gate return; 30610Sstevel@tonic-gate #else 30620Sstevel@tonic-gate halt(0); 30630Sstevel@tonic-gate #endif 30640Sstevel@tonic-gate } 30650Sstevel@tonic-gate 30660Sstevel@tonic-gate /* 30670Sstevel@tonic-gate * The other possibility is that this is a server running 30680Sstevel@tonic-gate * good firmware, but down-rev firmware was detected on at 30690Sstevel@tonic-gate * least one other cpu board. We just complain if we see 30700Sstevel@tonic-gate * that. 30710Sstevel@tonic-gate */ 30720Sstevel@tonic-gate cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 30730Sstevel@tonic-gate } 30740Sstevel@tonic-gate 30750Sstevel@tonic-gate 30760Sstevel@tonic-gate /* 30770Sstevel@tonic-gate * Must be defined in platform dependent code. 30780Sstevel@tonic-gate */ 30790Sstevel@tonic-gate extern caddr_t modtext; 30800Sstevel@tonic-gate extern size_t modtext_sz; 30810Sstevel@tonic-gate extern caddr_t moddata; 30820Sstevel@tonic-gate 30830Sstevel@tonic-gate #define HEAPTEXT_ARENA(addr) \ 30840Sstevel@tonic-gate ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 30850Sstevel@tonic-gate (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 30860Sstevel@tonic-gate (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 30870Sstevel@tonic-gate 30880Sstevel@tonic-gate #define HEAPTEXT_OVERSIZED(addr) \ 30890Sstevel@tonic-gate ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 30900Sstevel@tonic-gate 30917428SJonathan.Haslam@Sun.COM #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 30927428SJonathan.Haslam@Sun.COM (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 30937428SJonathan.Haslam@Sun.COM ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 30947428SJonathan.Haslam@Sun.COM 30950Sstevel@tonic-gate vmem_t *texthole_source[HEAPTEXT_NARENAS]; 30960Sstevel@tonic-gate vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 30970Sstevel@tonic-gate kmutex_t texthole_lock; 30980Sstevel@tonic-gate 30990Sstevel@tonic-gate char kern_bootargs[OBP_MAXPATHLEN]; 3100*7656SSherry.Moore@Sun.COM char kern_bootfile[OBP_MAXPATHLEN]; 31010Sstevel@tonic-gate 31020Sstevel@tonic-gate void 31030Sstevel@tonic-gate kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 31040Sstevel@tonic-gate { 31050Sstevel@tonic-gate uintptr_t addr, limit; 31060Sstevel@tonic-gate 31070Sstevel@tonic-gate addr = HEAPTEXT_BASE; 31080Sstevel@tonic-gate limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 31090Sstevel@tonic-gate 31100Sstevel@tonic-gate /* 31110Sstevel@tonic-gate * Before we initialize the text_arena, we want to punch holes in the 31120Sstevel@tonic-gate * underlying heaptext_arena. This guarantees that for any text 31130Sstevel@tonic-gate * address we can find a text hole less than HEAPTEXT_MAPPED away. 31140Sstevel@tonic-gate */ 31150Sstevel@tonic-gate for (; addr + HEAPTEXT_UNMAPPED <= limit; 31160Sstevel@tonic-gate addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 31170Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 31180Sstevel@tonic-gate 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 31190Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 31200Sstevel@tonic-gate } 31210Sstevel@tonic-gate 31220Sstevel@tonic-gate /* 31230Sstevel@tonic-gate * Allocate one page at the oversize to break up the text region 31240Sstevel@tonic-gate * from the oversized region. 31250Sstevel@tonic-gate */ 31260Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 31270Sstevel@tonic-gate (void *)limit, (void *)(limit + PAGESIZE), 31280Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 31290Sstevel@tonic-gate 31303791Skchow *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 31313791Skchow modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 31320Sstevel@tonic-gate heaptext_arena, 0, VM_SLEEP); 31330Sstevel@tonic-gate *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 31340Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 31350Sstevel@tonic-gate } 31360Sstevel@tonic-gate 31370Sstevel@tonic-gate caddr_t 31380Sstevel@tonic-gate kobj_text_alloc(vmem_t *arena, size_t size) 31390Sstevel@tonic-gate { 31400Sstevel@tonic-gate caddr_t rval, better; 31410Sstevel@tonic-gate 31420Sstevel@tonic-gate /* 31430Sstevel@tonic-gate * First, try a sleeping allocation. 31440Sstevel@tonic-gate */ 31450Sstevel@tonic-gate rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 31460Sstevel@tonic-gate 31470Sstevel@tonic-gate if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 31480Sstevel@tonic-gate return (rval); 31490Sstevel@tonic-gate 31500Sstevel@tonic-gate /* 31510Sstevel@tonic-gate * We didn't get the area that we wanted. We're going to try to do an 31520Sstevel@tonic-gate * allocation with explicit constraints. 31530Sstevel@tonic-gate */ 31540Sstevel@tonic-gate better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 31550Sstevel@tonic-gate (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 31560Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT); 31570Sstevel@tonic-gate 31580Sstevel@tonic-gate if (better != NULL) { 31590Sstevel@tonic-gate /* 31600Sstevel@tonic-gate * That worked. Free our first attempt and return. 31610Sstevel@tonic-gate */ 31620Sstevel@tonic-gate vmem_free(arena, rval, size); 31630Sstevel@tonic-gate return (better); 31640Sstevel@tonic-gate } 31650Sstevel@tonic-gate 31660Sstevel@tonic-gate /* 31670Sstevel@tonic-gate * That didn't work; we'll have to return our first attempt. 31680Sstevel@tonic-gate */ 31690Sstevel@tonic-gate return (rval); 31700Sstevel@tonic-gate } 31710Sstevel@tonic-gate 31720Sstevel@tonic-gate caddr_t 31730Sstevel@tonic-gate kobj_texthole_alloc(caddr_t addr, size_t size) 31740Sstevel@tonic-gate { 31750Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 31760Sstevel@tonic-gate char c[30]; 31770Sstevel@tonic-gate uintptr_t base; 31780Sstevel@tonic-gate 31797428SJonathan.Haslam@Sun.COM if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 31800Sstevel@tonic-gate /* 31817428SJonathan.Haslam@Sun.COM * If this is an oversized allocation or it is allocated in 31827428SJonathan.Haslam@Sun.COM * the nucleus data page, there is no text hole available for 31837428SJonathan.Haslam@Sun.COM * it; return NULL. 31840Sstevel@tonic-gate */ 31850Sstevel@tonic-gate return (NULL); 31860Sstevel@tonic-gate } 31870Sstevel@tonic-gate 31880Sstevel@tonic-gate mutex_enter(&texthole_lock); 31890Sstevel@tonic-gate 31900Sstevel@tonic-gate if (texthole_arena[arena] == NULL) { 31910Sstevel@tonic-gate ASSERT(texthole_source[arena] == NULL); 31920Sstevel@tonic-gate 31930Sstevel@tonic-gate if (arena == 0) { 31940Sstevel@tonic-gate texthole_source[0] = vmem_create("module_text_holesrc", 31950Sstevel@tonic-gate (void *)(KERNELBASE + MMU_PAGESIZE4M), 31960Sstevel@tonic-gate MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 31970Sstevel@tonic-gate 0, VM_SLEEP); 31980Sstevel@tonic-gate } else { 31990Sstevel@tonic-gate base = HEAPTEXT_BASE + 32000Sstevel@tonic-gate (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 32010Sstevel@tonic-gate 32020Sstevel@tonic-gate (void) snprintf(c, sizeof (c), 32030Sstevel@tonic-gate "heaptext_holesrc_%d", arena); 32040Sstevel@tonic-gate 32050Sstevel@tonic-gate texthole_source[arena] = vmem_create(c, (void *)base, 32060Sstevel@tonic-gate HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 32070Sstevel@tonic-gate 0, VM_SLEEP); 32080Sstevel@tonic-gate } 32090Sstevel@tonic-gate 32100Sstevel@tonic-gate (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 32110Sstevel@tonic-gate 32120Sstevel@tonic-gate texthole_arena[arena] = vmem_create(c, NULL, 0, 32130Sstevel@tonic-gate sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 32140Sstevel@tonic-gate texthole_source[arena], 0, VM_SLEEP); 32150Sstevel@tonic-gate } 32160Sstevel@tonic-gate 32170Sstevel@tonic-gate mutex_exit(&texthole_lock); 32180Sstevel@tonic-gate 32190Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 32200Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 32210Sstevel@tonic-gate return (vmem_alloc(texthole_arena[arena], size, 32220Sstevel@tonic-gate VM_BESTFIT | VM_NOSLEEP)); 32230Sstevel@tonic-gate } 32240Sstevel@tonic-gate 32250Sstevel@tonic-gate void 32260Sstevel@tonic-gate kobj_texthole_free(caddr_t addr, size_t size) 32270Sstevel@tonic-gate { 32280Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 32290Sstevel@tonic-gate 32300Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 32310Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 32320Sstevel@tonic-gate vmem_free(texthole_arena[arena], addr, size); 32330Sstevel@tonic-gate } 32345974Sjm22469 32355974Sjm22469 void 32365974Sjm22469 release_bootstrap(void) 32375974Sjm22469 { 32385974Sjm22469 if (&cif_init) 32395974Sjm22469 cif_init(); 32405974Sjm22469 } 3241