1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/machsystm.h> 30*0Sstevel@tonic-gate #include <sys/archsystm.h> 31*0Sstevel@tonic-gate #include <sys/vm.h> 32*0Sstevel@tonic-gate #include <sys/cpu.h> 33*0Sstevel@tonic-gate #include <sys/atomic.h> 34*0Sstevel@tonic-gate #include <sys/reboot.h> 35*0Sstevel@tonic-gate #include <sys/kdi.h> 36*0Sstevel@tonic-gate #include <sys/bootconf.h> 37*0Sstevel@tonic-gate #include <sys/memlist_plat.h> 38*0Sstevel@tonic-gate #include <sys/memlist_impl.h> 39*0Sstevel@tonic-gate #include <sys/prom_plat.h> 40*0Sstevel@tonic-gate #include <sys/prom_isa.h> 41*0Sstevel@tonic-gate #include <sys/autoconf.h> 42*0Sstevel@tonic-gate #include <sys/intreg.h> 43*0Sstevel@tonic-gate #include <sys/ivintr.h> 44*0Sstevel@tonic-gate #include <sys/fpu/fpusystm.h> 45*0Sstevel@tonic-gate #include <sys/iommutsb.h> 46*0Sstevel@tonic-gate #include <vm/vm_dep.h> 47*0Sstevel@tonic-gate #include <vm/seg_dev.h> 48*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 49*0Sstevel@tonic-gate #include <vm/seg_kpm.h> 50*0Sstevel@tonic-gate #include <vm/seg_map.h> 51*0Sstevel@tonic-gate #include <vm/seg_kp.h> 52*0Sstevel@tonic-gate #include <sys/sysconf.h> 53*0Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 54*0Sstevel@tonic-gate #include <sys/kobj.h> 55*0Sstevel@tonic-gate #include <sys/sun4asi.h> 56*0Sstevel@tonic-gate #include <sys/clconf.h> 57*0Sstevel@tonic-gate #include <sys/platform_module.h> 58*0Sstevel@tonic-gate #include <sys/panic.h> 59*0Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h> 60*0Sstevel@tonic-gate #include <sys/clock.h> 61*0Sstevel@tonic-gate #include <sys/cmn_err.h> 62*0Sstevel@tonic-gate #include <sys/promif.h> 63*0Sstevel@tonic-gate #include <sys/prom_debug.h> 64*0Sstevel@tonic-gate 65*0Sstevel@tonic-gate #ifdef TRAPTRACE 66*0Sstevel@tonic-gate #include <sys/traptrace.h> 67*0Sstevel@tonic-gate #endif /* TRAPTRACE */ 68*0Sstevel@tonic-gate 69*0Sstevel@tonic-gate #include <sys/memnode.h> 70*0Sstevel@tonic-gate #include <sys/mem_cage.h> 71*0Sstevel@tonic-gate 72*0Sstevel@tonic-gate 73*0Sstevel@tonic-gate extern void setup_trap_table(void); 74*0Sstevel@tonic-gate extern void cpu_intrq_setup(struct cpu *); 75*0Sstevel@tonic-gate extern void cpu_intrq_register(struct cpu *); 76*0Sstevel@tonic-gate extern void contig_mem_init(void); 77*0Sstevel@tonic-gate extern void mach_dump_buffer_init(void); 78*0Sstevel@tonic-gate extern void mach_descrip_init(void); 79*0Sstevel@tonic-gate extern void mach_memscrub(void); 80*0Sstevel@tonic-gate extern void mach_fpras(void); 81*0Sstevel@tonic-gate extern void mach_cpu_halt_idle(void); 82*0Sstevel@tonic-gate extern void mach_hw_copy_limit(void); 83*0Sstevel@tonic-gate extern void load_tod_module(void); 84*0Sstevel@tonic-gate #pragma weak load_tod_module 85*0Sstevel@tonic-gate 86*0Sstevel@tonic-gate extern int ndata_alloc_mmfsa(struct memlist *ndata); 87*0Sstevel@tonic-gate #pragma weak ndata_alloc_mmfsa 88*0Sstevel@tonic-gate 89*0Sstevel@tonic-gate extern void parse_idprom(void); 90*0Sstevel@tonic-gate extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91*0Sstevel@tonic-gate extern void mem_config_init(void); 92*0Sstevel@tonic-gate extern void memseg_remap_init(void); 93*0Sstevel@tonic-gate 94*0Sstevel@tonic-gate /* 95*0Sstevel@tonic-gate * External Data: 96*0Sstevel@tonic-gate */ 97*0Sstevel@tonic-gate extern int vac_size; /* cache size in bytes */ 98*0Sstevel@tonic-gate extern uint_t vac_mask; /* VAC alignment consistency mask */ 99*0Sstevel@tonic-gate extern uint_t vac_colors; 100*0Sstevel@tonic-gate 101*0Sstevel@tonic-gate /* 102*0Sstevel@tonic-gate * Global Data Definitions: 103*0Sstevel@tonic-gate */ 104*0Sstevel@tonic-gate 105*0Sstevel@tonic-gate /* 106*0Sstevel@tonic-gate * XXX - Don't port this to new architectures 107*0Sstevel@tonic-gate * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 108*0Sstevel@tonic-gate * 'romp' has no use with a prom with an IEEE 1275 client interface. 109*0Sstevel@tonic-gate * The driver doesn't use the value, but it depends on the symbol. 110*0Sstevel@tonic-gate */ 111*0Sstevel@tonic-gate void *romp; /* veritas driver won't load without romp 4154976 */ 112*0Sstevel@tonic-gate /* 113*0Sstevel@tonic-gate * Declare these as initialized data so we can patch them. 114*0Sstevel@tonic-gate */ 115*0Sstevel@tonic-gate pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 116*0Sstevel@tonic-gate pgcnt_t segkpsize = 117*0Sstevel@tonic-gate btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 118*0Sstevel@tonic-gate uint_t segmap_percent = 12; /* Size of segmap segment */ 119*0Sstevel@tonic-gate 120*0Sstevel@tonic-gate int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 121*0Sstevel@tonic-gate int vac_copyback = 1; 122*0Sstevel@tonic-gate char *cache_mode = NULL; 123*0Sstevel@tonic-gate int use_mix = 1; 124*0Sstevel@tonic-gate int prom_debug = 0; 125*0Sstevel@tonic-gate int usb_node_debug = 0; 126*0Sstevel@tonic-gate 127*0Sstevel@tonic-gate struct bootops *bootops = 0; /* passed in from boot in %o2 */ 128*0Sstevel@tonic-gate caddr_t boot_tba; /* %tba at boot - used by kmdb */ 129*0Sstevel@tonic-gate uint_t tba_taken_over = 0; 130*0Sstevel@tonic-gate 131*0Sstevel@tonic-gate caddr_t s_text; /* start of kernel text segment */ 132*0Sstevel@tonic-gate caddr_t e_text; /* end of kernel text segment */ 133*0Sstevel@tonic-gate caddr_t s_data; /* start of kernel data segment */ 134*0Sstevel@tonic-gate caddr_t e_data; /* end of kernel data segment */ 135*0Sstevel@tonic-gate 136*0Sstevel@tonic-gate caddr_t modtext; /* beginning of module text */ 137*0Sstevel@tonic-gate size_t modtext_sz; /* size of module text */ 138*0Sstevel@tonic-gate caddr_t moddata; /* beginning of module data reserve */ 139*0Sstevel@tonic-gate caddr_t e_moddata; /* end of module data reserve */ 140*0Sstevel@tonic-gate 141*0Sstevel@tonic-gate /* 142*0Sstevel@tonic-gate * End of first block of contiguous kernel in 32-bit virtual address space 143*0Sstevel@tonic-gate */ 144*0Sstevel@tonic-gate caddr_t econtig32; /* end of first blk of contiguous kernel */ 145*0Sstevel@tonic-gate 146*0Sstevel@tonic-gate caddr_t ncbase; /* beginning of non-cached segment */ 147*0Sstevel@tonic-gate caddr_t ncend; /* end of non-cached segment */ 148*0Sstevel@tonic-gate caddr_t sdata; /* beginning of data segment */ 149*0Sstevel@tonic-gate 150*0Sstevel@tonic-gate caddr_t extra_etva; /* beginning of unused nucleus text */ 151*0Sstevel@tonic-gate pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 152*0Sstevel@tonic-gate 153*0Sstevel@tonic-gate size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 154*0Sstevel@tonic-gate caddr_t nalloc_base; /* beginning of nucleus allocation */ 155*0Sstevel@tonic-gate caddr_t nalloc_end; /* end of nucleus allocatable memory */ 156*0Sstevel@tonic-gate caddr_t valloc_base; /* beginning of kvalloc segment */ 157*0Sstevel@tonic-gate 158*0Sstevel@tonic-gate caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 159*0Sstevel@tonic-gate caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 160*0Sstevel@tonic-gate 161*0Sstevel@tonic-gate uintptr_t shm_alignment = 0; /* VAC address consistency modulus */ 162*0Sstevel@tonic-gate struct memlist *phys_install; /* Total installed physical memory */ 163*0Sstevel@tonic-gate struct memlist *phys_avail; /* Available (unreserved) physical memory */ 164*0Sstevel@tonic-gate struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 165*0Sstevel@tonic-gate struct memlist ndata; /* memlist of nucleus allocatable memory */ 166*0Sstevel@tonic-gate int memexp_flag; /* memory expansion card flag */ 167*0Sstevel@tonic-gate uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 168*0Sstevel@tonic-gate pgcnt_t obp_pages; /* Physical pages used by OBP */ 169*0Sstevel@tonic-gate 170*0Sstevel@tonic-gate /* 171*0Sstevel@tonic-gate * VM data structures 172*0Sstevel@tonic-gate */ 173*0Sstevel@tonic-gate long page_hashsz; /* Size of page hash table (power of two) */ 174*0Sstevel@tonic-gate struct page *pp_base; /* Base of system page struct array */ 175*0Sstevel@tonic-gate size_t pp_sz; /* Size in bytes of page struct array */ 176*0Sstevel@tonic-gate struct page **page_hash; /* Page hash table */ 177*0Sstevel@tonic-gate struct seg ktextseg; /* Segment used for kernel executable image */ 178*0Sstevel@tonic-gate struct seg kvalloc; /* Segment used for "valloc" mapping */ 179*0Sstevel@tonic-gate struct seg kpseg; /* Segment used for pageable kernel virt mem */ 180*0Sstevel@tonic-gate struct seg ktexthole; /* Segment used for nucleus text hole */ 181*0Sstevel@tonic-gate struct seg kmapseg; /* Segment used for generic kernel mappings */ 182*0Sstevel@tonic-gate struct seg kpmseg; /* Segment used for physical mapping */ 183*0Sstevel@tonic-gate struct seg kdebugseg; /* Segment used for the kernel debugger */ 184*0Sstevel@tonic-gate 185*0Sstevel@tonic-gate uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 186*0Sstevel@tonic-gate size_t kpm_pp_sz; /* Size of system kpm_page array */ 187*0Sstevel@tonic-gate pgcnt_t kpm_npages; /* How many kpm pages are managed */ 188*0Sstevel@tonic-gate 189*0Sstevel@tonic-gate struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 190*0Sstevel@tonic-gate struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 191*0Sstevel@tonic-gate struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 192*0Sstevel@tonic-gate 193*0Sstevel@tonic-gate /* 194*0Sstevel@tonic-gate * debugger pages (if allocated) 195*0Sstevel@tonic-gate */ 196*0Sstevel@tonic-gate struct vnode kdebugvp; 197*0Sstevel@tonic-gate 198*0Sstevel@tonic-gate /* 199*0Sstevel@tonic-gate * Segment for relocated kernel structures in 64-bit large RAM kernels 200*0Sstevel@tonic-gate */ 201*0Sstevel@tonic-gate struct seg kmem64; 202*0Sstevel@tonic-gate 203*0Sstevel@tonic-gate struct memseg *memseg_base; 204*0Sstevel@tonic-gate size_t memseg_sz; /* Used to translate a va to page */ 205*0Sstevel@tonic-gate struct vnode unused_pages_vp; 206*0Sstevel@tonic-gate 207*0Sstevel@tonic-gate /* 208*0Sstevel@tonic-gate * VM data structures allocated early during boot. 209*0Sstevel@tonic-gate */ 210*0Sstevel@tonic-gate size_t pagehash_sz; 211*0Sstevel@tonic-gate uint64_t memlist_sz; 212*0Sstevel@tonic-gate 213*0Sstevel@tonic-gate char tbr_wr_addr_inited = 0; 214*0Sstevel@tonic-gate 215*0Sstevel@tonic-gate 216*0Sstevel@tonic-gate /* 217*0Sstevel@tonic-gate * Static Routines: 218*0Sstevel@tonic-gate */ 219*0Sstevel@tonic-gate static void memlist_add(uint64_t, uint64_t, struct memlist **, 220*0Sstevel@tonic-gate struct memlist **); 221*0Sstevel@tonic-gate static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 222*0Sstevel@tonic-gate pgcnt_t); 223*0Sstevel@tonic-gate static void kvm_init(void); 224*0Sstevel@tonic-gate 225*0Sstevel@tonic-gate static void startup_init(void); 226*0Sstevel@tonic-gate static void startup_memlist(void); 227*0Sstevel@tonic-gate static void startup_modules(void); 228*0Sstevel@tonic-gate static void startup_bop_gone(void); 229*0Sstevel@tonic-gate static void startup_vm(void); 230*0Sstevel@tonic-gate static void startup_end(void); 231*0Sstevel@tonic-gate static void setup_cage_params(void); 232*0Sstevel@tonic-gate static void startup_create_input_node(void); 233*0Sstevel@tonic-gate 234*0Sstevel@tonic-gate static pgcnt_t npages; 235*0Sstevel@tonic-gate static struct memlist *memlist; 236*0Sstevel@tonic-gate void *memlist_end; 237*0Sstevel@tonic-gate 238*0Sstevel@tonic-gate static pgcnt_t bop_alloc_pages; 239*0Sstevel@tonic-gate static caddr_t hblk_base; 240*0Sstevel@tonic-gate uint_t hblk_alloc_dynamic = 0; 241*0Sstevel@tonic-gate uint_t hblk1_min = H1MIN; 242*0Sstevel@tonic-gate uint_t hblk8_min; 243*0Sstevel@tonic-gate 244*0Sstevel@tonic-gate 245*0Sstevel@tonic-gate /* 246*0Sstevel@tonic-gate * Hooks for unsupported platforms and down-rev firmware 247*0Sstevel@tonic-gate */ 248*0Sstevel@tonic-gate int iam_positron(void); 249*0Sstevel@tonic-gate #pragma weak iam_positron 250*0Sstevel@tonic-gate static void do_prom_version_check(void); 251*0Sstevel@tonic-gate static void kpm_init(void); 252*0Sstevel@tonic-gate static void kpm_npages_setup(int); 253*0Sstevel@tonic-gate static void kpm_memseg_init(void); 254*0Sstevel@tonic-gate 255*0Sstevel@tonic-gate /* 256*0Sstevel@tonic-gate * After receiving a thermal interrupt, this is the number of seconds 257*0Sstevel@tonic-gate * to delay before shutting off the system, assuming 258*0Sstevel@tonic-gate * shutdown fails. Use /etc/system to change the delay if this isn't 259*0Sstevel@tonic-gate * large enough. 260*0Sstevel@tonic-gate */ 261*0Sstevel@tonic-gate int thermal_powerdown_delay = 1200; 262*0Sstevel@tonic-gate 263*0Sstevel@tonic-gate /* 264*0Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed 265*0Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel. 266*0Sstevel@tonic-gate */ 267*0Sstevel@tonic-gate int page_relocate_ready = 0; 268*0Sstevel@tonic-gate 269*0Sstevel@tonic-gate /* 270*0Sstevel@tonic-gate * Enable some debugging messages concerning memory usage... 271*0Sstevel@tonic-gate */ 272*0Sstevel@tonic-gate #ifdef DEBUGGING_MEM 273*0Sstevel@tonic-gate static int debugging_mem; 274*0Sstevel@tonic-gate static void 275*0Sstevel@tonic-gate printmemlist(char *title, struct memlist *list) 276*0Sstevel@tonic-gate { 277*0Sstevel@tonic-gate if (!debugging_mem) 278*0Sstevel@tonic-gate return; 279*0Sstevel@tonic-gate 280*0Sstevel@tonic-gate printf("%s\n", title); 281*0Sstevel@tonic-gate 282*0Sstevel@tonic-gate while (list) { 283*0Sstevel@tonic-gate prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 284*0Sstevel@tonic-gate (uint32_t)(list->address >> 32), (uint32_t)list->address, 285*0Sstevel@tonic-gate (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 286*0Sstevel@tonic-gate list = list->next; 287*0Sstevel@tonic-gate } 288*0Sstevel@tonic-gate } 289*0Sstevel@tonic-gate 290*0Sstevel@tonic-gate void 291*0Sstevel@tonic-gate printmemseg(struct memseg *memseg) 292*0Sstevel@tonic-gate { 293*0Sstevel@tonic-gate if (!debugging_mem) 294*0Sstevel@tonic-gate return; 295*0Sstevel@tonic-gate 296*0Sstevel@tonic-gate printf("memseg\n"); 297*0Sstevel@tonic-gate 298*0Sstevel@tonic-gate while (memseg) { 299*0Sstevel@tonic-gate prom_printf("\tpage = 0x%p, epage = 0x%p, " 300*0Sstevel@tonic-gate "pfn = 0x%x, epfn = 0x%x\n", 301*0Sstevel@tonic-gate memseg->pages, memseg->epages, 302*0Sstevel@tonic-gate memseg->pages_base, memseg->pages_end); 303*0Sstevel@tonic-gate memseg = memseg->next; 304*0Sstevel@tonic-gate } 305*0Sstevel@tonic-gate } 306*0Sstevel@tonic-gate 307*0Sstevel@tonic-gate #define debug_pause(str) halt((str)) 308*0Sstevel@tonic-gate #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 309*0Sstevel@tonic-gate #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 310*0Sstevel@tonic-gate #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 311*0Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) \ 312*0Sstevel@tonic-gate if (debugging_mem) prom_printf((str), (a), (b), (c)) 313*0Sstevel@tonic-gate #else /* DEBUGGING_MEM */ 314*0Sstevel@tonic-gate #define MPRINTF(str) 315*0Sstevel@tonic-gate #define MPRINTF1(str, a) 316*0Sstevel@tonic-gate #define MPRINTF2(str, a, b) 317*0Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) 318*0Sstevel@tonic-gate #endif /* DEBUGGING_MEM */ 319*0Sstevel@tonic-gate 320*0Sstevel@tonic-gate /* Simple message to indicate that the bootops pointer has been zeroed */ 321*0Sstevel@tonic-gate #ifdef DEBUG 322*0Sstevel@tonic-gate static int bootops_gone_on = 0; 323*0Sstevel@tonic-gate #define BOOTOPS_GONE() \ 324*0Sstevel@tonic-gate if (bootops_gone_on) \ 325*0Sstevel@tonic-gate prom_printf("The bootops vec is zeroed now!\n"); 326*0Sstevel@tonic-gate #else 327*0Sstevel@tonic-gate #define BOOTOPS_GONE() 328*0Sstevel@tonic-gate #endif /* DEBUG */ 329*0Sstevel@tonic-gate 330*0Sstevel@tonic-gate /* 331*0Sstevel@tonic-gate * Monitor pages may not be where this says they are. 332*0Sstevel@tonic-gate * and the debugger may not be there either. 333*0Sstevel@tonic-gate * 334*0Sstevel@tonic-gate * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 335*0Sstevel@tonic-gate * 336*0Sstevel@tonic-gate * Physical memory layout 337*0Sstevel@tonic-gate * (not necessarily contiguous) 338*0Sstevel@tonic-gate * (THIS IS SOMEWHAT WRONG) 339*0Sstevel@tonic-gate * /-----------------------\ 340*0Sstevel@tonic-gate * | monitor pages | 341*0Sstevel@tonic-gate * availmem -|-----------------------| 342*0Sstevel@tonic-gate * | | 343*0Sstevel@tonic-gate * | page pool | 344*0Sstevel@tonic-gate * | | 345*0Sstevel@tonic-gate * |-----------------------| 346*0Sstevel@tonic-gate * | configured tables | 347*0Sstevel@tonic-gate * | buffers | 348*0Sstevel@tonic-gate * firstaddr -|-----------------------| 349*0Sstevel@tonic-gate * | hat data structures | 350*0Sstevel@tonic-gate * |-----------------------| 351*0Sstevel@tonic-gate * | kernel data, bss | 352*0Sstevel@tonic-gate * |-----------------------| 353*0Sstevel@tonic-gate * | interrupt stack | 354*0Sstevel@tonic-gate * |-----------------------| 355*0Sstevel@tonic-gate * | kernel text (RO) | 356*0Sstevel@tonic-gate * |-----------------------| 357*0Sstevel@tonic-gate * | trap table (4k) | 358*0Sstevel@tonic-gate * |-----------------------| 359*0Sstevel@tonic-gate * page 1 | panicbuf | 360*0Sstevel@tonic-gate * |-----------------------| 361*0Sstevel@tonic-gate * page 0 | reclaimed | 362*0Sstevel@tonic-gate * |_______________________| 363*0Sstevel@tonic-gate * 364*0Sstevel@tonic-gate * 365*0Sstevel@tonic-gate * 366*0Sstevel@tonic-gate * Kernel's Virtual Memory Layout. 367*0Sstevel@tonic-gate * /-----------------------\ 368*0Sstevel@tonic-gate * 0xFFFFFFFF.FFFFFFFF -| |- 369*0Sstevel@tonic-gate * | OBP's virtual page | 370*0Sstevel@tonic-gate * | tables | 371*0Sstevel@tonic-gate * 0xFFFFFFFC.00000000 -|-----------------------|- 372*0Sstevel@tonic-gate * : : 373*0Sstevel@tonic-gate * : : 374*0Sstevel@tonic-gate * 0xFFFFFE00.00000000 -|-----------------------|- 375*0Sstevel@tonic-gate * | | Ultrasparc I/II support 376*0Sstevel@tonic-gate * | segkpm segment | up to 2TB of physical 377*0Sstevel@tonic-gate * | (64-bit kernel ONLY) | memory, VAC has 2 colors 378*0Sstevel@tonic-gate * | | 379*0Sstevel@tonic-gate * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 380*0Sstevel@tonic-gate * : : 381*0Sstevel@tonic-gate * : : 382*0Sstevel@tonic-gate * 0xFFFFF810.00000000 -|-----------------------|- hole_end 383*0Sstevel@tonic-gate * | | ^ 384*0Sstevel@tonic-gate * | UltraSPARC I/II call | | 385*0Sstevel@tonic-gate * | bug requires an extra | | 386*0Sstevel@tonic-gate * | 4 GB of space between | | 387*0Sstevel@tonic-gate * | hole and used RAM | | 388*0Sstevel@tonic-gate * | | | 389*0Sstevel@tonic-gate * 0xFFFFF800.00000000 -|-----------------------|- | 390*0Sstevel@tonic-gate * | | | 391*0Sstevel@tonic-gate * | Virtual Address Hole | UltraSPARC 392*0Sstevel@tonic-gate * | on UltraSPARC I/II | I/II * ONLY * 393*0Sstevel@tonic-gate * | | | 394*0Sstevel@tonic-gate * 0x00000800.00000000 -|-----------------------|- | 395*0Sstevel@tonic-gate * | | | 396*0Sstevel@tonic-gate * | UltraSPARC I/II call | | 397*0Sstevel@tonic-gate * | bug requires an extra | | 398*0Sstevel@tonic-gate * | 4 GB of space between | | 399*0Sstevel@tonic-gate * | hole and used RAM | | 400*0Sstevel@tonic-gate * | | v 401*0Sstevel@tonic-gate * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 402*0Sstevel@tonic-gate * : : ^ 403*0Sstevel@tonic-gate * : : | 404*0Sstevel@tonic-gate * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 405*0Sstevel@tonic-gate * | | | 406*0Sstevel@tonic-gate * | 64-bit kernel ONLY | | 407*0Sstevel@tonic-gate * | | | 408*0Sstevel@tonic-gate * | kmem64 segment | | 409*0Sstevel@tonic-gate * | | | 410*0Sstevel@tonic-gate * | (Relocated extra HME | Approximately 411*0Sstevel@tonic-gate * | block allocations, | 1 TB of virtual 412*0Sstevel@tonic-gate * | memnode freelists, | address space 413*0Sstevel@tonic-gate * | HME hash buckets, | | 414*0Sstevel@tonic-gate * | mml_table, kpmp_table,| | 415*0Sstevel@tonic-gate * | page_t array and | | 416*0Sstevel@tonic-gate * | hashblock pool to | | 417*0Sstevel@tonic-gate * | avoid hard-coded | | 418*0Sstevel@tonic-gate * | 32-bit vaddr | | 419*0Sstevel@tonic-gate * | limitations) | | 420*0Sstevel@tonic-gate * | | v 421*0Sstevel@tonic-gate * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 422*0Sstevel@tonic-gate * | | 423*0Sstevel@tonic-gate * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 424*0Sstevel@tonic-gate * | | 425*0Sstevel@tonic-gate * 0x00000300.00000000 -|-----------------------|- SYSBASE 426*0Sstevel@tonic-gate * : : 427*0Sstevel@tonic-gate * : : 428*0Sstevel@tonic-gate * -|-----------------------|- 429*0Sstevel@tonic-gate * | | 430*0Sstevel@tonic-gate * | segmap segment | SEGMAPSIZE (1/8th physmem, 431*0Sstevel@tonic-gate * | | 256G MAX) 432*0Sstevel@tonic-gate * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 433*0Sstevel@tonic-gate * : : 434*0Sstevel@tonic-gate * : : 435*0Sstevel@tonic-gate * -|-----------------------|- 436*0Sstevel@tonic-gate * | | 437*0Sstevel@tonic-gate * | segkp | SEGKPSIZE (2GB) 438*0Sstevel@tonic-gate * | | 439*0Sstevel@tonic-gate * | | 440*0Sstevel@tonic-gate * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 441*0Sstevel@tonic-gate * | | 442*0Sstevel@tonic-gate * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 443*0Sstevel@tonic-gate * | | (SEGKPBASE - 0x400000) 444*0Sstevel@tonic-gate * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 445*0Sstevel@tonic-gate * | | (MEMSCRUBBASE - NCARGS) 446*0Sstevel@tonic-gate * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 447*0Sstevel@tonic-gate * | | (ARGSBASE - PPMAPSIZE) 448*0Sstevel@tonic-gate * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 449*0Sstevel@tonic-gate * | | 450*0Sstevel@tonic-gate * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 451*0Sstevel@tonic-gate * | | 452*0Sstevel@tonic-gate * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 453*0Sstevel@tonic-gate * : : 454*0Sstevel@tonic-gate * : : 455*0Sstevel@tonic-gate * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 456*0Sstevel@tonic-gate * | | 457*0Sstevel@tonic-gate * | OBP | 458*0Sstevel@tonic-gate * | | 459*0Sstevel@tonic-gate * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 460*0Sstevel@tonic-gate * | kmdb | 461*0Sstevel@tonic-gate * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 462*0Sstevel@tonic-gate * : : 463*0Sstevel@tonic-gate * : : 464*0Sstevel@tonic-gate * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 465*0Sstevel@tonic-gate * | | 466*0Sstevel@tonic-gate * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 467*0Sstevel@tonic-gate * | | ~64MB) 468*0Sstevel@tonic-gate * 0x00000000.78002000 -|-----------------------| 469*0Sstevel@tonic-gate * | panicbuf | 470*0Sstevel@tonic-gate * 0x00000000.78000000 -|-----------------------|- SYSBASE32 471*0Sstevel@tonic-gate * : : 472*0Sstevel@tonic-gate * : : 473*0Sstevel@tonic-gate * | | 474*0Sstevel@tonic-gate * |-----------------------|- econtig32 475*0Sstevel@tonic-gate * | vm structures | 476*0Sstevel@tonic-gate * 0x00000000.01C00000 |-----------------------|- nalloc_end 477*0Sstevel@tonic-gate * | TSBs | 478*0Sstevel@tonic-gate * |-----------------------|- end/nalloc_base 479*0Sstevel@tonic-gate * | kernel data & bss | 480*0Sstevel@tonic-gate * 0x00000000.01800000 -|-----------------------| 481*0Sstevel@tonic-gate * : nucleus text hole : 482*0Sstevel@tonic-gate * 0x00000000.01400000 -|-----------------------| 483*0Sstevel@tonic-gate * : : 484*0Sstevel@tonic-gate * |-----------------------| 485*0Sstevel@tonic-gate * | module text | 486*0Sstevel@tonic-gate * |-----------------------|- e_text/modtext 487*0Sstevel@tonic-gate * | kernel text | 488*0Sstevel@tonic-gate * |-----------------------| 489*0Sstevel@tonic-gate * | trap table (48k) | 490*0Sstevel@tonic-gate * 0x00000000.01000000 -|-----------------------|- KERNELBASE 491*0Sstevel@tonic-gate * | reserved for trapstat |} TSTAT_TOTAL_SIZE 492*0Sstevel@tonic-gate * |-----------------------| 493*0Sstevel@tonic-gate * | | 494*0Sstevel@tonic-gate * | invalid | 495*0Sstevel@tonic-gate * | | 496*0Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 497*0Sstevel@tonic-gate * 498*0Sstevel@tonic-gate * 499*0Sstevel@tonic-gate * 500*0Sstevel@tonic-gate * 32-bit User Virtual Memory Layout. 501*0Sstevel@tonic-gate * /-----------------------\ 502*0Sstevel@tonic-gate * | | 503*0Sstevel@tonic-gate * | invalid | 504*0Sstevel@tonic-gate * | | 505*0Sstevel@tonic-gate * 0xFFC00000 -|-----------------------|- USERLIMIT 506*0Sstevel@tonic-gate * | user stack | 507*0Sstevel@tonic-gate * : : 508*0Sstevel@tonic-gate * : : 509*0Sstevel@tonic-gate * : : 510*0Sstevel@tonic-gate * | user data | 511*0Sstevel@tonic-gate * -|-----------------------|- 512*0Sstevel@tonic-gate * | user text | 513*0Sstevel@tonic-gate * 0x00002000 -|-----------------------|- 514*0Sstevel@tonic-gate * | invalid | 515*0Sstevel@tonic-gate * 0x00000000 _|_______________________| 516*0Sstevel@tonic-gate * 517*0Sstevel@tonic-gate * 518*0Sstevel@tonic-gate * 519*0Sstevel@tonic-gate * 64-bit User Virtual Memory Layout. 520*0Sstevel@tonic-gate * /-----------------------\ 521*0Sstevel@tonic-gate * | | 522*0Sstevel@tonic-gate * | invalid | 523*0Sstevel@tonic-gate * | | 524*0Sstevel@tonic-gate * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 525*0Sstevel@tonic-gate * | user stack | 526*0Sstevel@tonic-gate * : : 527*0Sstevel@tonic-gate * : : 528*0Sstevel@tonic-gate * : : 529*0Sstevel@tonic-gate * | user data | 530*0Sstevel@tonic-gate * -|-----------------------|- 531*0Sstevel@tonic-gate * | user text | 532*0Sstevel@tonic-gate * 0x00000000.00100000 -|-----------------------|- 533*0Sstevel@tonic-gate * | invalid | 534*0Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________| 535*0Sstevel@tonic-gate */ 536*0Sstevel@tonic-gate 537*0Sstevel@tonic-gate extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 538*0Sstevel@tonic-gate extern uint64_t ecache_flush_address(void); 539*0Sstevel@tonic-gate 540*0Sstevel@tonic-gate #pragma weak load_platform_modules 541*0Sstevel@tonic-gate #pragma weak starcat_startup_memlist 542*0Sstevel@tonic-gate #pragma weak ecache_init_scrub_flush_area 543*0Sstevel@tonic-gate #pragma weak ecache_flush_address 544*0Sstevel@tonic-gate 545*0Sstevel@tonic-gate 546*0Sstevel@tonic-gate /* 547*0Sstevel@tonic-gate * By default the DR Cage is enabled for maximum OS 548*0Sstevel@tonic-gate * MPSS performance. Users needing to disable the cage mechanism 549*0Sstevel@tonic-gate * can set this variable to zero via /etc/system. 550*0Sstevel@tonic-gate * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 551*0Sstevel@tonic-gate * will result in loss of DR functionality. 552*0Sstevel@tonic-gate * Platforms wishing to disable kernel Cage by default 553*0Sstevel@tonic-gate * should do so in their set_platform_defaults() routine. 554*0Sstevel@tonic-gate */ 555*0Sstevel@tonic-gate int kernel_cage_enable = 1; 556*0Sstevel@tonic-gate 557*0Sstevel@tonic-gate static void 558*0Sstevel@tonic-gate setup_cage_params(void) 559*0Sstevel@tonic-gate { 560*0Sstevel@tonic-gate void (*func)(void); 561*0Sstevel@tonic-gate 562*0Sstevel@tonic-gate func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 563*0Sstevel@tonic-gate if (func != NULL) { 564*0Sstevel@tonic-gate (*func)(); 565*0Sstevel@tonic-gate return; 566*0Sstevel@tonic-gate } 567*0Sstevel@tonic-gate 568*0Sstevel@tonic-gate if (kernel_cage_enable == 0) { 569*0Sstevel@tonic-gate return; 570*0Sstevel@tonic-gate } 571*0Sstevel@tonic-gate kcage_range_lock(); 572*0Sstevel@tonic-gate if (kcage_range_init(phys_avail, 1) == 0) { 573*0Sstevel@tonic-gate kcage_init(total_pages / 256); 574*0Sstevel@tonic-gate } 575*0Sstevel@tonic-gate kcage_range_unlock(); 576*0Sstevel@tonic-gate 577*0Sstevel@tonic-gate if (kcage_on) { 578*0Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 579*0Sstevel@tonic-gate } else { 580*0Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 581*0Sstevel@tonic-gate } 582*0Sstevel@tonic-gate 583*0Sstevel@tonic-gate } 584*0Sstevel@tonic-gate 585*0Sstevel@tonic-gate /* 586*0Sstevel@tonic-gate * Machine-dependent startup code 587*0Sstevel@tonic-gate */ 588*0Sstevel@tonic-gate void 589*0Sstevel@tonic-gate startup(void) 590*0Sstevel@tonic-gate { 591*0Sstevel@tonic-gate startup_init(); 592*0Sstevel@tonic-gate if (&startup_platform) 593*0Sstevel@tonic-gate startup_platform(); 594*0Sstevel@tonic-gate startup_memlist(); 595*0Sstevel@tonic-gate startup_modules(); 596*0Sstevel@tonic-gate setup_cage_params(); 597*0Sstevel@tonic-gate startup_bop_gone(); 598*0Sstevel@tonic-gate startup_vm(); 599*0Sstevel@tonic-gate startup_end(); 600*0Sstevel@tonic-gate } 601*0Sstevel@tonic-gate 602*0Sstevel@tonic-gate struct regs sync_reg_buf; 603*0Sstevel@tonic-gate uint64_t sync_tt; 604*0Sstevel@tonic-gate 605*0Sstevel@tonic-gate void 606*0Sstevel@tonic-gate sync_handler(void) 607*0Sstevel@tonic-gate { 608*0Sstevel@tonic-gate struct trap_info ti; 609*0Sstevel@tonic-gate int i; 610*0Sstevel@tonic-gate 611*0Sstevel@tonic-gate /* 612*0Sstevel@tonic-gate * Prevent trying to talk to the other CPUs since they are 613*0Sstevel@tonic-gate * sitting in the prom and won't reply. 614*0Sstevel@tonic-gate */ 615*0Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 616*0Sstevel@tonic-gate if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 617*0Sstevel@tonic-gate cpu[i]->cpu_flags &= ~CPU_READY; 618*0Sstevel@tonic-gate cpu[i]->cpu_flags |= CPU_QUIESCED; 619*0Sstevel@tonic-gate CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 620*0Sstevel@tonic-gate } 621*0Sstevel@tonic-gate } 622*0Sstevel@tonic-gate 623*0Sstevel@tonic-gate /* 624*0Sstevel@tonic-gate * We've managed to get here without going through the 625*0Sstevel@tonic-gate * normal panic code path. Try and save some useful 626*0Sstevel@tonic-gate * information. 627*0Sstevel@tonic-gate */ 628*0Sstevel@tonic-gate if (!panicstr && (curthread->t_panic_trap == NULL)) { 629*0Sstevel@tonic-gate ti.trap_type = sync_tt; 630*0Sstevel@tonic-gate ti.trap_regs = &sync_reg_buf; 631*0Sstevel@tonic-gate ti.trap_addr = NULL; 632*0Sstevel@tonic-gate ti.trap_mmu_fsr = 0x0; 633*0Sstevel@tonic-gate 634*0Sstevel@tonic-gate curthread->t_panic_trap = &ti; 635*0Sstevel@tonic-gate } 636*0Sstevel@tonic-gate 637*0Sstevel@tonic-gate /* 638*0Sstevel@tonic-gate * If we're re-entering the panic path, update the signature 639*0Sstevel@tonic-gate * block so that the SC knows we're in the second part of panic. 640*0Sstevel@tonic-gate */ 641*0Sstevel@tonic-gate if (panicstr) 642*0Sstevel@tonic-gate CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 643*0Sstevel@tonic-gate 644*0Sstevel@tonic-gate nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 645*0Sstevel@tonic-gate panic("sync initiated"); 646*0Sstevel@tonic-gate } 647*0Sstevel@tonic-gate 648*0Sstevel@tonic-gate 649*0Sstevel@tonic-gate static void 650*0Sstevel@tonic-gate startup_init(void) 651*0Sstevel@tonic-gate { 652*0Sstevel@tonic-gate /* 653*0Sstevel@tonic-gate * We want to save the registers while we're still in OBP 654*0Sstevel@tonic-gate * so that we know they haven't been fiddled with since. 655*0Sstevel@tonic-gate * (In principle, OBP can't change them just because it 656*0Sstevel@tonic-gate * makes a callback, but we'd rather not depend on that 657*0Sstevel@tonic-gate * behavior.) 658*0Sstevel@tonic-gate */ 659*0Sstevel@tonic-gate char sync_str[] = 660*0Sstevel@tonic-gate "warning @ warning off : sync " 661*0Sstevel@tonic-gate "%%tl-c %%tstate h# %p x! " 662*0Sstevel@tonic-gate "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 663*0Sstevel@tonic-gate "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 664*0Sstevel@tonic-gate "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 665*0Sstevel@tonic-gate "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 666*0Sstevel@tonic-gate "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 667*0Sstevel@tonic-gate "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 668*0Sstevel@tonic-gate "%%y h# %p l! %%tl-c %%tt h# %p x! " 669*0Sstevel@tonic-gate "sync ; warning !"; 670*0Sstevel@tonic-gate 671*0Sstevel@tonic-gate /* 672*0Sstevel@tonic-gate * 20 == num of %p substrings 673*0Sstevel@tonic-gate * 16 == max num of chars %p will expand to. 674*0Sstevel@tonic-gate */ 675*0Sstevel@tonic-gate char bp[sizeof (sync_str) + 16 * 20]; 676*0Sstevel@tonic-gate 677*0Sstevel@tonic-gate (void) check_boot_version(BOP_GETVERSION(bootops)); 678*0Sstevel@tonic-gate 679*0Sstevel@tonic-gate /* 680*0Sstevel@tonic-gate * Initialize ptl1 stack for the 1st CPU. 681*0Sstevel@tonic-gate */ 682*0Sstevel@tonic-gate ptl1_init_cpu(&cpu0); 683*0Sstevel@tonic-gate 684*0Sstevel@tonic-gate /* 685*0Sstevel@tonic-gate * Initialize the address map for cache consistent mappings 686*0Sstevel@tonic-gate * to random pages; must be done after vac_size is set. 687*0Sstevel@tonic-gate */ 688*0Sstevel@tonic-gate ppmapinit(); 689*0Sstevel@tonic-gate 690*0Sstevel@tonic-gate /* 691*0Sstevel@tonic-gate * Initialize the PROM callback handler. 692*0Sstevel@tonic-gate */ 693*0Sstevel@tonic-gate init_vx_handler(); 694*0Sstevel@tonic-gate 695*0Sstevel@tonic-gate /* 696*0Sstevel@tonic-gate * have prom call sync_callback() to handle the sync and 697*0Sstevel@tonic-gate * save some useful information which will be stored in the 698*0Sstevel@tonic-gate * core file later. 699*0Sstevel@tonic-gate */ 700*0Sstevel@tonic-gate (void) sprintf((char *)bp, sync_str, 701*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 702*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 703*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 704*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 705*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 706*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 707*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 708*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 709*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 710*0Sstevel@tonic-gate (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 711*0Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 712*0Sstevel@tonic-gate add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 713*0Sstevel@tonic-gate } 714*0Sstevel@tonic-gate 715*0Sstevel@tonic-gate static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 716*0Sstevel@tonic-gate static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 717*0Sstevel@tonic-gate 718*0Sstevel@tonic-gate #define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector)) 719*0Sstevel@tonic-gate 720*0Sstevel@tonic-gate /* 721*0Sstevel@tonic-gate * As OBP takes up some RAM when the system boots, pages will already be "lost" 722*0Sstevel@tonic-gate * to the system and reflected in npages by the time we see it. 723*0Sstevel@tonic-gate * 724*0Sstevel@tonic-gate * We only want to allocate kernel structures in the 64-bit virtual address 725*0Sstevel@tonic-gate * space on systems with enough RAM to make the overhead of keeping track of 726*0Sstevel@tonic-gate * an extra kernel memory segment worthwhile. 727*0Sstevel@tonic-gate * 728*0Sstevel@tonic-gate * Since OBP has already performed its memory allocations by this point, if we 729*0Sstevel@tonic-gate * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 730*0Sstevel@tonic-gate * memory in the 64-bit virtual address space; otherwise keep allocations 731*0Sstevel@tonic-gate * contiguous with we've mapped so far in the 32-bit virtual address space. 732*0Sstevel@tonic-gate */ 733*0Sstevel@tonic-gate #define MINMOVE_RAM_MB ((size_t)1900) 734*0Sstevel@tonic-gate #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 735*0Sstevel@tonic-gate 736*0Sstevel@tonic-gate pgcnt_t tune_npages = (pgcnt_t) 737*0Sstevel@tonic-gate (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 738*0Sstevel@tonic-gate 739*0Sstevel@tonic-gate static void 740*0Sstevel@tonic-gate startup_memlist(void) 741*0Sstevel@tonic-gate { 742*0Sstevel@tonic-gate size_t alloc_sz; 743*0Sstevel@tonic-gate size_t ctrs_sz; 744*0Sstevel@tonic-gate caddr_t alloc_base; 745*0Sstevel@tonic-gate caddr_t ctrs_base, ctrs_end; 746*0Sstevel@tonic-gate caddr_t memspace; 747*0Sstevel@tonic-gate caddr_t va; 748*0Sstevel@tonic-gate int memblocks = 0; 749*0Sstevel@tonic-gate struct memlist *cur; 750*0Sstevel@tonic-gate size_t syslimit = (size_t)SYSLIMIT; 751*0Sstevel@tonic-gate size_t sysbase = (size_t)SYSBASE; 752*0Sstevel@tonic-gate int alloc_alignsize = MMU_PAGESIZE; 753*0Sstevel@tonic-gate extern void page_coloring_init(void); 754*0Sstevel@tonic-gate 755*0Sstevel@tonic-gate /* 756*0Sstevel@tonic-gate * Initialize enough of the system to allow kmem_alloc to work by 757*0Sstevel@tonic-gate * calling boot to allocate its memory until the time that 758*0Sstevel@tonic-gate * kvm_init is completed. The page structs are allocated after 759*0Sstevel@tonic-gate * rounding up end to the nearest page boundary; the memsegs are 760*0Sstevel@tonic-gate * initialized and the space they use comes from the kernel heap. 761*0Sstevel@tonic-gate * With appropriate initialization, they can be reallocated later 762*0Sstevel@tonic-gate * to a size appropriate for the machine's configuration. 763*0Sstevel@tonic-gate * 764*0Sstevel@tonic-gate * At this point, memory is allocated for things that will never 765*0Sstevel@tonic-gate * need to be freed, this used to be "valloced". This allows a 766*0Sstevel@tonic-gate * savings as the pages don't need page structures to describe 767*0Sstevel@tonic-gate * them because them will not be managed by the vm system. 768*0Sstevel@tonic-gate */ 769*0Sstevel@tonic-gate 770*0Sstevel@tonic-gate /* 771*0Sstevel@tonic-gate * We're loaded by boot with the following configuration (as 772*0Sstevel@tonic-gate * specified in the sun4u/conf/Mapfile): 773*0Sstevel@tonic-gate * 774*0Sstevel@tonic-gate * text: 4 MB chunk aligned on a 4MB boundary 775*0Sstevel@tonic-gate * data & bss: 4 MB chunk aligned on a 4MB boundary 776*0Sstevel@tonic-gate * 777*0Sstevel@tonic-gate * These two chunks will eventually be mapped by 2 locked 4MB 778*0Sstevel@tonic-gate * ttes and will represent the nucleus of the kernel. This gives 779*0Sstevel@tonic-gate * us some free space that is already allocated, some or all of 780*0Sstevel@tonic-gate * which is made available to kernel module text. 781*0Sstevel@tonic-gate * 782*0Sstevel@tonic-gate * The free space in the data-bss chunk is used for nucleus 783*0Sstevel@tonic-gate * allocatable data structures and we reserve it using the 784*0Sstevel@tonic-gate * nalloc_base and nalloc_end variables. This space is currently 785*0Sstevel@tonic-gate * being used for hat data structures required for tlb miss 786*0Sstevel@tonic-gate * handling operations. We align nalloc_base to a l2 cache 787*0Sstevel@tonic-gate * linesize because this is the line size the hardware uses to 788*0Sstevel@tonic-gate * maintain cache coherency. 789*0Sstevel@tonic-gate * 256K is carved out for module data. 790*0Sstevel@tonic-gate */ 791*0Sstevel@tonic-gate 792*0Sstevel@tonic-gate nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 793*0Sstevel@tonic-gate moddata = nalloc_base; 794*0Sstevel@tonic-gate e_moddata = nalloc_base + MODDATA; 795*0Sstevel@tonic-gate nalloc_base = e_moddata; 796*0Sstevel@tonic-gate 797*0Sstevel@tonic-gate nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 798*0Sstevel@tonic-gate valloc_base = nalloc_base; 799*0Sstevel@tonic-gate 800*0Sstevel@tonic-gate /* 801*0Sstevel@tonic-gate * Calculate the start of the data segment. 802*0Sstevel@tonic-gate */ 803*0Sstevel@tonic-gate sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 804*0Sstevel@tonic-gate 805*0Sstevel@tonic-gate PRM_DEBUG(moddata); 806*0Sstevel@tonic-gate PRM_DEBUG(nalloc_base); 807*0Sstevel@tonic-gate PRM_DEBUG(nalloc_end); 808*0Sstevel@tonic-gate PRM_DEBUG(sdata); 809*0Sstevel@tonic-gate 810*0Sstevel@tonic-gate /* 811*0Sstevel@tonic-gate * Remember any slop after e_text so we can give it to the modules. 812*0Sstevel@tonic-gate */ 813*0Sstevel@tonic-gate PRM_DEBUG(e_text); 814*0Sstevel@tonic-gate modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 815*0Sstevel@tonic-gate if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 816*0Sstevel@tonic-gate panic("nucleus text overflow"); 817*0Sstevel@tonic-gate modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 818*0Sstevel@tonic-gate modtext; 819*0Sstevel@tonic-gate PRM_DEBUG(modtext); 820*0Sstevel@tonic-gate PRM_DEBUG(modtext_sz); 821*0Sstevel@tonic-gate 822*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 823*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 824*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 825*0Sstevel@tonic-gate /* 826*0Sstevel@tonic-gate * Remember what the physically available highest page is 827*0Sstevel@tonic-gate * so that dumpsys works properly, and find out how much 828*0Sstevel@tonic-gate * memory is installed. 829*0Sstevel@tonic-gate */ 830*0Sstevel@tonic-gate installed_top_size_memlist_array(boot_physinstalled, 831*0Sstevel@tonic-gate boot_physinstalled_len, &physmax, &physinstalled); 832*0Sstevel@tonic-gate PRM_DEBUG(physinstalled); 833*0Sstevel@tonic-gate PRM_DEBUG(physmax); 834*0Sstevel@tonic-gate 835*0Sstevel@tonic-gate /* Fill out memory nodes config structure */ 836*0Sstevel@tonic-gate startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 837*0Sstevel@tonic-gate 838*0Sstevel@tonic-gate /* 839*0Sstevel@tonic-gate * Get the list of physically available memory to size 840*0Sstevel@tonic-gate * the number of page structures needed. 841*0Sstevel@tonic-gate */ 842*0Sstevel@tonic-gate size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 843*0Sstevel@tonic-gate /* 844*0Sstevel@tonic-gate * This first snap shot of npages can represent the pages used 845*0Sstevel@tonic-gate * by OBP's text and data approximately. This is used in the 846*0Sstevel@tonic-gate * the calculation of the kernel size 847*0Sstevel@tonic-gate */ 848*0Sstevel@tonic-gate obp_pages = physinstalled - npages; 849*0Sstevel@tonic-gate 850*0Sstevel@tonic-gate 851*0Sstevel@tonic-gate /* 852*0Sstevel@tonic-gate * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 853*0Sstevel@tonic-gate * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 854*0Sstevel@tonic-gate * 2MB) and any excess pages are put on physavail. The assumption is 855*0Sstevel@tonic-gate * that small-memory systems will need more pages more than they'll 856*0Sstevel@tonic-gate * need efficiently-mapped module texts. 857*0Sstevel@tonic-gate */ 858*0Sstevel@tonic-gate if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 859*0Sstevel@tonic-gate modtext_sz > MODTEXT_SM_CAP) { 860*0Sstevel@tonic-gate extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 861*0Sstevel@tonic-gate modtext_sz = MODTEXT_SM_CAP; 862*0Sstevel@tonic-gate } else 863*0Sstevel@tonic-gate extra_etpg = 0; 864*0Sstevel@tonic-gate PRM_DEBUG(extra_etpg); 865*0Sstevel@tonic-gate PRM_DEBUG(modtext_sz); 866*0Sstevel@tonic-gate extra_etva = modtext + modtext_sz; 867*0Sstevel@tonic-gate PRM_DEBUG(extra_etva); 868*0Sstevel@tonic-gate 869*0Sstevel@tonic-gate /* 870*0Sstevel@tonic-gate * Account for any pages after e_text and e_data. 871*0Sstevel@tonic-gate */ 872*0Sstevel@tonic-gate npages += extra_etpg; 873*0Sstevel@tonic-gate npages += mmu_btopr(nalloc_end - nalloc_base); 874*0Sstevel@tonic-gate PRM_DEBUG(npages); 875*0Sstevel@tonic-gate 876*0Sstevel@tonic-gate /* 877*0Sstevel@tonic-gate * npages is the maximum of available physical memory possible. 878*0Sstevel@tonic-gate * (ie. it will never be more than this) 879*0Sstevel@tonic-gate */ 880*0Sstevel@tonic-gate 881*0Sstevel@tonic-gate /* 882*0Sstevel@tonic-gate * initialize the nucleus memory allocator. 883*0Sstevel@tonic-gate */ 884*0Sstevel@tonic-gate ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 885*0Sstevel@tonic-gate 886*0Sstevel@tonic-gate /* 887*0Sstevel@tonic-gate * Allocate mmu fault status area from the nucleus data area. 888*0Sstevel@tonic-gate */ 889*0Sstevel@tonic-gate if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 890*0Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 891*0Sstevel@tonic-gate 892*0Sstevel@tonic-gate /* 893*0Sstevel@tonic-gate * Allocate kernel TSBs from the nucleus data area. 894*0Sstevel@tonic-gate */ 895*0Sstevel@tonic-gate if (ndata_alloc_tsbs(&ndata, npages) != 0) 896*0Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 897*0Sstevel@tonic-gate 898*0Sstevel@tonic-gate /* 899*0Sstevel@tonic-gate * Allocate cpus structs from the nucleus data area. 900*0Sstevel@tonic-gate */ 901*0Sstevel@tonic-gate if (ndata_alloc_cpus(&ndata) != 0) 902*0Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc"); 903*0Sstevel@tonic-gate 904*0Sstevel@tonic-gate /* 905*0Sstevel@tonic-gate * Allocate dmv dispatch table from the nucleus data area. 906*0Sstevel@tonic-gate */ 907*0Sstevel@tonic-gate if (ndata_alloc_dmv(&ndata) != 0) 908*0Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 909*0Sstevel@tonic-gate 910*0Sstevel@tonic-gate 911*0Sstevel@tonic-gate page_coloring_init(); 912*0Sstevel@tonic-gate 913*0Sstevel@tonic-gate /* 914*0Sstevel@tonic-gate * Allocate page_freelists bin headers for memnode 0 from the 915*0Sstevel@tonic-gate * nucleus data area. 916*0Sstevel@tonic-gate */ 917*0Sstevel@tonic-gate if (ndata_alloc_page_freelists(&ndata, 0) != 0) 918*0Sstevel@tonic-gate cmn_err(CE_PANIC, 919*0Sstevel@tonic-gate "no more nucleus memory after page free lists alloc"); 920*0Sstevel@tonic-gate 921*0Sstevel@tonic-gate if (kpm_enable) { 922*0Sstevel@tonic-gate kpm_init(); 923*0Sstevel@tonic-gate /* 924*0Sstevel@tonic-gate * kpm page space -- Update kpm_npages and make the 925*0Sstevel@tonic-gate * same assumption about fragmenting as it is done 926*0Sstevel@tonic-gate * for memseg_sz. 927*0Sstevel@tonic-gate */ 928*0Sstevel@tonic-gate kpm_npages_setup(memblocks + 4); 929*0Sstevel@tonic-gate } 930*0Sstevel@tonic-gate 931*0Sstevel@tonic-gate /* 932*0Sstevel@tonic-gate * Allocate hat related structs from the nucleus data area. 933*0Sstevel@tonic-gate */ 934*0Sstevel@tonic-gate if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 935*0Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 936*0Sstevel@tonic-gate 937*0Sstevel@tonic-gate /* 938*0Sstevel@tonic-gate * We want to do the BOP_ALLOCs before the real allocation of page 939*0Sstevel@tonic-gate * structs in order to not have to allocate page structs for this 940*0Sstevel@tonic-gate * memory. We need to calculate a virtual address because we want 941*0Sstevel@tonic-gate * the page structs to come before other allocations in virtual address 942*0Sstevel@tonic-gate * space. This is so some (if not all) of page structs can actually 943*0Sstevel@tonic-gate * live in the nucleus. 944*0Sstevel@tonic-gate */ 945*0Sstevel@tonic-gate 946*0Sstevel@tonic-gate /* 947*0Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 948*0Sstevel@tonic-gate * 949*0Sstevel@tonic-gate * There are comments all over the SFMMU code warning of dire 950*0Sstevel@tonic-gate * consequences if the TSBs are moved out of 32-bit space. This 951*0Sstevel@tonic-gate * is largely because the asm code uses "sethi %hi(addr)"-type 952*0Sstevel@tonic-gate * instructions which will not provide the expected result if the 953*0Sstevel@tonic-gate * address is a 64-bit one. 954*0Sstevel@tonic-gate * 955*0Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 956*0Sstevel@tonic-gate */ 957*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 958*0Sstevel@tonic-gate alloc_base = sfmmu_ktsb_alloc(alloc_base); 959*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 960*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 961*0Sstevel@tonic-gate 962*0Sstevel@tonic-gate /* 963*0Sstevel@tonic-gate * Allocate IOMMU TSB array. We do this here so that the physical 964*0Sstevel@tonic-gate * memory gets deducted from the PROM's physical memory list. 965*0Sstevel@tonic-gate */ 966*0Sstevel@tonic-gate alloc_base = iommu_tsb_init(alloc_base); 967*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 968*0Sstevel@tonic-gate ecache_alignsize); 969*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 970*0Sstevel@tonic-gate 971*0Sstevel@tonic-gate /* 972*0Sstevel@tonic-gate * Starcat needs its special structures assigned in 32-bit virtual 973*0Sstevel@tonic-gate * address space because its probing routines execute FCode, and FCode 974*0Sstevel@tonic-gate * can't handle 64-bit virtual addresses... 975*0Sstevel@tonic-gate */ 976*0Sstevel@tonic-gate if (&starcat_startup_memlist) { 977*0Sstevel@tonic-gate alloc_base = starcat_startup_memlist(alloc_base); 978*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 979*0Sstevel@tonic-gate ecache_alignsize); 980*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 981*0Sstevel@tonic-gate } 982*0Sstevel@tonic-gate 983*0Sstevel@tonic-gate /* 984*0Sstevel@tonic-gate * If we have enough memory, use 4M pages for alignment because it 985*0Sstevel@tonic-gate * greatly reduces the number of TLB misses we take albeit at the cost 986*0Sstevel@tonic-gate * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 987*0Sstevel@tonic-gate * allocation.) Still, the speedup on large memory systems (e.g. > 64 988*0Sstevel@tonic-gate * GB) is quite noticeable, so it is worth the effort to do if we can. 989*0Sstevel@tonic-gate * 990*0Sstevel@tonic-gate * Note, however, that this speedup will only occur if the boot PROM 991*0Sstevel@tonic-gate * uses the largest possible MMU page size possible to map memory 992*0Sstevel@tonic-gate * requests that are properly aligned and sized (for example, a request 993*0Sstevel@tonic-gate * for a multiple of 4MB of memory aligned to a 4MB boundary will 994*0Sstevel@tonic-gate * result in a mapping using a 4MB MMU page.) 995*0Sstevel@tonic-gate * 996*0Sstevel@tonic-gate * Even then, the large page mappings will only speed things up until 997*0Sstevel@tonic-gate * the startup process proceeds a bit further, as when 998*0Sstevel@tonic-gate * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 999*0Sstevel@tonic-gate * kernel it remaps everything but the TSBs using 8K pages anyway... 1000*0Sstevel@tonic-gate * 1001*0Sstevel@tonic-gate * At some point in the future, sfmmu_map_prom_mappings() will be 1002*0Sstevel@tonic-gate * rewritten to copy memory mappings to the kernel using the same MMU 1003*0Sstevel@tonic-gate * page sizes the PROM used. When that occurs, if the PROM did use 1004*0Sstevel@tonic-gate * large MMU pages to map memory, the alignment/sizing work we're 1005*0Sstevel@tonic-gate * doing now should give us a nice extra performance boost, albeit at 1006*0Sstevel@tonic-gate * the cost of greater RAM usage... 1007*0Sstevel@tonic-gate */ 1008*0Sstevel@tonic-gate alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1009*0Sstevel@tonic-gate MMU_PAGESIZE); 1010*0Sstevel@tonic-gate 1011*0Sstevel@tonic-gate PRM_DEBUG(tune_npages); 1012*0Sstevel@tonic-gate PRM_DEBUG(alloc_alignsize); 1013*0Sstevel@tonic-gate 1014*0Sstevel@tonic-gate /* 1015*0Sstevel@tonic-gate * Save off where the contiguous allocations to date have ended 1016*0Sstevel@tonic-gate * in econtig32. 1017*0Sstevel@tonic-gate */ 1018*0Sstevel@tonic-gate econtig32 = alloc_base; 1019*0Sstevel@tonic-gate PRM_DEBUG(econtig32); 1020*0Sstevel@tonic-gate 1021*0Sstevel@tonic-gate if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1022*0Sstevel@tonic-gate cmn_err(CE_PANIC, "econtig32 too big"); 1023*0Sstevel@tonic-gate 1024*0Sstevel@tonic-gate /* 1025*0Sstevel@tonic-gate * To avoid memory allocation collisions in the 32-bit virtual address 1026*0Sstevel@tonic-gate * space, make allocations from this point forward in 64-bit virtual 1027*0Sstevel@tonic-gate * address space starting at syslimit and working up. Also use the 1028*0Sstevel@tonic-gate * alignment specified by alloc_alignsize, as we may be able to save 1029*0Sstevel@tonic-gate * ourselves TLB misses by using larger page sizes if they're 1030*0Sstevel@tonic-gate * available. 1031*0Sstevel@tonic-gate * 1032*0Sstevel@tonic-gate * All this is needed because on large memory systems, the default 1033*0Sstevel@tonic-gate * Solaris allocations will collide with SYSBASE32, which is hard 1034*0Sstevel@tonic-gate * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1035*0Sstevel@tonic-gate * kernels, move the allocations to a location in the 64-bit virtual 1036*0Sstevel@tonic-gate * address space space, allowing those structures to grow without 1037*0Sstevel@tonic-gate * worry. 1038*0Sstevel@tonic-gate * 1039*0Sstevel@tonic-gate * On current CPUs we'll run out of physical memory address bits before 1040*0Sstevel@tonic-gate * we need to worry about the allocations running into anything else in 1041*0Sstevel@tonic-gate * VM or the virtual address holes on US-I and II, as there's currently 1042*0Sstevel@tonic-gate * about 1 TB of addressable space before the US-I/II VA hole. 1043*0Sstevel@tonic-gate */ 1044*0Sstevel@tonic-gate kmem64_base = (caddr_t)syslimit; 1045*0Sstevel@tonic-gate PRM_DEBUG(kmem64_base); 1046*0Sstevel@tonic-gate 1047*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1048*0Sstevel@tonic-gate 1049*0Sstevel@tonic-gate /* 1050*0Sstevel@tonic-gate * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1051*0Sstevel@tonic-gate * them here. 1052*0Sstevel@tonic-gate */ 1053*0Sstevel@tonic-gate if (khme_hash == NULL || uhme_hash == NULL) { 1054*0Sstevel@tonic-gate /* 1055*0Sstevel@tonic-gate * alloc_hme_buckets() will align alloc_base properly before 1056*0Sstevel@tonic-gate * assigning the hash buckets, so we don't need to do it 1057*0Sstevel@tonic-gate * before the call... 1058*0Sstevel@tonic-gate */ 1059*0Sstevel@tonic-gate alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1060*0Sstevel@tonic-gate 1061*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1062*0Sstevel@tonic-gate PRM_DEBUG(khme_hash); 1063*0Sstevel@tonic-gate PRM_DEBUG(uhme_hash); 1064*0Sstevel@tonic-gate } 1065*0Sstevel@tonic-gate 1066*0Sstevel@tonic-gate /* 1067*0Sstevel@tonic-gate * Allocate the remaining page freelists. NUMA systems can 1068*0Sstevel@tonic-gate * have lots of page freelists, one per node, which quickly 1069*0Sstevel@tonic-gate * outgrow the amount of nucleus memory available. 1070*0Sstevel@tonic-gate */ 1071*0Sstevel@tonic-gate if (max_mem_nodes > 1) { 1072*0Sstevel@tonic-gate int mnode; 1073*0Sstevel@tonic-gate caddr_t alloc_start = alloc_base; 1074*0Sstevel@tonic-gate 1075*0Sstevel@tonic-gate for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1076*0Sstevel@tonic-gate alloc_base = alloc_page_freelists(mnode, alloc_base, 1077*0Sstevel@tonic-gate ecache_alignsize); 1078*0Sstevel@tonic-gate } 1079*0Sstevel@tonic-gate 1080*0Sstevel@tonic-gate if (alloc_base > alloc_start) { 1081*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1082*0Sstevel@tonic-gate alloc_alignsize); 1083*0Sstevel@tonic-gate if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1084*0Sstevel@tonic-gate alloc_base - alloc_start, 1085*0Sstevel@tonic-gate alloc_alignsize) != alloc_start) 1086*0Sstevel@tonic-gate cmn_err(CE_PANIC, 1087*0Sstevel@tonic-gate "Unable to alloc page freelists\n"); 1088*0Sstevel@tonic-gate } 1089*0Sstevel@tonic-gate 1090*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1091*0Sstevel@tonic-gate } 1092*0Sstevel@tonic-gate 1093*0Sstevel@tonic-gate if (!mml_table) { 1094*0Sstevel@tonic-gate size_t mmltable_sz; 1095*0Sstevel@tonic-gate 1096*0Sstevel@tonic-gate /* 1097*0Sstevel@tonic-gate * We need to allocate the mml_table here because there 1098*0Sstevel@tonic-gate * was not enough space within the nucleus. 1099*0Sstevel@tonic-gate */ 1100*0Sstevel@tonic-gate mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1101*0Sstevel@tonic-gate alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1102*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1103*0Sstevel@tonic-gate alloc_alignsize); 1104*0Sstevel@tonic-gate 1105*0Sstevel@tonic-gate if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1106*0Sstevel@tonic-gate alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1107*0Sstevel@tonic-gate panic("mml_table alloc failure"); 1108*0Sstevel@tonic-gate 1109*0Sstevel@tonic-gate alloc_base += alloc_sz; 1110*0Sstevel@tonic-gate PRM_DEBUG(mml_table); 1111*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1112*0Sstevel@tonic-gate } 1113*0Sstevel@tonic-gate 1114*0Sstevel@tonic-gate if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1115*0Sstevel@tonic-gate size_t kpmptable_sz; 1116*0Sstevel@tonic-gate caddr_t table; 1117*0Sstevel@tonic-gate 1118*0Sstevel@tonic-gate /* 1119*0Sstevel@tonic-gate * We need to allocate either kpmp_table or kpmp_stable here 1120*0Sstevel@tonic-gate * because there was not enough space within the nucleus. 1121*0Sstevel@tonic-gate */ 1122*0Sstevel@tonic-gate kpmptable_sz = (kpm_smallpages == 0) ? 1123*0Sstevel@tonic-gate sizeof (kpm_hlk_t) * kpmp_table_sz : 1124*0Sstevel@tonic-gate sizeof (kpm_shlk_t) * kpmp_stable_sz; 1125*0Sstevel@tonic-gate 1126*0Sstevel@tonic-gate alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1127*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1128*0Sstevel@tonic-gate alloc_alignsize); 1129*0Sstevel@tonic-gate 1130*0Sstevel@tonic-gate table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1131*0Sstevel@tonic-gate alloc_alignsize); 1132*0Sstevel@tonic-gate 1133*0Sstevel@tonic-gate if (table != alloc_base) 1134*0Sstevel@tonic-gate panic("kpmp_table or kpmp_stable alloc failure"); 1135*0Sstevel@tonic-gate 1136*0Sstevel@tonic-gate if (kpm_smallpages == 0) { 1137*0Sstevel@tonic-gate kpmp_table = (kpm_hlk_t *)table; 1138*0Sstevel@tonic-gate PRM_DEBUG(kpmp_table); 1139*0Sstevel@tonic-gate } else { 1140*0Sstevel@tonic-gate kpmp_stable = (kpm_shlk_t *)table; 1141*0Sstevel@tonic-gate PRM_DEBUG(kpmp_stable); 1142*0Sstevel@tonic-gate } 1143*0Sstevel@tonic-gate 1144*0Sstevel@tonic-gate alloc_base += alloc_sz; 1145*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1146*0Sstevel@tonic-gate } 1147*0Sstevel@tonic-gate 1148*0Sstevel@tonic-gate if (&ecache_init_scrub_flush_area) { 1149*0Sstevel@tonic-gate /* 1150*0Sstevel@tonic-gate * Pass alloc_base directly, as the routine itself is 1151*0Sstevel@tonic-gate * responsible for any special alignment requirements... 1152*0Sstevel@tonic-gate */ 1153*0Sstevel@tonic-gate alloc_base = ecache_init_scrub_flush_area(alloc_base); 1154*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1155*0Sstevel@tonic-gate } 1156*0Sstevel@tonic-gate 1157*0Sstevel@tonic-gate /* 1158*0Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 1159*0Sstevel@tonic-gate */ 1160*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1161*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 1162*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 1163*0Sstevel@tonic-gate 1164*0Sstevel@tonic-gate /* 1165*0Sstevel@tonic-gate * Reset npages and memblocks based on boot_physavail list. 1166*0Sstevel@tonic-gate */ 1167*0Sstevel@tonic-gate size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1168*0Sstevel@tonic-gate PRM_DEBUG(npages); 1169*0Sstevel@tonic-gate 1170*0Sstevel@tonic-gate /* 1171*0Sstevel@tonic-gate * Account for extra memory after e_text. 1172*0Sstevel@tonic-gate */ 1173*0Sstevel@tonic-gate npages += extra_etpg; 1174*0Sstevel@tonic-gate 1175*0Sstevel@tonic-gate /* 1176*0Sstevel@tonic-gate * Calculate the largest free memory chunk in the nucleus data area. 1177*0Sstevel@tonic-gate * We need to figure out if page structs can fit in there or not. 1178*0Sstevel@tonic-gate * We also make sure enough page structs get created for any physical 1179*0Sstevel@tonic-gate * memory we might be returning to the system. 1180*0Sstevel@tonic-gate */ 1181*0Sstevel@tonic-gate ndata_remain_sz = ndata_maxsize(&ndata); 1182*0Sstevel@tonic-gate PRM_DEBUG(ndata_remain_sz); 1183*0Sstevel@tonic-gate 1184*0Sstevel@tonic-gate pp_sz = sizeof (struct page) * npages; 1185*0Sstevel@tonic-gate 1186*0Sstevel@tonic-gate /* 1187*0Sstevel@tonic-gate * Here's a nice bit of code based on somewhat recursive logic: 1188*0Sstevel@tonic-gate * 1189*0Sstevel@tonic-gate * If the page array would fit within the nucleus, we want to 1190*0Sstevel@tonic-gate * add npages to cover any extra memory we may be returning back 1191*0Sstevel@tonic-gate * to the system. 1192*0Sstevel@tonic-gate * 1193*0Sstevel@tonic-gate * HOWEVER, the page array is sized by calculating the size of 1194*0Sstevel@tonic-gate * (struct page * npages), as are the pagehash table, ctrs and 1195*0Sstevel@tonic-gate * memseg_list, so the very act of performing the calculation below may 1196*0Sstevel@tonic-gate * in fact make the array large enough that it no longer fits in the 1197*0Sstevel@tonic-gate * nucleus, meaning there would now be a much larger area of the 1198*0Sstevel@tonic-gate * nucleus free that should really be added to npages, which would 1199*0Sstevel@tonic-gate * make the page array that much larger, and so on. 1200*0Sstevel@tonic-gate * 1201*0Sstevel@tonic-gate * This also ignores the memory possibly used in the nucleus for the 1202*0Sstevel@tonic-gate * the page hash, ctrs and memseg list and the fact that whether they 1203*0Sstevel@tonic-gate * fit there or not varies with the npages calculation below, but we 1204*0Sstevel@tonic-gate * don't even factor them into the equation at this point; perhaps we 1205*0Sstevel@tonic-gate * should or perhaps we should just take the approach that the few 1206*0Sstevel@tonic-gate * extra pages we could add via this calculation REALLY aren't worth 1207*0Sstevel@tonic-gate * the hassle... 1208*0Sstevel@tonic-gate */ 1209*0Sstevel@tonic-gate if (ndata_remain_sz > pp_sz) { 1210*0Sstevel@tonic-gate size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1211*0Sstevel@tonic-gate 1212*0Sstevel@tonic-gate npages += mmu_btop(spare); 1213*0Sstevel@tonic-gate 1214*0Sstevel@tonic-gate pp_sz = npages * sizeof (struct page); 1215*0Sstevel@tonic-gate 1216*0Sstevel@tonic-gate pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1217*0Sstevel@tonic-gate } 1218*0Sstevel@tonic-gate 1219*0Sstevel@tonic-gate /* 1220*0Sstevel@tonic-gate * If physmem is patched to be non-zero, use it instead of 1221*0Sstevel@tonic-gate * the monitor value unless physmem is larger than the total 1222*0Sstevel@tonic-gate * amount of memory on hand. 1223*0Sstevel@tonic-gate */ 1224*0Sstevel@tonic-gate if (physmem == 0 || physmem > npages) 1225*0Sstevel@tonic-gate physmem = npages; 1226*0Sstevel@tonic-gate 1227*0Sstevel@tonic-gate /* 1228*0Sstevel@tonic-gate * If pp_base is NULL that means the routines above have determined 1229*0Sstevel@tonic-gate * the page array will not fit in the nucleus; we'll have to 1230*0Sstevel@tonic-gate * BOP_ALLOC() ourselves some space for them. 1231*0Sstevel@tonic-gate */ 1232*0Sstevel@tonic-gate if (pp_base == NULL) { 1233*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1234*0Sstevel@tonic-gate alloc_alignsize); 1235*0Sstevel@tonic-gate 1236*0Sstevel@tonic-gate alloc_sz = roundup(pp_sz, alloc_alignsize); 1237*0Sstevel@tonic-gate 1238*0Sstevel@tonic-gate if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1239*0Sstevel@tonic-gate alloc_base, alloc_sz, alloc_alignsize)) != 1240*0Sstevel@tonic-gate (struct page *)alloc_base) 1241*0Sstevel@tonic-gate panic("page alloc failure"); 1242*0Sstevel@tonic-gate 1243*0Sstevel@tonic-gate alloc_base += alloc_sz; 1244*0Sstevel@tonic-gate } 1245*0Sstevel@tonic-gate 1246*0Sstevel@tonic-gate /* 1247*0Sstevel@tonic-gate * The page structure hash table size is a power of 2 1248*0Sstevel@tonic-gate * such that the average hash chain length is PAGE_HASHAVELEN. 1249*0Sstevel@tonic-gate */ 1250*0Sstevel@tonic-gate page_hashsz = npages / PAGE_HASHAVELEN; 1251*0Sstevel@tonic-gate page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1252*0Sstevel@tonic-gate pagehash_sz = sizeof (struct page *) * page_hashsz; 1253*0Sstevel@tonic-gate 1254*0Sstevel@tonic-gate /* 1255*0Sstevel@tonic-gate * We want to TRY to fit the page structure hash table, 1256*0Sstevel@tonic-gate * the page size free list counters, the memseg list and 1257*0Sstevel@tonic-gate * and the kpm page space in the nucleus if possible. 1258*0Sstevel@tonic-gate * 1259*0Sstevel@tonic-gate * alloc_sz counts how much memory needs to be allocated by 1260*0Sstevel@tonic-gate * BOP_ALLOC(). 1261*0Sstevel@tonic-gate */ 1262*0Sstevel@tonic-gate page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1263*0Sstevel@tonic-gate 1264*0Sstevel@tonic-gate alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1265*0Sstevel@tonic-gate 1266*0Sstevel@tonic-gate /* 1267*0Sstevel@tonic-gate * Size up per page size free list counters. 1268*0Sstevel@tonic-gate */ 1269*0Sstevel@tonic-gate ctrs_sz = page_ctrs_sz(); 1270*0Sstevel@tonic-gate ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1271*0Sstevel@tonic-gate 1272*0Sstevel@tonic-gate if (ctrs_base == NULL) 1273*0Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1274*0Sstevel@tonic-gate 1275*0Sstevel@tonic-gate /* 1276*0Sstevel@tonic-gate * The memseg list is for the chunks of physical memory that 1277*0Sstevel@tonic-gate * will be managed by the vm system. The number calculated is 1278*0Sstevel@tonic-gate * a guess as boot may fragment it more when memory allocations 1279*0Sstevel@tonic-gate * are made before kphysm_init(). Currently, there are two 1280*0Sstevel@tonic-gate * allocations before then, so we assume each causes fragmen- 1281*0Sstevel@tonic-gate * tation, and add a couple more for good measure. 1282*0Sstevel@tonic-gate */ 1283*0Sstevel@tonic-gate memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1284*0Sstevel@tonic-gate memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1285*0Sstevel@tonic-gate 1286*0Sstevel@tonic-gate if (memseg_base == NULL) 1287*0Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1288*0Sstevel@tonic-gate 1289*0Sstevel@tonic-gate 1290*0Sstevel@tonic-gate if (kpm_enable) { 1291*0Sstevel@tonic-gate /* 1292*0Sstevel@tonic-gate * kpm page space -- Update kpm_npages and make the 1293*0Sstevel@tonic-gate * same assumption about fragmenting as it is done 1294*0Sstevel@tonic-gate * for memseg_sz above. 1295*0Sstevel@tonic-gate */ 1296*0Sstevel@tonic-gate kpm_npages_setup(memblocks + 4); 1297*0Sstevel@tonic-gate kpm_pp_sz = (kpm_smallpages == 0) ? 1298*0Sstevel@tonic-gate kpm_npages * sizeof (kpm_page_t): 1299*0Sstevel@tonic-gate kpm_npages * sizeof (kpm_spage_t); 1300*0Sstevel@tonic-gate 1301*0Sstevel@tonic-gate kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1302*0Sstevel@tonic-gate ecache_alignsize); 1303*0Sstevel@tonic-gate 1304*0Sstevel@tonic-gate if (kpm_pp_base == NULL) 1305*0Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1306*0Sstevel@tonic-gate kpm_pp_sz; 1307*0Sstevel@tonic-gate } 1308*0Sstevel@tonic-gate 1309*0Sstevel@tonic-gate if (alloc_sz > 0) { 1310*0Sstevel@tonic-gate uintptr_t bop_base; 1311*0Sstevel@tonic-gate 1312*0Sstevel@tonic-gate /* 1313*0Sstevel@tonic-gate * We need extra memory allocated through BOP_ALLOC. 1314*0Sstevel@tonic-gate */ 1315*0Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1316*0Sstevel@tonic-gate alloc_alignsize); 1317*0Sstevel@tonic-gate 1318*0Sstevel@tonic-gate alloc_sz = roundup(alloc_sz, alloc_alignsize); 1319*0Sstevel@tonic-gate 1320*0Sstevel@tonic-gate if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1321*0Sstevel@tonic-gate alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1322*0Sstevel@tonic-gate panic("system page struct alloc failure"); 1323*0Sstevel@tonic-gate 1324*0Sstevel@tonic-gate alloc_base += alloc_sz; 1325*0Sstevel@tonic-gate 1326*0Sstevel@tonic-gate if (page_hash == NULL) { 1327*0Sstevel@tonic-gate page_hash = (struct page **)bop_base; 1328*0Sstevel@tonic-gate bop_base = roundup(bop_base + pagehash_sz, 1329*0Sstevel@tonic-gate ecache_alignsize); 1330*0Sstevel@tonic-gate } 1331*0Sstevel@tonic-gate 1332*0Sstevel@tonic-gate if (ctrs_base == NULL) { 1333*0Sstevel@tonic-gate ctrs_base = (caddr_t)bop_base; 1334*0Sstevel@tonic-gate bop_base = roundup(bop_base + ctrs_sz, 1335*0Sstevel@tonic-gate ecache_alignsize); 1336*0Sstevel@tonic-gate } 1337*0Sstevel@tonic-gate 1338*0Sstevel@tonic-gate if (memseg_base == NULL) { 1339*0Sstevel@tonic-gate memseg_base = (struct memseg *)bop_base; 1340*0Sstevel@tonic-gate bop_base = roundup(bop_base + memseg_sz, 1341*0Sstevel@tonic-gate ecache_alignsize); 1342*0Sstevel@tonic-gate } 1343*0Sstevel@tonic-gate 1344*0Sstevel@tonic-gate if (kpm_enable && kpm_pp_base == NULL) { 1345*0Sstevel@tonic-gate kpm_pp_base = (uintptr_t)bop_base; 1346*0Sstevel@tonic-gate bop_base = roundup(bop_base + kpm_pp_sz, 1347*0Sstevel@tonic-gate ecache_alignsize); 1348*0Sstevel@tonic-gate } 1349*0Sstevel@tonic-gate 1350*0Sstevel@tonic-gate ASSERT(bop_base <= (uintptr_t)alloc_base); 1351*0Sstevel@tonic-gate } 1352*0Sstevel@tonic-gate 1353*0Sstevel@tonic-gate /* 1354*0Sstevel@tonic-gate * Initialize per page size free list counters. 1355*0Sstevel@tonic-gate */ 1356*0Sstevel@tonic-gate ctrs_end = page_ctrs_alloc(ctrs_base); 1357*0Sstevel@tonic-gate ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1358*0Sstevel@tonic-gate 1359*0Sstevel@tonic-gate PRM_DEBUG(page_hash); 1360*0Sstevel@tonic-gate PRM_DEBUG(memseg_base); 1361*0Sstevel@tonic-gate PRM_DEBUG(kpm_pp_base); 1362*0Sstevel@tonic-gate PRM_DEBUG(kpm_pp_sz); 1363*0Sstevel@tonic-gate PRM_DEBUG(pp_base); 1364*0Sstevel@tonic-gate PRM_DEBUG(pp_sz); 1365*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1366*0Sstevel@tonic-gate 1367*0Sstevel@tonic-gate #ifdef TRAPTRACE 1368*0Sstevel@tonic-gate /* 1369*0Sstevel@tonic-gate * Allocate trap trace buffer last so as not to affect 1370*0Sstevel@tonic-gate * the 4M alignments of the allocations above on V9 SPARCs... 1371*0Sstevel@tonic-gate */ 1372*0Sstevel@tonic-gate alloc_base = trap_trace_alloc(alloc_base); 1373*0Sstevel@tonic-gate PRM_DEBUG(alloc_base); 1374*0Sstevel@tonic-gate #endif /* TRAPTRACE */ 1375*0Sstevel@tonic-gate 1376*0Sstevel@tonic-gate if (kmem64_base) { 1377*0Sstevel@tonic-gate /* 1378*0Sstevel@tonic-gate * Set the end of the kmem64 segment for V9 SPARCs, if 1379*0Sstevel@tonic-gate * appropriate... 1380*0Sstevel@tonic-gate */ 1381*0Sstevel@tonic-gate kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1382*0Sstevel@tonic-gate alloc_alignsize); 1383*0Sstevel@tonic-gate 1384*0Sstevel@tonic-gate PRM_DEBUG(kmem64_base); 1385*0Sstevel@tonic-gate PRM_DEBUG(kmem64_end); 1386*0Sstevel@tonic-gate } 1387*0Sstevel@tonic-gate 1388*0Sstevel@tonic-gate /* 1389*0Sstevel@tonic-gate * Allocate space for the interrupt vector table. 1390*0Sstevel@tonic-gate */ 1391*0Sstevel@tonic-gate memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector, 1392*0Sstevel@tonic-gate IVSIZE, MMU_PAGESIZE); 1393*0Sstevel@tonic-gate if (memspace != (caddr_t)intr_vector) 1394*0Sstevel@tonic-gate panic("interrupt table allocation failure"); 1395*0Sstevel@tonic-gate 1396*0Sstevel@tonic-gate /* 1397*0Sstevel@tonic-gate * The memory lists from boot are allocated from the heap arena 1398*0Sstevel@tonic-gate * so that later they can be freed and/or reallocated. 1399*0Sstevel@tonic-gate */ 1400*0Sstevel@tonic-gate if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1401*0Sstevel@tonic-gate panic("could not retrieve property \"extent\""); 1402*0Sstevel@tonic-gate 1403*0Sstevel@tonic-gate /* 1404*0Sstevel@tonic-gate * Between now and when we finish copying in the memory lists, 1405*0Sstevel@tonic-gate * allocations happen so the space gets fragmented and the 1406*0Sstevel@tonic-gate * lists longer. Leave enough space for lists twice as long 1407*0Sstevel@tonic-gate * as what boot says it has now; roundup to a pagesize. 1408*0Sstevel@tonic-gate * Also add space for the final phys-avail copy in the fixup 1409*0Sstevel@tonic-gate * routine. 1410*0Sstevel@tonic-gate */ 1411*0Sstevel@tonic-gate va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1412*0Sstevel@tonic-gate roundup(IVSIZE, MMU_PAGESIZE)); 1413*0Sstevel@tonic-gate memlist_sz *= 4; 1414*0Sstevel@tonic-gate memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1415*0Sstevel@tonic-gate memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1416*0Sstevel@tonic-gate if (memspace == NULL) 1417*0Sstevel@tonic-gate halt("Boot allocation failed."); 1418*0Sstevel@tonic-gate 1419*0Sstevel@tonic-gate memlist = (struct memlist *)memspace; 1420*0Sstevel@tonic-gate memlist_end = (char *)memspace + memlist_sz; 1421*0Sstevel@tonic-gate 1422*0Sstevel@tonic-gate PRM_DEBUG(memlist); 1423*0Sstevel@tonic-gate PRM_DEBUG(memlist_end); 1424*0Sstevel@tonic-gate PRM_DEBUG(sysbase); 1425*0Sstevel@tonic-gate PRM_DEBUG(syslimit); 1426*0Sstevel@tonic-gate 1427*0Sstevel@tonic-gate kernelheap_init((void *)sysbase, (void *)syslimit, 1428*0Sstevel@tonic-gate (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1429*0Sstevel@tonic-gate 1430*0Sstevel@tonic-gate /* 1431*0Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 1432*0Sstevel@tonic-gate */ 1433*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1434*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 1435*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 1436*0Sstevel@tonic-gate 1437*0Sstevel@tonic-gate /* 1438*0Sstevel@tonic-gate * Remove the space used by BOP_ALLOC from the kernel heap 1439*0Sstevel@tonic-gate * plus the area actually used by the OBP (if any) 1440*0Sstevel@tonic-gate * ignoring virtual addresses in virt_avail, above syslimit. 1441*0Sstevel@tonic-gate */ 1442*0Sstevel@tonic-gate virt_avail = memlist; 1443*0Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1444*0Sstevel@tonic-gate 1445*0Sstevel@tonic-gate for (cur = virt_avail; cur->next; cur = cur->next) { 1446*0Sstevel@tonic-gate uint64_t range_base, range_size; 1447*0Sstevel@tonic-gate 1448*0Sstevel@tonic-gate if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1449*0Sstevel@tonic-gate continue; 1450*0Sstevel@tonic-gate if (range_base >= (uint64_t)syslimit) 1451*0Sstevel@tonic-gate break; 1452*0Sstevel@tonic-gate /* 1453*0Sstevel@tonic-gate * Limit the range to end at syslimit. 1454*0Sstevel@tonic-gate */ 1455*0Sstevel@tonic-gate range_size = MIN(cur->next->address, 1456*0Sstevel@tonic-gate (uint64_t)syslimit) - range_base; 1457*0Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1458*0Sstevel@tonic-gate 0, 0, (void *)range_base, (void *)(range_base + range_size), 1459*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1460*0Sstevel@tonic-gate } 1461*0Sstevel@tonic-gate 1462*0Sstevel@tonic-gate phys_avail = memlist; 1463*0Sstevel@tonic-gate (void) copy_physavail(boot_physavail, boot_physavail_len, 1464*0Sstevel@tonic-gate &memlist, 0, 0); 1465*0Sstevel@tonic-gate 1466*0Sstevel@tonic-gate /* 1467*0Sstevel@tonic-gate * Add any extra memory after e_text to the phys_avail list, as long 1468*0Sstevel@tonic-gate * as there's at least a page to add. 1469*0Sstevel@tonic-gate */ 1470*0Sstevel@tonic-gate if (extra_etpg) 1471*0Sstevel@tonic-gate memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1472*0Sstevel@tonic-gate &memlist, &phys_avail); 1473*0Sstevel@tonic-gate 1474*0Sstevel@tonic-gate /* 1475*0Sstevel@tonic-gate * Add any extra memory after e_data to the phys_avail list as long 1476*0Sstevel@tonic-gate * as there's at least a page to add. Usually, there isn't any, 1477*0Sstevel@tonic-gate * since extra HME blocks typically get allocated there first before 1478*0Sstevel@tonic-gate * using RAM elsewhere. 1479*0Sstevel@tonic-gate */ 1480*0Sstevel@tonic-gate if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1481*0Sstevel@tonic-gate nalloc_base = nalloc_end; 1482*0Sstevel@tonic-gate ndata_remain_sz = nalloc_end - nalloc_base; 1483*0Sstevel@tonic-gate 1484*0Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE) 1485*0Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base), 1486*0Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1487*0Sstevel@tonic-gate 1488*0Sstevel@tonic-gate PRM_DEBUG(memlist); 1489*0Sstevel@tonic-gate PRM_DEBUG(memlist_sz); 1490*0Sstevel@tonic-gate PRM_DEBUG(memspace); 1491*0Sstevel@tonic-gate 1492*0Sstevel@tonic-gate if ((caddr_t)memlist > (memspace + memlist_sz)) 1493*0Sstevel@tonic-gate panic("memlist overflow"); 1494*0Sstevel@tonic-gate 1495*0Sstevel@tonic-gate PRM_DEBUG(pp_base); 1496*0Sstevel@tonic-gate PRM_DEBUG(memseg_base); 1497*0Sstevel@tonic-gate PRM_DEBUG(npages); 1498*0Sstevel@tonic-gate 1499*0Sstevel@tonic-gate /* 1500*0Sstevel@tonic-gate * Initialize the page structures from the memory lists. 1501*0Sstevel@tonic-gate */ 1502*0Sstevel@tonic-gate kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1503*0Sstevel@tonic-gate 1504*0Sstevel@tonic-gate availrmem_initial = availrmem = freemem; 1505*0Sstevel@tonic-gate PRM_DEBUG(availrmem); 1506*0Sstevel@tonic-gate 1507*0Sstevel@tonic-gate /* 1508*0Sstevel@tonic-gate * Some of the locks depend on page_hashsz being set! 1509*0Sstevel@tonic-gate * kmem_init() depends on this; so, keep it here. 1510*0Sstevel@tonic-gate */ 1511*0Sstevel@tonic-gate page_lock_init(); 1512*0Sstevel@tonic-gate 1513*0Sstevel@tonic-gate /* 1514*0Sstevel@tonic-gate * Initialize kernel memory allocator. 1515*0Sstevel@tonic-gate */ 1516*0Sstevel@tonic-gate kmem_init(); 1517*0Sstevel@tonic-gate 1518*0Sstevel@tonic-gate /* 1519*0Sstevel@tonic-gate * Initialize bp_mapin(). 1520*0Sstevel@tonic-gate */ 1521*0Sstevel@tonic-gate bp_init(shm_alignment, HAT_STRICTORDER); 1522*0Sstevel@tonic-gate 1523*0Sstevel@tonic-gate /* 1524*0Sstevel@tonic-gate * Reserve space for panicbuf and intr_vector from the 32-bit heap 1525*0Sstevel@tonic-gate */ 1526*0Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1527*0Sstevel@tonic-gate panicbuf, panicbuf + PANICBUFSIZE, 1528*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1529*0Sstevel@tonic-gate 1530*0Sstevel@tonic-gate (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1531*0Sstevel@tonic-gate intr_vector, (caddr_t)intr_vector + IVSIZE, 1532*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1533*0Sstevel@tonic-gate 1534*0Sstevel@tonic-gate mem_config_init(); 1535*0Sstevel@tonic-gate } 1536*0Sstevel@tonic-gate 1537*0Sstevel@tonic-gate static void 1538*0Sstevel@tonic-gate startup_modules(void) 1539*0Sstevel@tonic-gate { 1540*0Sstevel@tonic-gate int proplen, nhblk1, nhblk8; 1541*0Sstevel@tonic-gate size_t nhblksz; 1542*0Sstevel@tonic-gate pgcnt_t hblk_pages, pages_per_hblk; 1543*0Sstevel@tonic-gate size_t hme8blk_sz, hme1blk_sz; 1544*0Sstevel@tonic-gate 1545*0Sstevel@tonic-gate /* 1546*0Sstevel@tonic-gate * Log any optional messages from the boot program 1547*0Sstevel@tonic-gate */ 1548*0Sstevel@tonic-gate proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1549*0Sstevel@tonic-gate if (proplen > 0) { 1550*0Sstevel@tonic-gate char *msg; 1551*0Sstevel@tonic-gate size_t len = (size_t)proplen; 1552*0Sstevel@tonic-gate 1553*0Sstevel@tonic-gate msg = kmem_zalloc(len, KM_SLEEP); 1554*0Sstevel@tonic-gate (void) BOP_GETPROP(bootops, "boot-message", msg); 1555*0Sstevel@tonic-gate cmn_err(CE_CONT, "?%s\n", msg); 1556*0Sstevel@tonic-gate kmem_free(msg, len); 1557*0Sstevel@tonic-gate } 1558*0Sstevel@tonic-gate 1559*0Sstevel@tonic-gate /* 1560*0Sstevel@tonic-gate * Let the platforms have a chance to change default 1561*0Sstevel@tonic-gate * values before reading system file. 1562*0Sstevel@tonic-gate */ 1563*0Sstevel@tonic-gate if (&set_platform_defaults) 1564*0Sstevel@tonic-gate set_platform_defaults(); 1565*0Sstevel@tonic-gate 1566*0Sstevel@tonic-gate /* 1567*0Sstevel@tonic-gate * Calculate default settings of system parameters based upon 1568*0Sstevel@tonic-gate * maxusers, yet allow to be overridden via the /etc/system file. 1569*0Sstevel@tonic-gate */ 1570*0Sstevel@tonic-gate param_calc(0); 1571*0Sstevel@tonic-gate 1572*0Sstevel@tonic-gate mod_setup(); 1573*0Sstevel@tonic-gate 1574*0Sstevel@tonic-gate /* 1575*0Sstevel@tonic-gate * If this is a positron, complain and halt. 1576*0Sstevel@tonic-gate */ 1577*0Sstevel@tonic-gate if (&iam_positron && iam_positron()) { 1578*0Sstevel@tonic-gate cmn_err(CE_WARN, "This hardware platform is not supported" 1579*0Sstevel@tonic-gate " by this release of Solaris.\n"); 1580*0Sstevel@tonic-gate #ifdef DEBUG 1581*0Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to resume */ 1582*0Sstevel@tonic-gate cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1583*0Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1584*0Sstevel@tonic-gate 1585*0Sstevel@tonic-gate #else /* DEBUG */ 1586*0Sstevel@tonic-gate halt(0); 1587*0Sstevel@tonic-gate #endif /* DEBUG */ 1588*0Sstevel@tonic-gate } 1589*0Sstevel@tonic-gate 1590*0Sstevel@tonic-gate /* 1591*0Sstevel@tonic-gate * If we are running firmware that isn't 64-bit ready 1592*0Sstevel@tonic-gate * then complain and halt. 1593*0Sstevel@tonic-gate */ 1594*0Sstevel@tonic-gate do_prom_version_check(); 1595*0Sstevel@tonic-gate 1596*0Sstevel@tonic-gate /* 1597*0Sstevel@tonic-gate * Initialize system parameters 1598*0Sstevel@tonic-gate */ 1599*0Sstevel@tonic-gate param_init(); 1600*0Sstevel@tonic-gate 1601*0Sstevel@tonic-gate /* 1602*0Sstevel@tonic-gate * maxmem is the amount of physical memory we're playing with. 1603*0Sstevel@tonic-gate */ 1604*0Sstevel@tonic-gate maxmem = physmem; 1605*0Sstevel@tonic-gate 1606*0Sstevel@tonic-gate /* Set segkp limits. */ 1607*0Sstevel@tonic-gate ncbase = (caddr_t)SEGDEBUGBASE; 1608*0Sstevel@tonic-gate ncend = (caddr_t)SEGDEBUGBASE; 1609*0Sstevel@tonic-gate 1610*0Sstevel@tonic-gate /* 1611*0Sstevel@tonic-gate * Initialize the hat layer. 1612*0Sstevel@tonic-gate */ 1613*0Sstevel@tonic-gate hat_init(); 1614*0Sstevel@tonic-gate 1615*0Sstevel@tonic-gate /* 1616*0Sstevel@tonic-gate * Initialize segment management stuff. 1617*0Sstevel@tonic-gate */ 1618*0Sstevel@tonic-gate seg_init(); 1619*0Sstevel@tonic-gate 1620*0Sstevel@tonic-gate /* 1621*0Sstevel@tonic-gate * Create the va>tte handler, so the prom can understand 1622*0Sstevel@tonic-gate * kernel translations. The handler is installed later, just 1623*0Sstevel@tonic-gate * as we are about to take over the trap table from the prom. 1624*0Sstevel@tonic-gate */ 1625*0Sstevel@tonic-gate create_va_to_tte(); 1626*0Sstevel@tonic-gate 1627*0Sstevel@tonic-gate /* 1628*0Sstevel@tonic-gate * Load the forthdebugger (optional) 1629*0Sstevel@tonic-gate */ 1630*0Sstevel@tonic-gate forthdebug_init(); 1631*0Sstevel@tonic-gate 1632*0Sstevel@tonic-gate /* 1633*0Sstevel@tonic-gate * Create OBP node for console input callbacks 1634*0Sstevel@tonic-gate * if it is needed. 1635*0Sstevel@tonic-gate */ 1636*0Sstevel@tonic-gate startup_create_input_node(); 1637*0Sstevel@tonic-gate 1638*0Sstevel@tonic-gate if (modloadonly("fs", "specfs") == -1) 1639*0Sstevel@tonic-gate halt("Can't load specfs"); 1640*0Sstevel@tonic-gate 1641*0Sstevel@tonic-gate if (modloadonly("fs", "devfs") == -1) 1642*0Sstevel@tonic-gate halt("Can't load devfs"); 1643*0Sstevel@tonic-gate 1644*0Sstevel@tonic-gate if (modloadonly("misc", "swapgeneric") == -1) 1645*0Sstevel@tonic-gate halt("Can't load swapgeneric"); 1646*0Sstevel@tonic-gate 1647*0Sstevel@tonic-gate dispinit(); 1648*0Sstevel@tonic-gate 1649*0Sstevel@tonic-gate /* 1650*0Sstevel@tonic-gate * Infer meanings to the members of the idprom buffer. 1651*0Sstevel@tonic-gate */ 1652*0Sstevel@tonic-gate parse_idprom(); 1653*0Sstevel@tonic-gate 1654*0Sstevel@tonic-gate /* Read cluster configuration data. */ 1655*0Sstevel@tonic-gate clconf_init(); 1656*0Sstevel@tonic-gate 1657*0Sstevel@tonic-gate setup_ddi(); 1658*0Sstevel@tonic-gate 1659*0Sstevel@tonic-gate /* 1660*0Sstevel@tonic-gate * Lets take this opportunity to load the root device. 1661*0Sstevel@tonic-gate */ 1662*0Sstevel@tonic-gate if (loadrootmodules() != 0) 1663*0Sstevel@tonic-gate debug_enter("Can't load the root filesystem"); 1664*0Sstevel@tonic-gate 1665*0Sstevel@tonic-gate /* 1666*0Sstevel@tonic-gate * Load tod driver module for the tod part found on this system. 1667*0Sstevel@tonic-gate * Recompute the cpu frequency/delays based on tod as tod part 1668*0Sstevel@tonic-gate * tends to keep time more accurately. 1669*0Sstevel@tonic-gate */ 1670*0Sstevel@tonic-gate if (&load_tod_module) 1671*0Sstevel@tonic-gate load_tod_module(); 1672*0Sstevel@tonic-gate 1673*0Sstevel@tonic-gate /* 1674*0Sstevel@tonic-gate * Allow platforms to load modules which might 1675*0Sstevel@tonic-gate * be needed after bootops are gone. 1676*0Sstevel@tonic-gate */ 1677*0Sstevel@tonic-gate if (&load_platform_modules) 1678*0Sstevel@tonic-gate load_platform_modules(); 1679*0Sstevel@tonic-gate 1680*0Sstevel@tonic-gate setcpudelay(); 1681*0Sstevel@tonic-gate 1682*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1683*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 1684*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 1685*0Sstevel@tonic-gate 1686*0Sstevel@tonic-gate bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1687*0Sstevel@tonic-gate 1688*0Sstevel@tonic-gate /* 1689*0Sstevel@tonic-gate * Calculation and allocation of hmeblks needed to remap 1690*0Sstevel@tonic-gate * the memory allocated by PROM till now: 1691*0Sstevel@tonic-gate * 1692*0Sstevel@tonic-gate * (1) calculate how much virtual memory has been bop_alloc'ed. 1693*0Sstevel@tonic-gate * (2) roundup this memory to span of hme8blk, i.e. 64KB 1694*0Sstevel@tonic-gate * (3) calculate number of hme8blk's needed to remap this memory 1695*0Sstevel@tonic-gate * (4) calculate amount of memory that's consumed by these hme8blk's 1696*0Sstevel@tonic-gate * (5) add memory calculated in steps (2) and (4) above. 1697*0Sstevel@tonic-gate * (6) roundup this memory to span of hme8blk, i.e. 64KB 1698*0Sstevel@tonic-gate * (7) calculate number of hme8blk's needed to remap this memory 1699*0Sstevel@tonic-gate * (8) calculate amount of memory that's consumed by these hme8blk's 1700*0Sstevel@tonic-gate * (9) allocate additional hme1blk's to hold large mappings. 1701*0Sstevel@tonic-gate * H8TOH1 determines this. The current SWAG gives enough hblk1's 1702*0Sstevel@tonic-gate * to remap everything with 4M mappings. 1703*0Sstevel@tonic-gate * (10) account for partially used hblk8's due to non-64K aligned 1704*0Sstevel@tonic-gate * PROM mapping entries. 1705*0Sstevel@tonic-gate * (11) add memory calculated in steps (8), (9), and (10) above. 1706*0Sstevel@tonic-gate * (12) kmem_zalloc the memory calculated in (11); since segkmem 1707*0Sstevel@tonic-gate * is not ready yet, this gets bop_alloc'ed. 1708*0Sstevel@tonic-gate * (13) there will be very few bop_alloc's after this point before 1709*0Sstevel@tonic-gate * trap table takes over 1710*0Sstevel@tonic-gate */ 1711*0Sstevel@tonic-gate 1712*0Sstevel@tonic-gate /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1713*0Sstevel@tonic-gate hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1714*0Sstevel@tonic-gate hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1715*0Sstevel@tonic-gate 1716*0Sstevel@tonic-gate pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1717*0Sstevel@tonic-gate bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1718*0Sstevel@tonic-gate nhblk8 = bop_alloc_pages / pages_per_hblk; 1719*0Sstevel@tonic-gate nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1720*0Sstevel@tonic-gate hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1721*0Sstevel@tonic-gate bop_alloc_pages += hblk_pages; 1722*0Sstevel@tonic-gate bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1723*0Sstevel@tonic-gate nhblk8 = bop_alloc_pages / pages_per_hblk; 1724*0Sstevel@tonic-gate nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1725*0Sstevel@tonic-gate if (nhblk1 < hblk1_min) 1726*0Sstevel@tonic-gate nhblk1 = hblk1_min; 1727*0Sstevel@tonic-gate if (nhblk8 < hblk8_min) 1728*0Sstevel@tonic-gate nhblk8 = hblk8_min; 1729*0Sstevel@tonic-gate 1730*0Sstevel@tonic-gate /* 1731*0Sstevel@tonic-gate * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1732*0Sstevel@tonic-gate * boundary, the number of hblk8's needed to map the entries in the 1733*0Sstevel@tonic-gate * boot_virtavail list needs to be adjusted to take this into 1734*0Sstevel@tonic-gate * consideration. Thus, we need to add additional hblk8's since it 1735*0Sstevel@tonic-gate * is possible that an hblk8 will not have all 8 slots used due to 1736*0Sstevel@tonic-gate * alignment constraints. Since there were boot_virtavail_len entries 1737*0Sstevel@tonic-gate * in that list, we need to add that many hblk8's to the number 1738*0Sstevel@tonic-gate * already calculated to make sure we don't underestimate. 1739*0Sstevel@tonic-gate */ 1740*0Sstevel@tonic-gate nhblk8 += boot_virtavail_len; 1741*0Sstevel@tonic-gate nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1742*0Sstevel@tonic-gate 1743*0Sstevel@tonic-gate /* Allocate in pagesize chunks */ 1744*0Sstevel@tonic-gate nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1745*0Sstevel@tonic-gate hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1746*0Sstevel@tonic-gate sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1747*0Sstevel@tonic-gate } 1748*0Sstevel@tonic-gate 1749*0Sstevel@tonic-gate static void 1750*0Sstevel@tonic-gate startup_bop_gone(void) 1751*0Sstevel@tonic-gate { 1752*0Sstevel@tonic-gate extern int bop_io_quiesced; 1753*0Sstevel@tonic-gate 1754*0Sstevel@tonic-gate /* 1755*0Sstevel@tonic-gate * Call back into boot and release boots resources. 1756*0Sstevel@tonic-gate */ 1757*0Sstevel@tonic-gate BOP_QUIESCE_IO(bootops); 1758*0Sstevel@tonic-gate bop_io_quiesced = 1; 1759*0Sstevel@tonic-gate 1760*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1761*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 1762*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 1763*0Sstevel@tonic-gate /* 1764*0Sstevel@tonic-gate * Copy physinstalled list into kernel space. 1765*0Sstevel@tonic-gate */ 1766*0Sstevel@tonic-gate phys_install = memlist; 1767*0Sstevel@tonic-gate copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1768*0Sstevel@tonic-gate 1769*0Sstevel@tonic-gate /* 1770*0Sstevel@tonic-gate * setup physically contiguous area twice as large as the ecache. 1771*0Sstevel@tonic-gate * this is used while doing displacement flush of ecaches 1772*0Sstevel@tonic-gate */ 1773*0Sstevel@tonic-gate if (&ecache_flush_address) { 1774*0Sstevel@tonic-gate ecache_flushaddr = ecache_flush_address(); 1775*0Sstevel@tonic-gate if (ecache_flushaddr == (uint64_t)-1) { 1776*0Sstevel@tonic-gate cmn_err(CE_PANIC, 1777*0Sstevel@tonic-gate "startup: no memory to set ecache_flushaddr"); 1778*0Sstevel@tonic-gate } 1779*0Sstevel@tonic-gate } 1780*0Sstevel@tonic-gate 1781*0Sstevel@tonic-gate /* 1782*0Sstevel@tonic-gate * Virtual available next. 1783*0Sstevel@tonic-gate */ 1784*0Sstevel@tonic-gate ASSERT(virt_avail != NULL); 1785*0Sstevel@tonic-gate memlist_free_list(virt_avail); 1786*0Sstevel@tonic-gate virt_avail = memlist; 1787*0Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1788*0Sstevel@tonic-gate 1789*0Sstevel@tonic-gate /* 1790*0Sstevel@tonic-gate * Last chance to ask our booter questions .. 1791*0Sstevel@tonic-gate */ 1792*0Sstevel@tonic-gate } 1793*0Sstevel@tonic-gate 1794*0Sstevel@tonic-gate 1795*0Sstevel@tonic-gate /* 1796*0Sstevel@tonic-gate * startup_fixup_physavail - called from mach_sfmmu.c after the final 1797*0Sstevel@tonic-gate * allocations have been performed. We can't call it in startup_bop_gone 1798*0Sstevel@tonic-gate * since later operations can cause obp to allocate more memory. 1799*0Sstevel@tonic-gate */ 1800*0Sstevel@tonic-gate void 1801*0Sstevel@tonic-gate startup_fixup_physavail(void) 1802*0Sstevel@tonic-gate { 1803*0Sstevel@tonic-gate struct memlist *cur; 1804*0Sstevel@tonic-gate 1805*0Sstevel@tonic-gate /* 1806*0Sstevel@tonic-gate * take the most current snapshot we can by calling mem-update 1807*0Sstevel@tonic-gate */ 1808*0Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1809*0Sstevel@tonic-gate &boot_physavail, &boot_physavail_len, 1810*0Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len); 1811*0Sstevel@tonic-gate 1812*0Sstevel@tonic-gate /* 1813*0Sstevel@tonic-gate * Copy phys_avail list, again. 1814*0Sstevel@tonic-gate * Both the kernel/boot and the prom have been allocating 1815*0Sstevel@tonic-gate * from the original list we copied earlier. 1816*0Sstevel@tonic-gate */ 1817*0Sstevel@tonic-gate cur = memlist; 1818*0Sstevel@tonic-gate (void) copy_physavail(boot_physavail, boot_physavail_len, 1819*0Sstevel@tonic-gate &memlist, 0, 0); 1820*0Sstevel@tonic-gate 1821*0Sstevel@tonic-gate /* 1822*0Sstevel@tonic-gate * Add any extra memory after e_text we added to the phys_avail list 1823*0Sstevel@tonic-gate * back to the old list. 1824*0Sstevel@tonic-gate */ 1825*0Sstevel@tonic-gate if (extra_etpg) 1826*0Sstevel@tonic-gate memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1827*0Sstevel@tonic-gate &memlist, &cur); 1828*0Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE) 1829*0Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base), 1830*0Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &cur); 1831*0Sstevel@tonic-gate 1832*0Sstevel@tonic-gate /* 1833*0Sstevel@tonic-gate * There isn't any bounds checking on the memlist area 1834*0Sstevel@tonic-gate * so ensure it hasn't overgrown. 1835*0Sstevel@tonic-gate */ 1836*0Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 1837*0Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1838*0Sstevel@tonic-gate 1839*0Sstevel@tonic-gate /* 1840*0Sstevel@tonic-gate * The kernel removes the pages that were allocated for it from 1841*0Sstevel@tonic-gate * the freelist, but we now have to find any -extra- pages that 1842*0Sstevel@tonic-gate * the prom has allocated for it's own book-keeping, and remove 1843*0Sstevel@tonic-gate * them from the freelist too. sigh. 1844*0Sstevel@tonic-gate */ 1845*0Sstevel@tonic-gate fix_prom_pages(phys_avail, cur); 1846*0Sstevel@tonic-gate 1847*0Sstevel@tonic-gate ASSERT(phys_avail != NULL); 1848*0Sstevel@tonic-gate memlist_free_list(phys_avail); 1849*0Sstevel@tonic-gate phys_avail = cur; 1850*0Sstevel@tonic-gate 1851*0Sstevel@tonic-gate /* 1852*0Sstevel@tonic-gate * We're done with boot. Just after this point in time, boot 1853*0Sstevel@tonic-gate * gets unmapped, so we can no longer rely on its services. 1854*0Sstevel@tonic-gate * Zero the bootops to indicate this fact. 1855*0Sstevel@tonic-gate */ 1856*0Sstevel@tonic-gate bootops = (struct bootops *)NULL; 1857*0Sstevel@tonic-gate BOOTOPS_GONE(); 1858*0Sstevel@tonic-gate } 1859*0Sstevel@tonic-gate 1860*0Sstevel@tonic-gate static void 1861*0Sstevel@tonic-gate startup_vm(void) 1862*0Sstevel@tonic-gate { 1863*0Sstevel@tonic-gate size_t i; 1864*0Sstevel@tonic-gate struct segmap_crargs a; 1865*0Sstevel@tonic-gate struct segkpm_crargs b; 1866*0Sstevel@tonic-gate 1867*0Sstevel@tonic-gate uint64_t avmem; 1868*0Sstevel@tonic-gate caddr_t va; 1869*0Sstevel@tonic-gate pgcnt_t max_phys_segkp; 1870*0Sstevel@tonic-gate int mnode; 1871*0Sstevel@tonic-gate 1872*0Sstevel@tonic-gate /* 1873*0Sstevel@tonic-gate * get prom's mappings, create hments for them and switch 1874*0Sstevel@tonic-gate * to the kernel context. 1875*0Sstevel@tonic-gate */ 1876*0Sstevel@tonic-gate hat_kern_setup(); 1877*0Sstevel@tonic-gate 1878*0Sstevel@tonic-gate /* 1879*0Sstevel@tonic-gate * Take over trap table 1880*0Sstevel@tonic-gate */ 1881*0Sstevel@tonic-gate setup_trap_table(); 1882*0Sstevel@tonic-gate 1883*0Sstevel@tonic-gate /* 1884*0Sstevel@tonic-gate * Install the va>tte handler, so that the prom can handle 1885*0Sstevel@tonic-gate * misses and understand the kernel table layout in case 1886*0Sstevel@tonic-gate * we need call into the prom. 1887*0Sstevel@tonic-gate */ 1888*0Sstevel@tonic-gate install_va_to_tte(); 1889*0Sstevel@tonic-gate 1890*0Sstevel@tonic-gate /* 1891*0Sstevel@tonic-gate * Set a flag to indicate that the tba has been taken over. 1892*0Sstevel@tonic-gate */ 1893*0Sstevel@tonic-gate tba_taken_over = 1; 1894*0Sstevel@tonic-gate 1895*0Sstevel@tonic-gate /* initialize MMU primary context register */ 1896*0Sstevel@tonic-gate mmu_init_kcontext(); 1897*0Sstevel@tonic-gate 1898*0Sstevel@tonic-gate /* 1899*0Sstevel@tonic-gate * The boot cpu can now take interrupts, x-calls, x-traps 1900*0Sstevel@tonic-gate */ 1901*0Sstevel@tonic-gate CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1902*0Sstevel@tonic-gate CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1903*0Sstevel@tonic-gate 1904*0Sstevel@tonic-gate /* 1905*0Sstevel@tonic-gate * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1906*0Sstevel@tonic-gate */ 1907*0Sstevel@tonic-gate tbr_wr_addr_inited = 1; 1908*0Sstevel@tonic-gate 1909*0Sstevel@tonic-gate /* 1910*0Sstevel@tonic-gate * Initialize VM system, and map kernel address space. 1911*0Sstevel@tonic-gate */ 1912*0Sstevel@tonic-gate kvm_init(); 1913*0Sstevel@tonic-gate 1914*0Sstevel@tonic-gate /* 1915*0Sstevel@tonic-gate * XXX4U: previously, we initialized and turned on 1916*0Sstevel@tonic-gate * the caches at this point. But of course we have 1917*0Sstevel@tonic-gate * nothing to do, as the prom has already done this 1918*0Sstevel@tonic-gate * for us -- main memory must be E$able at all times. 1919*0Sstevel@tonic-gate */ 1920*0Sstevel@tonic-gate 1921*0Sstevel@tonic-gate /* 1922*0Sstevel@tonic-gate * If the following is true, someone has patched 1923*0Sstevel@tonic-gate * phsymem to be less than the number of pages that 1924*0Sstevel@tonic-gate * the system actually has. Remove pages until system 1925*0Sstevel@tonic-gate * memory is limited to the requested amount. Since we 1926*0Sstevel@tonic-gate * have allocated page structures for all pages, we 1927*0Sstevel@tonic-gate * correct the amount of memory we want to remove 1928*0Sstevel@tonic-gate * by the size of the memory used to hold page structures 1929*0Sstevel@tonic-gate * for the non-used pages. 1930*0Sstevel@tonic-gate */ 1931*0Sstevel@tonic-gate if (physmem < npages) { 1932*0Sstevel@tonic-gate pgcnt_t diff, off; 1933*0Sstevel@tonic-gate struct page *pp; 1934*0Sstevel@tonic-gate struct seg kseg; 1935*0Sstevel@tonic-gate 1936*0Sstevel@tonic-gate cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1937*0Sstevel@tonic-gate 1938*0Sstevel@tonic-gate off = 0; 1939*0Sstevel@tonic-gate diff = npages - physmem; 1940*0Sstevel@tonic-gate diff -= mmu_btopr(diff * sizeof (struct page)); 1941*0Sstevel@tonic-gate kseg.s_as = &kas; 1942*0Sstevel@tonic-gate while (diff--) { 1943*0Sstevel@tonic-gate pp = page_create_va(&unused_pages_vp, (offset_t)off, 1944*0Sstevel@tonic-gate MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1945*0Sstevel@tonic-gate &kseg, (caddr_t)off); 1946*0Sstevel@tonic-gate if (pp == NULL) 1947*0Sstevel@tonic-gate cmn_err(CE_PANIC, "limited physmem too much!"); 1948*0Sstevel@tonic-gate page_io_unlock(pp); 1949*0Sstevel@tonic-gate page_downgrade(pp); 1950*0Sstevel@tonic-gate availrmem--; 1951*0Sstevel@tonic-gate off += MMU_PAGESIZE; 1952*0Sstevel@tonic-gate } 1953*0Sstevel@tonic-gate } 1954*0Sstevel@tonic-gate 1955*0Sstevel@tonic-gate /* 1956*0Sstevel@tonic-gate * When printing memory, show the total as physmem less 1957*0Sstevel@tonic-gate * that stolen by a debugger. 1958*0Sstevel@tonic-gate */ 1959*0Sstevel@tonic-gate cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1960*0Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1961*0Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1962*0Sstevel@tonic-gate 1963*0Sstevel@tonic-gate avmem = (uint64_t)freemem << PAGESHIFT; 1964*0Sstevel@tonic-gate cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1965*0Sstevel@tonic-gate 1966*0Sstevel@tonic-gate /* 1967*0Sstevel@tonic-gate * Perform platform specific freelist processing 1968*0Sstevel@tonic-gate */ 1969*0Sstevel@tonic-gate if (&plat_freelist_process) { 1970*0Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 1971*0Sstevel@tonic-gate if (mem_node_config[mnode].exists) 1972*0Sstevel@tonic-gate plat_freelist_process(mnode); 1973*0Sstevel@tonic-gate } 1974*0Sstevel@tonic-gate 1975*0Sstevel@tonic-gate /* 1976*0Sstevel@tonic-gate * Initialize the segkp segment type. We position it 1977*0Sstevel@tonic-gate * after the configured tables and buffers (whose end 1978*0Sstevel@tonic-gate * is given by econtig) and before V_WKBASE_ADDR. 1979*0Sstevel@tonic-gate * Also in this area is segkmap (size SEGMAPSIZE). 1980*0Sstevel@tonic-gate */ 1981*0Sstevel@tonic-gate 1982*0Sstevel@tonic-gate /* XXX - cache alignment? */ 1983*0Sstevel@tonic-gate va = (caddr_t)SEGKPBASE; 1984*0Sstevel@tonic-gate ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1985*0Sstevel@tonic-gate 1986*0Sstevel@tonic-gate max_phys_segkp = (physmem * 2); 1987*0Sstevel@tonic-gate 1988*0Sstevel@tonic-gate if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1989*0Sstevel@tonic-gate segkpsize = btop(SEGKPDEFSIZE); 1990*0Sstevel@tonic-gate cmn_err(CE_WARN, "Illegal value for segkpsize. " 1991*0Sstevel@tonic-gate "segkpsize has been reset to %ld pages", segkpsize); 1992*0Sstevel@tonic-gate } 1993*0Sstevel@tonic-gate 1994*0Sstevel@tonic-gate i = ptob(MIN(segkpsize, max_phys_segkp)); 1995*0Sstevel@tonic-gate 1996*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 1997*0Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkp) < 0) 1998*0Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1999*0Sstevel@tonic-gate if (segkp_create(segkp) != 0) 2000*0Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: segkp_create failed"); 2001*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 2002*0Sstevel@tonic-gate 2003*0Sstevel@tonic-gate /* 2004*0Sstevel@tonic-gate * kpm segment 2005*0Sstevel@tonic-gate */ 2006*0Sstevel@tonic-gate segmap_kpm = kpm_enable && 2007*0Sstevel@tonic-gate segmap_kpm && PAGESIZE == MAXBSIZE; 2008*0Sstevel@tonic-gate 2009*0Sstevel@tonic-gate if (kpm_enable) { 2010*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 2011*0Sstevel@tonic-gate 2012*0Sstevel@tonic-gate /* 2013*0Sstevel@tonic-gate * The segkpm virtual range range is larger than the 2014*0Sstevel@tonic-gate * actual physical memory size and also covers gaps in 2015*0Sstevel@tonic-gate * the physical address range for the following reasons: 2016*0Sstevel@tonic-gate * . keep conversion between segkpm and physical addresses 2017*0Sstevel@tonic-gate * simple, cheap and unambiguous. 2018*0Sstevel@tonic-gate * . avoid extension/shrink of the the segkpm in case of DR. 2019*0Sstevel@tonic-gate * . avoid complexity for handling of virtual addressed 2020*0Sstevel@tonic-gate * caches, segkpm and the regular mapping scheme must be 2021*0Sstevel@tonic-gate * kept in sync wrt. the virtual color of mapped pages. 2022*0Sstevel@tonic-gate * Any accesses to virtual segkpm ranges not backed by 2023*0Sstevel@tonic-gate * physical memory will fall through the memseg pfn hash 2024*0Sstevel@tonic-gate * and will be handled in segkpm_fault. 2025*0Sstevel@tonic-gate * Additional kpm_size spaces needed for vac alias prevention. 2026*0Sstevel@tonic-gate */ 2027*0Sstevel@tonic-gate if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2028*0Sstevel@tonic-gate segkpm) < 0) 2029*0Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkpm"); 2030*0Sstevel@tonic-gate 2031*0Sstevel@tonic-gate b.prot = PROT_READ | PROT_WRITE; 2032*0Sstevel@tonic-gate b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2033*0Sstevel@tonic-gate 2034*0Sstevel@tonic-gate if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2035*0Sstevel@tonic-gate panic("segkpm_create segkpm"); 2036*0Sstevel@tonic-gate 2037*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 2038*0Sstevel@tonic-gate } 2039*0Sstevel@tonic-gate 2040*0Sstevel@tonic-gate /* 2041*0Sstevel@tonic-gate * Now create generic mapping segment. This mapping 2042*0Sstevel@tonic-gate * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2043*0Sstevel@tonic-gate * virtual address is greater than the amount of free 2044*0Sstevel@tonic-gate * memory that is available, then we trim back the 2045*0Sstevel@tonic-gate * segment size to that amount 2046*0Sstevel@tonic-gate */ 2047*0Sstevel@tonic-gate va = (caddr_t)SEGMAPBASE; 2048*0Sstevel@tonic-gate 2049*0Sstevel@tonic-gate /* 2050*0Sstevel@tonic-gate * 1201049: segkmap base address must be MAXBSIZE aligned 2051*0Sstevel@tonic-gate */ 2052*0Sstevel@tonic-gate ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2053*0Sstevel@tonic-gate 2054*0Sstevel@tonic-gate /* 2055*0Sstevel@tonic-gate * Set size of segmap to percentage of freemem at boot, 2056*0Sstevel@tonic-gate * but stay within the allowable range 2057*0Sstevel@tonic-gate * Note we take percentage before converting from pages 2058*0Sstevel@tonic-gate * to bytes to avoid an overflow on 32-bit kernels. 2059*0Sstevel@tonic-gate */ 2060*0Sstevel@tonic-gate i = mmu_ptob((freemem * segmap_percent) / 100); 2061*0Sstevel@tonic-gate 2062*0Sstevel@tonic-gate if (i < MINMAPSIZE) 2063*0Sstevel@tonic-gate i = MINMAPSIZE; 2064*0Sstevel@tonic-gate 2065*0Sstevel@tonic-gate if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2066*0Sstevel@tonic-gate i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2067*0Sstevel@tonic-gate 2068*0Sstevel@tonic-gate i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2069*0Sstevel@tonic-gate 2070*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 2071*0Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkmap) < 0) 2072*0Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkmap"); 2073*0Sstevel@tonic-gate 2074*0Sstevel@tonic-gate a.prot = PROT_READ | PROT_WRITE; 2075*0Sstevel@tonic-gate a.shmsize = shm_alignment; 2076*0Sstevel@tonic-gate a.nfreelist = 0; /* use segmap driver defaults */ 2077*0Sstevel@tonic-gate 2078*0Sstevel@tonic-gate if (segmap_create(segkmap, (caddr_t)&a) != 0) 2079*0Sstevel@tonic-gate panic("segmap_create segkmap"); 2080*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 2081*0Sstevel@tonic-gate 2082*0Sstevel@tonic-gate segdev_init(); 2083*0Sstevel@tonic-gate } 2084*0Sstevel@tonic-gate 2085*0Sstevel@tonic-gate static void 2086*0Sstevel@tonic-gate startup_end(void) 2087*0Sstevel@tonic-gate { 2088*0Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end) 2089*0Sstevel@tonic-gate panic("memlist overflow 2"); 2090*0Sstevel@tonic-gate memlist_free_block((caddr_t)memlist, 2091*0Sstevel@tonic-gate ((caddr_t)memlist_end - (caddr_t)memlist)); 2092*0Sstevel@tonic-gate memlist = NULL; 2093*0Sstevel@tonic-gate 2094*0Sstevel@tonic-gate /* enable page_relocation since OBP is now done */ 2095*0Sstevel@tonic-gate page_relocate_ready = 1; 2096*0Sstevel@tonic-gate 2097*0Sstevel@tonic-gate /* 2098*0Sstevel@tonic-gate * Perform tasks that get done after most of the VM 2099*0Sstevel@tonic-gate * initialization has been done but before the clock 2100*0Sstevel@tonic-gate * and other devices get started. 2101*0Sstevel@tonic-gate */ 2102*0Sstevel@tonic-gate kern_setup1(); 2103*0Sstevel@tonic-gate 2104*0Sstevel@tonic-gate /* 2105*0Sstevel@tonic-gate * Intialize the VM arenas for allocating physically 2106*0Sstevel@tonic-gate * contiguus memory chunk for interrupt queues snd 2107*0Sstevel@tonic-gate * allocate/register boot cpu's queues, if any and 2108*0Sstevel@tonic-gate * allocate dump buffer for sun4v systems to store 2109*0Sstevel@tonic-gate * extra crash information during crash dump 2110*0Sstevel@tonic-gate */ 2111*0Sstevel@tonic-gate contig_mem_init(); 2112*0Sstevel@tonic-gate mach_descrip_init(); 2113*0Sstevel@tonic-gate cpu_intrq_setup(CPU); 2114*0Sstevel@tonic-gate cpu_intrq_register(CPU); 2115*0Sstevel@tonic-gate mach_dump_buffer_init(); 2116*0Sstevel@tonic-gate 2117*0Sstevel@tonic-gate /* 2118*0Sstevel@tonic-gate * Initialize interrupt related stuff 2119*0Sstevel@tonic-gate */ 2120*0Sstevel@tonic-gate init_intr_threads(CPU); 2121*0Sstevel@tonic-gate 2122*0Sstevel@tonic-gate (void) splzs(); /* allow hi clock ints but not zs */ 2123*0Sstevel@tonic-gate 2124*0Sstevel@tonic-gate /* 2125*0Sstevel@tonic-gate * Initialize errors. 2126*0Sstevel@tonic-gate */ 2127*0Sstevel@tonic-gate error_init(); 2128*0Sstevel@tonic-gate 2129*0Sstevel@tonic-gate /* 2130*0Sstevel@tonic-gate * Note that we may have already used kernel bcopy before this 2131*0Sstevel@tonic-gate * point - but if you really care about this, adb the use_hw_* 2132*0Sstevel@tonic-gate * variables to 0 before rebooting. 2133*0Sstevel@tonic-gate */ 2134*0Sstevel@tonic-gate mach_hw_copy_limit(); 2135*0Sstevel@tonic-gate 2136*0Sstevel@tonic-gate /* 2137*0Sstevel@tonic-gate * Install the "real" preemption guards before DDI services 2138*0Sstevel@tonic-gate * are available. 2139*0Sstevel@tonic-gate */ 2140*0Sstevel@tonic-gate (void) prom_set_preprom(kern_preprom); 2141*0Sstevel@tonic-gate (void) prom_set_postprom(kern_postprom); 2142*0Sstevel@tonic-gate CPU->cpu_m.mutex_ready = 1; 2143*0Sstevel@tonic-gate 2144*0Sstevel@tonic-gate /* 2145*0Sstevel@tonic-gate * Initialize segnf (kernel support for non-faulting loads). 2146*0Sstevel@tonic-gate */ 2147*0Sstevel@tonic-gate segnf_init(); 2148*0Sstevel@tonic-gate 2149*0Sstevel@tonic-gate /* 2150*0Sstevel@tonic-gate * Configure the root devinfo node. 2151*0Sstevel@tonic-gate */ 2152*0Sstevel@tonic-gate configure(); /* set up devices */ 2153*0Sstevel@tonic-gate mach_cpu_halt_idle(); 2154*0Sstevel@tonic-gate } 2155*0Sstevel@tonic-gate 2156*0Sstevel@tonic-gate 2157*0Sstevel@tonic-gate void 2158*0Sstevel@tonic-gate post_startup(void) 2159*0Sstevel@tonic-gate { 2160*0Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 2161*0Sstevel@tonic-gate extern void init_ptl1_thread(void); 2162*0Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 2163*0Sstevel@tonic-gate extern void abort_sequence_init(void); 2164*0Sstevel@tonic-gate 2165*0Sstevel@tonic-gate /* 2166*0Sstevel@tonic-gate * Set the system wide, processor-specific flags to be passed 2167*0Sstevel@tonic-gate * to userland via the aux vector for performance hints and 2168*0Sstevel@tonic-gate * instruction set extensions. 2169*0Sstevel@tonic-gate */ 2170*0Sstevel@tonic-gate bind_hwcap(); 2171*0Sstevel@tonic-gate 2172*0Sstevel@tonic-gate /* 2173*0Sstevel@tonic-gate * Startup memory scrubber (if any) 2174*0Sstevel@tonic-gate */ 2175*0Sstevel@tonic-gate mach_memscrub(); 2176*0Sstevel@tonic-gate 2177*0Sstevel@tonic-gate /* 2178*0Sstevel@tonic-gate * Allocate soft interrupt to handle abort sequence. 2179*0Sstevel@tonic-gate */ 2180*0Sstevel@tonic-gate abort_sequence_init(); 2181*0Sstevel@tonic-gate 2182*0Sstevel@tonic-gate /* 2183*0Sstevel@tonic-gate * Configure the rest of the system. 2184*0Sstevel@tonic-gate * Perform forceloading tasks for /etc/system. 2185*0Sstevel@tonic-gate */ 2186*0Sstevel@tonic-gate (void) mod_sysctl(SYS_FORCELOAD, NULL); 2187*0Sstevel@tonic-gate /* 2188*0Sstevel@tonic-gate * ON4.0: Force /proc module in until clock interrupt handle fixed 2189*0Sstevel@tonic-gate * ON4.0: This must be fixed or restated in /etc/systems. 2190*0Sstevel@tonic-gate */ 2191*0Sstevel@tonic-gate (void) modload("fs", "procfs"); 2192*0Sstevel@tonic-gate 2193*0Sstevel@tonic-gate if (&load_platform_drivers) 2194*0Sstevel@tonic-gate load_platform_drivers(); 2195*0Sstevel@tonic-gate 2196*0Sstevel@tonic-gate /* load vis simulation module, if we are running w/fpu off */ 2197*0Sstevel@tonic-gate if (!fpu_exists) { 2198*0Sstevel@tonic-gate if (modload("misc", "vis") == -1) 2199*0Sstevel@tonic-gate halt("Can't load vis"); 2200*0Sstevel@tonic-gate } 2201*0Sstevel@tonic-gate 2202*0Sstevel@tonic-gate mach_fpras(); 2203*0Sstevel@tonic-gate 2204*0Sstevel@tonic-gate maxmem = freemem; 2205*0Sstevel@tonic-gate 2206*0Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 2207*0Sstevel@tonic-gate init_ptl1_thread(); 2208*0Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 2209*0Sstevel@tonic-gate } 2210*0Sstevel@tonic-gate 2211*0Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG 2212*0Sstevel@tonic-gate int ptl1_panic_test = 0; 2213*0Sstevel@tonic-gate int ptl1_panic_xc_one_test = 0; 2214*0Sstevel@tonic-gate int ptl1_panic_xc_all_test = 0; 2215*0Sstevel@tonic-gate int ptl1_panic_xt_one_test = 0; 2216*0Sstevel@tonic-gate int ptl1_panic_xt_all_test = 0; 2217*0Sstevel@tonic-gate kthread_id_t ptl1_thread_p = NULL; 2218*0Sstevel@tonic-gate kcondvar_t ptl1_cv; 2219*0Sstevel@tonic-gate kmutex_t ptl1_mutex; 2220*0Sstevel@tonic-gate int ptl1_recurse_count_threshold = 0x40; 2221*0Sstevel@tonic-gate int ptl1_recurse_trap_threshold = 0x3d; 2222*0Sstevel@tonic-gate extern void ptl1_recurse(int, int); 2223*0Sstevel@tonic-gate extern void ptl1_panic_xt(int, int); 2224*0Sstevel@tonic-gate 2225*0Sstevel@tonic-gate /* 2226*0Sstevel@tonic-gate * Called once per second by timeout() to wake up 2227*0Sstevel@tonic-gate * the ptl1_panic thread to see if it should cause 2228*0Sstevel@tonic-gate * a trap to the ptl1_panic() code. 2229*0Sstevel@tonic-gate */ 2230*0Sstevel@tonic-gate /* ARGSUSED */ 2231*0Sstevel@tonic-gate static void 2232*0Sstevel@tonic-gate ptl1_wakeup(void *arg) 2233*0Sstevel@tonic-gate { 2234*0Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 2235*0Sstevel@tonic-gate cv_signal(&ptl1_cv); 2236*0Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 2237*0Sstevel@tonic-gate } 2238*0Sstevel@tonic-gate 2239*0Sstevel@tonic-gate /* 2240*0Sstevel@tonic-gate * ptl1_panic cross call function: 2241*0Sstevel@tonic-gate * Needed because xc_one() and xc_some() can pass 2242*0Sstevel@tonic-gate * 64 bit args but ptl1_recurse() expects ints. 2243*0Sstevel@tonic-gate */ 2244*0Sstevel@tonic-gate static void 2245*0Sstevel@tonic-gate ptl1_panic_xc(void) 2246*0Sstevel@tonic-gate { 2247*0Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 2248*0Sstevel@tonic-gate ptl1_recurse_trap_threshold); 2249*0Sstevel@tonic-gate } 2250*0Sstevel@tonic-gate 2251*0Sstevel@tonic-gate /* 2252*0Sstevel@tonic-gate * The ptl1 thread waits for a global flag to be set 2253*0Sstevel@tonic-gate * and uses the recurse thresholds to set the stack depth 2254*0Sstevel@tonic-gate * to cause a ptl1_panic() directly via a call to ptl1_recurse 2255*0Sstevel@tonic-gate * or indirectly via the cross call and cross trap functions. 2256*0Sstevel@tonic-gate * 2257*0Sstevel@tonic-gate * This is useful testing stack overflows and normal 2258*0Sstevel@tonic-gate * ptl1_panic() states with a know stack frame. 2259*0Sstevel@tonic-gate * 2260*0Sstevel@tonic-gate * ptl1_recurse() is an asm function in ptl1_panic.s that 2261*0Sstevel@tonic-gate * sets the {In, Local, Out, and Global} registers to a 2262*0Sstevel@tonic-gate * know state on the stack and just prior to causing a 2263*0Sstevel@tonic-gate * test ptl1_panic trap. 2264*0Sstevel@tonic-gate */ 2265*0Sstevel@tonic-gate static void 2266*0Sstevel@tonic-gate ptl1_thread(void) 2267*0Sstevel@tonic-gate { 2268*0Sstevel@tonic-gate mutex_enter(&ptl1_mutex); 2269*0Sstevel@tonic-gate while (ptl1_thread_p) { 2270*0Sstevel@tonic-gate cpuset_t other_cpus; 2271*0Sstevel@tonic-gate int cpu_id; 2272*0Sstevel@tonic-gate int my_cpu_id; 2273*0Sstevel@tonic-gate int target_cpu_id; 2274*0Sstevel@tonic-gate int target_found; 2275*0Sstevel@tonic-gate 2276*0Sstevel@tonic-gate if (ptl1_panic_test) { 2277*0Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold, 2278*0Sstevel@tonic-gate ptl1_recurse_trap_threshold); 2279*0Sstevel@tonic-gate } 2280*0Sstevel@tonic-gate 2281*0Sstevel@tonic-gate /* 2282*0Sstevel@tonic-gate * Find potential targets for x-call and x-trap, 2283*0Sstevel@tonic-gate * if any exist while preempt is disabled we 2284*0Sstevel@tonic-gate * start a ptl1_panic if requested via a 2285*0Sstevel@tonic-gate * globals. 2286*0Sstevel@tonic-gate */ 2287*0Sstevel@tonic-gate kpreempt_disable(); 2288*0Sstevel@tonic-gate my_cpu_id = CPU->cpu_id; 2289*0Sstevel@tonic-gate other_cpus = cpu_ready_set; 2290*0Sstevel@tonic-gate CPUSET_DEL(other_cpus, CPU->cpu_id); 2291*0Sstevel@tonic-gate target_found = 0; 2292*0Sstevel@tonic-gate if (!CPUSET_ISNULL(other_cpus)) { 2293*0Sstevel@tonic-gate /* 2294*0Sstevel@tonic-gate * Pick the first one 2295*0Sstevel@tonic-gate */ 2296*0Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2297*0Sstevel@tonic-gate if (cpu_id == my_cpu_id) 2298*0Sstevel@tonic-gate continue; 2299*0Sstevel@tonic-gate 2300*0Sstevel@tonic-gate if (CPU_XCALL_READY(cpu_id)) { 2301*0Sstevel@tonic-gate target_cpu_id = cpu_id; 2302*0Sstevel@tonic-gate target_found = 1; 2303*0Sstevel@tonic-gate break; 2304*0Sstevel@tonic-gate } 2305*0Sstevel@tonic-gate } 2306*0Sstevel@tonic-gate ASSERT(target_found); 2307*0Sstevel@tonic-gate 2308*0Sstevel@tonic-gate if (ptl1_panic_xc_one_test) { 2309*0Sstevel@tonic-gate xc_one(target_cpu_id, 2310*0Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 2311*0Sstevel@tonic-gate } 2312*0Sstevel@tonic-gate if (ptl1_panic_xc_all_test) { 2313*0Sstevel@tonic-gate xc_some(other_cpus, 2314*0Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0); 2315*0Sstevel@tonic-gate } 2316*0Sstevel@tonic-gate if (ptl1_panic_xt_one_test) { 2317*0Sstevel@tonic-gate xt_one(target_cpu_id, 2318*0Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 2319*0Sstevel@tonic-gate } 2320*0Sstevel@tonic-gate if (ptl1_panic_xt_all_test) { 2321*0Sstevel@tonic-gate xt_some(other_cpus, 2322*0Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0); 2323*0Sstevel@tonic-gate } 2324*0Sstevel@tonic-gate } 2325*0Sstevel@tonic-gate kpreempt_enable(); 2326*0Sstevel@tonic-gate (void) timeout(ptl1_wakeup, NULL, hz); 2327*0Sstevel@tonic-gate (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2328*0Sstevel@tonic-gate } 2329*0Sstevel@tonic-gate mutex_exit(&ptl1_mutex); 2330*0Sstevel@tonic-gate } 2331*0Sstevel@tonic-gate 2332*0Sstevel@tonic-gate /* 2333*0Sstevel@tonic-gate * Called during early startup to create the ptl1_thread 2334*0Sstevel@tonic-gate */ 2335*0Sstevel@tonic-gate void 2336*0Sstevel@tonic-gate init_ptl1_thread(void) 2337*0Sstevel@tonic-gate { 2338*0Sstevel@tonic-gate ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2339*0Sstevel@tonic-gate &p0, TS_RUN, 0); 2340*0Sstevel@tonic-gate } 2341*0Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */ 2342*0Sstevel@tonic-gate 2343*0Sstevel@tonic-gate 2344*0Sstevel@tonic-gate /* 2345*0Sstevel@tonic-gate * Add to a memory list. 2346*0Sstevel@tonic-gate * start = start of new memory segment 2347*0Sstevel@tonic-gate * len = length of new memory segment in bytes 2348*0Sstevel@tonic-gate * memlistp = pointer to array of available memory segment structures 2349*0Sstevel@tonic-gate * curmemlistp = memory list to which to add segment. 2350*0Sstevel@tonic-gate */ 2351*0Sstevel@tonic-gate static void 2352*0Sstevel@tonic-gate memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2353*0Sstevel@tonic-gate struct memlist **curmemlistp) 2354*0Sstevel@tonic-gate { 2355*0Sstevel@tonic-gate struct memlist *new; 2356*0Sstevel@tonic-gate 2357*0Sstevel@tonic-gate new = *memlistp; 2358*0Sstevel@tonic-gate new->address = start; 2359*0Sstevel@tonic-gate new->size = len; 2360*0Sstevel@tonic-gate *memlistp = new + 1; 2361*0Sstevel@tonic-gate 2362*0Sstevel@tonic-gate memlist_insert(new, curmemlistp); 2363*0Sstevel@tonic-gate } 2364*0Sstevel@tonic-gate 2365*0Sstevel@tonic-gate /* 2366*0Sstevel@tonic-gate * In the case of architectures that support dynamic addition of 2367*0Sstevel@tonic-gate * memory at run-time there are two cases where memsegs need to 2368*0Sstevel@tonic-gate * be initialized and added to the memseg list. 2369*0Sstevel@tonic-gate * 1) memsegs that are constructed at startup. 2370*0Sstevel@tonic-gate * 2) memsegs that are constructed at run-time on 2371*0Sstevel@tonic-gate * hot-plug capable architectures. 2372*0Sstevel@tonic-gate * This code was originally part of the function kphysm_init(). 2373*0Sstevel@tonic-gate */ 2374*0Sstevel@tonic-gate 2375*0Sstevel@tonic-gate static void 2376*0Sstevel@tonic-gate memseg_list_add(struct memseg *memsegp) 2377*0Sstevel@tonic-gate { 2378*0Sstevel@tonic-gate struct memseg **prev_memsegp; 2379*0Sstevel@tonic-gate pgcnt_t num; 2380*0Sstevel@tonic-gate 2381*0Sstevel@tonic-gate /* insert in memseg list, decreasing number of pages order */ 2382*0Sstevel@tonic-gate 2383*0Sstevel@tonic-gate num = MSEG_NPAGES(memsegp); 2384*0Sstevel@tonic-gate 2385*0Sstevel@tonic-gate for (prev_memsegp = &memsegs; *prev_memsegp; 2386*0Sstevel@tonic-gate prev_memsegp = &((*prev_memsegp)->next)) { 2387*0Sstevel@tonic-gate if (num > MSEG_NPAGES(*prev_memsegp)) 2388*0Sstevel@tonic-gate break; 2389*0Sstevel@tonic-gate } 2390*0Sstevel@tonic-gate 2391*0Sstevel@tonic-gate memsegp->next = *prev_memsegp; 2392*0Sstevel@tonic-gate *prev_memsegp = memsegp; 2393*0Sstevel@tonic-gate 2394*0Sstevel@tonic-gate if (kpm_enable) { 2395*0Sstevel@tonic-gate memsegp->nextpa = (memsegp->next) ? 2396*0Sstevel@tonic-gate va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2397*0Sstevel@tonic-gate 2398*0Sstevel@tonic-gate if (prev_memsegp != &memsegs) { 2399*0Sstevel@tonic-gate struct memseg *msp; 2400*0Sstevel@tonic-gate msp = (struct memseg *)((caddr_t)prev_memsegp - 2401*0Sstevel@tonic-gate offsetof(struct memseg, next)); 2402*0Sstevel@tonic-gate msp->nextpa = va_to_pa(memsegp); 2403*0Sstevel@tonic-gate } else { 2404*0Sstevel@tonic-gate memsegspa = va_to_pa(memsegs); 2405*0Sstevel@tonic-gate } 2406*0Sstevel@tonic-gate } 2407*0Sstevel@tonic-gate } 2408*0Sstevel@tonic-gate 2409*0Sstevel@tonic-gate /* 2410*0Sstevel@tonic-gate * PSM add_physmem_cb(). US-II and newer processors have some 2411*0Sstevel@tonic-gate * flavor of the prefetch capability implemented. We exploit 2412*0Sstevel@tonic-gate * this capability for optimum performance. 2413*0Sstevel@tonic-gate */ 2414*0Sstevel@tonic-gate #define PREFETCH_BYTES 64 2415*0Sstevel@tonic-gate 2416*0Sstevel@tonic-gate void 2417*0Sstevel@tonic-gate add_physmem_cb(page_t *pp, pfn_t pnum) 2418*0Sstevel@tonic-gate { 2419*0Sstevel@tonic-gate extern void prefetch_page_w(void *); 2420*0Sstevel@tonic-gate 2421*0Sstevel@tonic-gate pp->p_pagenum = pnum; 2422*0Sstevel@tonic-gate 2423*0Sstevel@tonic-gate /* 2424*0Sstevel@tonic-gate * Prefetch one more page_t into E$. To prevent future 2425*0Sstevel@tonic-gate * mishaps with the sizeof(page_t) changing on us, we 2426*0Sstevel@tonic-gate * catch this on debug kernels if we can't bring in the 2427*0Sstevel@tonic-gate * entire hpage with 2 PREFETCH_BYTES reads. See 2428*0Sstevel@tonic-gate * also, sun4u/cpu/cpu_module.c 2429*0Sstevel@tonic-gate */ 2430*0Sstevel@tonic-gate /*LINTED*/ 2431*0Sstevel@tonic-gate ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2432*0Sstevel@tonic-gate prefetch_page_w((char *)pp); 2433*0Sstevel@tonic-gate } 2434*0Sstevel@tonic-gate 2435*0Sstevel@tonic-gate /* 2436*0Sstevel@tonic-gate * kphysm_init() tackles the problem of initializing physical memory. 2437*0Sstevel@tonic-gate * The old startup made some assumptions about the kernel living in 2438*0Sstevel@tonic-gate * physically contiguous space which is no longer valid. 2439*0Sstevel@tonic-gate */ 2440*0Sstevel@tonic-gate static void 2441*0Sstevel@tonic-gate kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2442*0Sstevel@tonic-gate uintptr_t kpm_pp, pgcnt_t kpm_npages) 2443*0Sstevel@tonic-gate { 2444*0Sstevel@tonic-gate struct memlist *pmem; 2445*0Sstevel@tonic-gate struct memseg *msp; 2446*0Sstevel@tonic-gate pfn_t base; 2447*0Sstevel@tonic-gate pgcnt_t num; 2448*0Sstevel@tonic-gate pfn_t lastseg_pages_end = 0; 2449*0Sstevel@tonic-gate pgcnt_t nelem_used = 0; 2450*0Sstevel@tonic-gate 2451*0Sstevel@tonic-gate ASSERT(page_hash != NULL && page_hashsz != 0); 2452*0Sstevel@tonic-gate 2453*0Sstevel@tonic-gate msp = memsegp; 2454*0Sstevel@tonic-gate for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2455*0Sstevel@tonic-gate 2456*0Sstevel@tonic-gate /* 2457*0Sstevel@tonic-gate * Build the memsegs entry 2458*0Sstevel@tonic-gate */ 2459*0Sstevel@tonic-gate num = btop(pmem->size); 2460*0Sstevel@tonic-gate if (num > npages) 2461*0Sstevel@tonic-gate num = npages; 2462*0Sstevel@tonic-gate npages -= num; 2463*0Sstevel@tonic-gate base = btop(pmem->address); 2464*0Sstevel@tonic-gate 2465*0Sstevel@tonic-gate msp->pages = pp; 2466*0Sstevel@tonic-gate msp->epages = pp + num; 2467*0Sstevel@tonic-gate msp->pages_base = base; 2468*0Sstevel@tonic-gate msp->pages_end = base + num; 2469*0Sstevel@tonic-gate 2470*0Sstevel@tonic-gate if (kpm_enable) { 2471*0Sstevel@tonic-gate pfn_t pbase_a; 2472*0Sstevel@tonic-gate pfn_t pend_a; 2473*0Sstevel@tonic-gate pfn_t prev_pend_a; 2474*0Sstevel@tonic-gate pgcnt_t nelem; 2475*0Sstevel@tonic-gate 2476*0Sstevel@tonic-gate msp->pagespa = va_to_pa(pp); 2477*0Sstevel@tonic-gate msp->epagespa = va_to_pa(pp + num); 2478*0Sstevel@tonic-gate pbase_a = kpmptop(ptokpmp(base)); 2479*0Sstevel@tonic-gate pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2480*0Sstevel@tonic-gate nelem = ptokpmp(pend_a - pbase_a); 2481*0Sstevel@tonic-gate msp->kpm_nkpmpgs = nelem; 2482*0Sstevel@tonic-gate msp->kpm_pbase = pbase_a; 2483*0Sstevel@tonic-gate if (lastseg_pages_end) { 2484*0Sstevel@tonic-gate /* 2485*0Sstevel@tonic-gate * Assume phys_avail is in ascending order 2486*0Sstevel@tonic-gate * of physical addresses. 2487*0Sstevel@tonic-gate */ 2488*0Sstevel@tonic-gate ASSERT(base + num > lastseg_pages_end); 2489*0Sstevel@tonic-gate prev_pend_a = kpmptop( 2490*0Sstevel@tonic-gate ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2491*0Sstevel@tonic-gate 2492*0Sstevel@tonic-gate if (prev_pend_a > pbase_a) { 2493*0Sstevel@tonic-gate /* 2494*0Sstevel@tonic-gate * Overlap, more than one memseg may 2495*0Sstevel@tonic-gate * point to the same kpm_page range. 2496*0Sstevel@tonic-gate */ 2497*0Sstevel@tonic-gate if (kpm_smallpages == 0) { 2498*0Sstevel@tonic-gate msp->kpm_pages = 2499*0Sstevel@tonic-gate (kpm_page_t *)kpm_pp - 1; 2500*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2501*0Sstevel@tonic-gate ((kpm_page_t *)kpm_pp 2502*0Sstevel@tonic-gate + nelem - 1); 2503*0Sstevel@tonic-gate } else { 2504*0Sstevel@tonic-gate msp->kpm_spages = 2505*0Sstevel@tonic-gate (kpm_spage_t *)kpm_pp - 1; 2506*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2507*0Sstevel@tonic-gate ((kpm_spage_t *)kpm_pp 2508*0Sstevel@tonic-gate + nelem - 1); 2509*0Sstevel@tonic-gate } 2510*0Sstevel@tonic-gate nelem_used += nelem - 1; 2511*0Sstevel@tonic-gate 2512*0Sstevel@tonic-gate } else { 2513*0Sstevel@tonic-gate if (kpm_smallpages == 0) { 2514*0Sstevel@tonic-gate msp->kpm_pages = 2515*0Sstevel@tonic-gate (kpm_page_t *)kpm_pp; 2516*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2517*0Sstevel@tonic-gate ((kpm_page_t *)kpm_pp 2518*0Sstevel@tonic-gate + nelem); 2519*0Sstevel@tonic-gate } else { 2520*0Sstevel@tonic-gate msp->kpm_spages = 2521*0Sstevel@tonic-gate (kpm_spage_t *)kpm_pp; 2522*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2523*0Sstevel@tonic-gate ((kpm_spage_t *) 2524*0Sstevel@tonic-gate kpm_pp + nelem); 2525*0Sstevel@tonic-gate } 2526*0Sstevel@tonic-gate nelem_used += nelem; 2527*0Sstevel@tonic-gate } 2528*0Sstevel@tonic-gate 2529*0Sstevel@tonic-gate } else { 2530*0Sstevel@tonic-gate if (kpm_smallpages == 0) { 2531*0Sstevel@tonic-gate msp->kpm_pages = (kpm_page_t *)kpm_pp; 2532*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2533*0Sstevel@tonic-gate ((kpm_page_t *)kpm_pp + nelem); 2534*0Sstevel@tonic-gate } else { 2535*0Sstevel@tonic-gate msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2536*0Sstevel@tonic-gate kpm_pp = (uintptr_t) 2537*0Sstevel@tonic-gate ((kpm_spage_t *)kpm_pp + nelem); 2538*0Sstevel@tonic-gate } 2539*0Sstevel@tonic-gate nelem_used = nelem; 2540*0Sstevel@tonic-gate } 2541*0Sstevel@tonic-gate 2542*0Sstevel@tonic-gate if (nelem_used > kpm_npages) 2543*0Sstevel@tonic-gate panic("kphysm_init: kpm_pp overflow\n"); 2544*0Sstevel@tonic-gate 2545*0Sstevel@tonic-gate msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2546*0Sstevel@tonic-gate lastseg_pages_end = msp->pages_end; 2547*0Sstevel@tonic-gate } 2548*0Sstevel@tonic-gate 2549*0Sstevel@tonic-gate memseg_list_add(msp); 2550*0Sstevel@tonic-gate 2551*0Sstevel@tonic-gate /* 2552*0Sstevel@tonic-gate * add_physmem() initializes the PSM part of the page 2553*0Sstevel@tonic-gate * struct by calling the PSM back with add_physmem_cb(). 2554*0Sstevel@tonic-gate * In addition it coalesces pages into larger pages as 2555*0Sstevel@tonic-gate * it initializes them. 2556*0Sstevel@tonic-gate */ 2557*0Sstevel@tonic-gate add_physmem(pp, num, base); 2558*0Sstevel@tonic-gate pp += num; 2559*0Sstevel@tonic-gate msp++; 2560*0Sstevel@tonic-gate } 2561*0Sstevel@tonic-gate 2562*0Sstevel@tonic-gate build_pfn_hash(); 2563*0Sstevel@tonic-gate } 2564*0Sstevel@tonic-gate 2565*0Sstevel@tonic-gate /* 2566*0Sstevel@tonic-gate * Kernel VM initialization. 2567*0Sstevel@tonic-gate * Assumptions about kernel address space ordering: 2568*0Sstevel@tonic-gate * (1) gap (user space) 2569*0Sstevel@tonic-gate * (2) kernel text 2570*0Sstevel@tonic-gate * (3) kernel data/bss 2571*0Sstevel@tonic-gate * (4) gap 2572*0Sstevel@tonic-gate * (5) kernel data structures 2573*0Sstevel@tonic-gate * (6) gap 2574*0Sstevel@tonic-gate * (7) debugger (optional) 2575*0Sstevel@tonic-gate * (8) monitor 2576*0Sstevel@tonic-gate * (9) gap (possibly null) 2577*0Sstevel@tonic-gate * (10) dvma 2578*0Sstevel@tonic-gate * (11) devices 2579*0Sstevel@tonic-gate */ 2580*0Sstevel@tonic-gate static void 2581*0Sstevel@tonic-gate kvm_init(void) 2582*0Sstevel@tonic-gate { 2583*0Sstevel@tonic-gate /* 2584*0Sstevel@tonic-gate * Put the kernel segments in kernel address space. 2585*0Sstevel@tonic-gate */ 2586*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 2587*0Sstevel@tonic-gate as_avlinit(&kas); 2588*0Sstevel@tonic-gate 2589*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2590*0Sstevel@tonic-gate (size_t)(e_moddata - KERNELBASE), &ktextseg); 2591*0Sstevel@tonic-gate (void) segkmem_create(&ktextseg); 2592*0Sstevel@tonic-gate 2593*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2594*0Sstevel@tonic-gate (size_t)(MMU_PAGESIZE4M), &ktexthole); 2595*0Sstevel@tonic-gate (void) segkmem_create(&ktexthole); 2596*0Sstevel@tonic-gate 2597*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)valloc_base, 2598*0Sstevel@tonic-gate (size_t)(econtig32 - valloc_base), &kvalloc); 2599*0Sstevel@tonic-gate (void) segkmem_create(&kvalloc); 2600*0Sstevel@tonic-gate 2601*0Sstevel@tonic-gate if (kmem64_base) { 2602*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)kmem64_base, 2603*0Sstevel@tonic-gate (size_t)(kmem64_end - kmem64_base), &kmem64); 2604*0Sstevel@tonic-gate (void) segkmem_create(&kmem64); 2605*0Sstevel@tonic-gate } 2606*0Sstevel@tonic-gate 2607*0Sstevel@tonic-gate /* 2608*0Sstevel@tonic-gate * We're about to map out /boot. This is the beginning of the 2609*0Sstevel@tonic-gate * system resource management transition. We can no longer 2610*0Sstevel@tonic-gate * call into /boot for I/O or memory allocations. 2611*0Sstevel@tonic-gate */ 2612*0Sstevel@tonic-gate (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2613*0Sstevel@tonic-gate (void) segkmem_create(&kvseg); 2614*0Sstevel@tonic-gate hblk_alloc_dynamic = 1; 2615*0Sstevel@tonic-gate 2616*0Sstevel@tonic-gate /* 2617*0Sstevel@tonic-gate * we need to preallocate pages for DR operations before enabling large 2618*0Sstevel@tonic-gate * page kernel heap because of memseg_remap_init() hat_unload() hack. 2619*0Sstevel@tonic-gate */ 2620*0Sstevel@tonic-gate memseg_remap_init(); 2621*0Sstevel@tonic-gate 2622*0Sstevel@tonic-gate /* at this point we are ready to use large page heap */ 2623*0Sstevel@tonic-gate segkmem_heap_lp_init(); 2624*0Sstevel@tonic-gate 2625*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2626*0Sstevel@tonic-gate &kvseg32); 2627*0Sstevel@tonic-gate (void) segkmem_create(&kvseg32); 2628*0Sstevel@tonic-gate 2629*0Sstevel@tonic-gate /* 2630*0Sstevel@tonic-gate * Create a segment for the debugger. 2631*0Sstevel@tonic-gate */ 2632*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2633*0Sstevel@tonic-gate &kdebugseg); 2634*0Sstevel@tonic-gate (void) segkmem_create(&kdebugseg); 2635*0Sstevel@tonic-gate 2636*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 2637*0Sstevel@tonic-gate } 2638*0Sstevel@tonic-gate 2639*0Sstevel@tonic-gate char obp_tte_str[] = 2640*0Sstevel@tonic-gate "h# %x constant MMU_PAGESHIFT " 2641*0Sstevel@tonic-gate "h# %x constant TTE8K " 2642*0Sstevel@tonic-gate "h# %x constant SFHME_SIZE " 2643*0Sstevel@tonic-gate "h# %x constant SFHME_TTE " 2644*0Sstevel@tonic-gate "h# %x constant HMEBLK_TAG " 2645*0Sstevel@tonic-gate "h# %x constant HMEBLK_NEXT " 2646*0Sstevel@tonic-gate "h# %x constant HMEBLK_MISC " 2647*0Sstevel@tonic-gate "h# %x constant HMEBLK_HME1 " 2648*0Sstevel@tonic-gate "h# %x constant NHMENTS " 2649*0Sstevel@tonic-gate "h# %x constant HBLK_SZMASK " 2650*0Sstevel@tonic-gate "h# %x constant HBLK_RANGE_SHIFT " 2651*0Sstevel@tonic-gate "h# %x constant HMEBP_HBLK " 2652*0Sstevel@tonic-gate "h# %x constant HMEBUCKET_SIZE " 2653*0Sstevel@tonic-gate "h# %x constant HTAG_SFMMUPSZ " 2654*0Sstevel@tonic-gate "h# %x constant HTAG_REHASHSZ " 2655*0Sstevel@tonic-gate "h# %x constant mmu_hashcnt " 2656*0Sstevel@tonic-gate "h# %p constant uhme_hash " 2657*0Sstevel@tonic-gate "h# %p constant khme_hash " 2658*0Sstevel@tonic-gate "h# %x constant UHMEHASH_SZ " 2659*0Sstevel@tonic-gate "h# %x constant KHMEHASH_SZ " 2660*0Sstevel@tonic-gate "h# %p constant KHATID " 2661*0Sstevel@tonic-gate "h# %x constant CTX_SIZE " 2662*0Sstevel@tonic-gate "h# %x constant CTX_SFMMU " 2663*0Sstevel@tonic-gate "h# %p constant ctxs " 2664*0Sstevel@tonic-gate "h# %x constant ASI_MEM " 2665*0Sstevel@tonic-gate 2666*0Sstevel@tonic-gate ": PHYS-X@ ( phys -- data ) " 2667*0Sstevel@tonic-gate " ASI_MEM spacex@ " 2668*0Sstevel@tonic-gate "; " 2669*0Sstevel@tonic-gate 2670*0Sstevel@tonic-gate ": PHYS-W@ ( phys -- data ) " 2671*0Sstevel@tonic-gate " ASI_MEM spacew@ " 2672*0Sstevel@tonic-gate "; " 2673*0Sstevel@tonic-gate 2674*0Sstevel@tonic-gate ": PHYS-L@ ( phys -- data ) " 2675*0Sstevel@tonic-gate " ASI_MEM spaceL@ " 2676*0Sstevel@tonic-gate "; " 2677*0Sstevel@tonic-gate 2678*0Sstevel@tonic-gate ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2679*0Sstevel@tonic-gate " 3 * MMU_PAGESHIFT + " 2680*0Sstevel@tonic-gate "; " 2681*0Sstevel@tonic-gate 2682*0Sstevel@tonic-gate ": TTE_IS_VALID ( ttep -- flag ) " 2683*0Sstevel@tonic-gate " PHYS-X@ 0< " 2684*0Sstevel@tonic-gate "; " 2685*0Sstevel@tonic-gate 2686*0Sstevel@tonic-gate ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2687*0Sstevel@tonic-gate " dup TTE8K = if " 2688*0Sstevel@tonic-gate " drop HBLK_RANGE_SHIFT " 2689*0Sstevel@tonic-gate " else " 2690*0Sstevel@tonic-gate " TTE_PAGE_SHIFT " 2691*0Sstevel@tonic-gate " then " 2692*0Sstevel@tonic-gate "; " 2693*0Sstevel@tonic-gate 2694*0Sstevel@tonic-gate ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2695*0Sstevel@tonic-gate " tuck >> swap MMU_PAGESHIFT - << " 2696*0Sstevel@tonic-gate "; " 2697*0Sstevel@tonic-gate 2698*0Sstevel@tonic-gate ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2699*0Sstevel@tonic-gate " >> over xor swap ( hash sfmmup ) " 2700*0Sstevel@tonic-gate " KHATID <> if ( hash ) " 2701*0Sstevel@tonic-gate " UHMEHASH_SZ and ( bucket ) " 2702*0Sstevel@tonic-gate " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2703*0Sstevel@tonic-gate " else ( hash ) " 2704*0Sstevel@tonic-gate " KHMEHASH_SZ and ( bucket ) " 2705*0Sstevel@tonic-gate " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2706*0Sstevel@tonic-gate " then ( hmebp ) " 2707*0Sstevel@tonic-gate "; " 2708*0Sstevel@tonic-gate 2709*0Sstevel@tonic-gate ": HME_HASH_TABLE_SEARCH " 2710*0Sstevel@tonic-gate " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2711*0Sstevel@tonic-gate " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2712*0Sstevel@tonic-gate " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2713*0Sstevel@tonic-gate " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2714*0Sstevel@tonic-gate " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2715*0Sstevel@tonic-gate " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2716*0Sstevel@tonic-gate " else " 2717*0Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 2718*0Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2719*0Sstevel@tonic-gate " then " 2720*0Sstevel@tonic-gate " else " 2721*0Sstevel@tonic-gate " hmeblk_next + phys-x@ false " 2722*0Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2723*0Sstevel@tonic-gate " then " 2724*0Sstevel@tonic-gate " else " 2725*0Sstevel@tonic-gate " true " 2726*0Sstevel@tonic-gate " then " 2727*0Sstevel@tonic-gate " until r> drop " 2728*0Sstevel@tonic-gate "; " 2729*0Sstevel@tonic-gate 2730*0Sstevel@tonic-gate ": CNUM_TO_SFMMUP ( cnum -- sfmmup ) " 2731*0Sstevel@tonic-gate " CTX_SIZE * ctxs + CTX_SFMMU + " 2732*0Sstevel@tonic-gate "x@ " 2733*0Sstevel@tonic-gate "; " 2734*0Sstevel@tonic-gate 2735*0Sstevel@tonic-gate ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2736*0Sstevel@tonic-gate " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2737*0Sstevel@tonic-gate " HTAG_REHASHSZ << or nip ( hblktag ) " 2738*0Sstevel@tonic-gate "; " 2739*0Sstevel@tonic-gate 2740*0Sstevel@tonic-gate ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2741*0Sstevel@tonic-gate " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2742*0Sstevel@tonic-gate " TTE8K = if ( hmeblkp addr ) " 2743*0Sstevel@tonic-gate " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2744*0Sstevel@tonic-gate " else ( hmeblkp addr ) " 2745*0Sstevel@tonic-gate " drop 0 ( hmeblkp 0 ) " 2746*0Sstevel@tonic-gate " then ( hmeblkp hme-index ) " 2747*0Sstevel@tonic-gate " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2748*0Sstevel@tonic-gate " SFHME_TTE + ( ttep ) " 2749*0Sstevel@tonic-gate "; " 2750*0Sstevel@tonic-gate 2751*0Sstevel@tonic-gate ": unix-tte ( addr cnum -- false | tte-data true ) " 2752*0Sstevel@tonic-gate " CNUM_TO_SFMMUP ( addr sfmmup ) " 2753*0Sstevel@tonic-gate " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2754*0Sstevel@tonic-gate " 2dup swap i HME_HASH_SHIFT " 2755*0Sstevel@tonic-gate "( addr sfmmup sfmmup addr hmeshift ) " 2756*0Sstevel@tonic-gate " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2757*0Sstevel@tonic-gate " over i 4 pick " 2758*0Sstevel@tonic-gate "( addr sfmmup hmebp sfmmup rehash addr ) " 2759*0Sstevel@tonic-gate " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2760*0Sstevel@tonic-gate " HME_HASH_TABLE_SEARCH " 2761*0Sstevel@tonic-gate "( addr sfmmup { null | hmeblkp } ) " 2762*0Sstevel@tonic-gate " ?dup if ( addr sfmmup hmeblkp ) " 2763*0Sstevel@tonic-gate " nip swap HBLK_TO_TTEP ( ttep ) " 2764*0Sstevel@tonic-gate " dup TTE_IS_VALID if ( valid-ttep ) " 2765*0Sstevel@tonic-gate " PHYS-X@ true ( tte-data true ) " 2766*0Sstevel@tonic-gate " else ( invalid-tte ) " 2767*0Sstevel@tonic-gate " drop false ( false ) " 2768*0Sstevel@tonic-gate " then ( false | tte-data true ) " 2769*0Sstevel@tonic-gate " unloop exit ( false | tte-data true ) " 2770*0Sstevel@tonic-gate " then ( addr sfmmup ) " 2771*0Sstevel@tonic-gate " loop ( addr sfmmup ) " 2772*0Sstevel@tonic-gate " 2drop false ( false ) " 2773*0Sstevel@tonic-gate "; " 2774*0Sstevel@tonic-gate ; 2775*0Sstevel@tonic-gate 2776*0Sstevel@tonic-gate void 2777*0Sstevel@tonic-gate create_va_to_tte(void) 2778*0Sstevel@tonic-gate { 2779*0Sstevel@tonic-gate char *bp; 2780*0Sstevel@tonic-gate extern int khmehash_num, uhmehash_num; 2781*0Sstevel@tonic-gate extern struct hmehash_bucket *khme_hash, *uhme_hash; 2782*0Sstevel@tonic-gate 2783*0Sstevel@tonic-gate #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2784*0Sstevel@tonic-gate 2785*0Sstevel@tonic-gate bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2786*0Sstevel@tonic-gate 2787*0Sstevel@tonic-gate /* 2788*0Sstevel@tonic-gate * Teach obp how to parse our sw ttes. 2789*0Sstevel@tonic-gate */ 2790*0Sstevel@tonic-gate (void) sprintf(bp, obp_tte_str, 2791*0Sstevel@tonic-gate MMU_PAGESHIFT, 2792*0Sstevel@tonic-gate TTE8K, 2793*0Sstevel@tonic-gate sizeof (struct sf_hment), 2794*0Sstevel@tonic-gate OFFSET(struct sf_hment, hme_tte), 2795*0Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_tag), 2796*0Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_nextpa), 2797*0Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_misc), 2798*0Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_hme), 2799*0Sstevel@tonic-gate NHMENTS, 2800*0Sstevel@tonic-gate HBLK_SZMASK, 2801*0Sstevel@tonic-gate HBLK_RANGE_SHIFT, 2802*0Sstevel@tonic-gate OFFSET(struct hmehash_bucket, hmeh_nextpa), 2803*0Sstevel@tonic-gate sizeof (struct hmehash_bucket), 2804*0Sstevel@tonic-gate HTAG_SFMMUPSZ, 2805*0Sstevel@tonic-gate HTAG_REHASHSZ, 2806*0Sstevel@tonic-gate mmu_hashcnt, 2807*0Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)uhme_hash), 2808*0Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)khme_hash), 2809*0Sstevel@tonic-gate UHMEHASH_SZ, 2810*0Sstevel@tonic-gate KHMEHASH_SZ, 2811*0Sstevel@tonic-gate KHATID, 2812*0Sstevel@tonic-gate sizeof (struct ctx), 2813*0Sstevel@tonic-gate OFFSET(struct ctx, ctx_sfmmu), 2814*0Sstevel@tonic-gate ctxs, 2815*0Sstevel@tonic-gate ASI_MEM); 2816*0Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0); 2817*0Sstevel@tonic-gate 2818*0Sstevel@tonic-gate kobj_free(bp, MMU_PAGESIZE); 2819*0Sstevel@tonic-gate } 2820*0Sstevel@tonic-gate 2821*0Sstevel@tonic-gate void 2822*0Sstevel@tonic-gate install_va_to_tte(void) 2823*0Sstevel@tonic-gate { 2824*0Sstevel@tonic-gate /* 2825*0Sstevel@tonic-gate * advise prom that he can use unix-tte 2826*0Sstevel@tonic-gate */ 2827*0Sstevel@tonic-gate prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2828*0Sstevel@tonic-gate } 2829*0Sstevel@tonic-gate 2830*0Sstevel@tonic-gate 2831*0Sstevel@tonic-gate static char *create_node = 2832*0Sstevel@tonic-gate "root-device " 2833*0Sstevel@tonic-gate "new-device " 2834*0Sstevel@tonic-gate "\" os-io\" device-name " 2835*0Sstevel@tonic-gate ": cb-r/w ( adr,len method$ -- #read/#written ) " 2836*0Sstevel@tonic-gate " 2>r swap 2 2r> ['] $callback catch if " 2837*0Sstevel@tonic-gate " 2drop 3drop 0 " 2838*0Sstevel@tonic-gate " then " 2839*0Sstevel@tonic-gate "; " 2840*0Sstevel@tonic-gate ": read ( adr,len -- #read ) " 2841*0Sstevel@tonic-gate " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2842*0Sstevel@tonic-gate " ( retN ... ret1 N ) " 2843*0Sstevel@tonic-gate " ?dup if " 2844*0Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 2845*0Sstevel@tonic-gate " else " 2846*0Sstevel@tonic-gate " -2 " 2847*0Sstevel@tonic-gate " then l->n " 2848*0Sstevel@tonic-gate "; " 2849*0Sstevel@tonic-gate ": write ( adr,len -- #written ) " 2850*0Sstevel@tonic-gate " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2851*0Sstevel@tonic-gate " ( retN ... ret1 N ) " 2852*0Sstevel@tonic-gate " ?dup if " 2853*0Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> " 2854*0Sstevel@tonic-gate " else " 2855*0Sstevel@tonic-gate " 0 " 2856*0Sstevel@tonic-gate " then " 2857*0Sstevel@tonic-gate "; " 2858*0Sstevel@tonic-gate ": poll-tty ( -- ) ; " 2859*0Sstevel@tonic-gate ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2860*0Sstevel@tonic-gate ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2861*0Sstevel@tonic-gate ": cb-give/take ( $method -- ) " 2862*0Sstevel@tonic-gate " 0 -rot ['] $callback catch ?dup if " 2863*0Sstevel@tonic-gate " >r 2drop 2drop r> throw " 2864*0Sstevel@tonic-gate " else " 2865*0Sstevel@tonic-gate " 0 ?do drop loop " 2866*0Sstevel@tonic-gate " then " 2867*0Sstevel@tonic-gate "; " 2868*0Sstevel@tonic-gate ": give ( -- ) \" exit-input\" cb-give/take ; " 2869*0Sstevel@tonic-gate ": take ( -- ) \" enter-input\" cb-give/take ; " 2870*0Sstevel@tonic-gate ": open ( -- ok? ) true ; " 2871*0Sstevel@tonic-gate ": close ( -- ) ; " 2872*0Sstevel@tonic-gate "finish-device " 2873*0Sstevel@tonic-gate "device-end "; 2874*0Sstevel@tonic-gate 2875*0Sstevel@tonic-gate /* 2876*0Sstevel@tonic-gate * Create the obp input/output node only if the USB keyboard is the 2877*0Sstevel@tonic-gate * standard input device. When the USB software takes over the 2878*0Sstevel@tonic-gate * input device at the time consconfig runs, it will switch OBP's 2879*0Sstevel@tonic-gate * notion of the input device to this node. Whenever the 2880*0Sstevel@tonic-gate * forth user interface is used after this switch, the node will 2881*0Sstevel@tonic-gate * call back into the kernel for console input. 2882*0Sstevel@tonic-gate * 2883*0Sstevel@tonic-gate * This callback mechanism is currently only used when the USB keyboard 2884*0Sstevel@tonic-gate * is the input device. If a serial device such as ttya or 2885*0Sstevel@tonic-gate * a UART with a Type 5 keyboard attached is used, obp takes over the 2886*0Sstevel@tonic-gate * serial device when the system goes to the debugger after the system is 2887*0Sstevel@tonic-gate * booted. This sharing of the relatively simple serial device is difficult 2888*0Sstevel@tonic-gate * but possible. Sharing the USB host controller is impossible due 2889*0Sstevel@tonic-gate * its complexity 2890*0Sstevel@tonic-gate */ 2891*0Sstevel@tonic-gate static void 2892*0Sstevel@tonic-gate startup_create_input_node(void) 2893*0Sstevel@tonic-gate { 2894*0Sstevel@tonic-gate char *stdin_path; 2895*0Sstevel@tonic-gate 2896*0Sstevel@tonic-gate /* 2897*0Sstevel@tonic-gate * If usb_node_debug is set in /etc/system 2898*0Sstevel@tonic-gate * then the user would like to test the callbacks 2899*0Sstevel@tonic-gate * from the input node regardless of whether or 2900*0Sstevel@tonic-gate * not the USB keyboard is the console input. 2901*0Sstevel@tonic-gate * This variable is useful for debugging. 2902*0Sstevel@tonic-gate */ 2903*0Sstevel@tonic-gate if (usb_node_debug) { 2904*0Sstevel@tonic-gate 2905*0Sstevel@tonic-gate prom_interpret(create_node, 0, 0, 0, 0, 0); 2906*0Sstevel@tonic-gate 2907*0Sstevel@tonic-gate return; 2908*0Sstevel@tonic-gate } 2909*0Sstevel@tonic-gate 2910*0Sstevel@tonic-gate /* Obtain the console input device */ 2911*0Sstevel@tonic-gate stdin_path = prom_stdinpath(); 2912*0Sstevel@tonic-gate 2913*0Sstevel@tonic-gate /* 2914*0Sstevel@tonic-gate * If the string "usb" and "keyboard" are in the path 2915*0Sstevel@tonic-gate * then a USB keyboard is the console input device, 2916*0Sstevel@tonic-gate * create the node. 2917*0Sstevel@tonic-gate */ 2918*0Sstevel@tonic-gate if ((strstr(stdin_path, "usb") != 0) && 2919*0Sstevel@tonic-gate (strstr(stdin_path, "keyboard") != 0)) { 2920*0Sstevel@tonic-gate 2921*0Sstevel@tonic-gate prom_interpret(create_node, 0, 0, 0, 0, 0); 2922*0Sstevel@tonic-gate } 2923*0Sstevel@tonic-gate } 2924*0Sstevel@tonic-gate 2925*0Sstevel@tonic-gate 2926*0Sstevel@tonic-gate static void 2927*0Sstevel@tonic-gate do_prom_version_check(void) 2928*0Sstevel@tonic-gate { 2929*0Sstevel@tonic-gate int i; 2930*0Sstevel@tonic-gate dnode_t node; 2931*0Sstevel@tonic-gate char buf[64]; 2932*0Sstevel@tonic-gate static char drev[] = "Down-rev firmware detected%s\n" 2933*0Sstevel@tonic-gate "\tPlease upgrade to the following minimum version:\n" 2934*0Sstevel@tonic-gate "\t\t%s\n"; 2935*0Sstevel@tonic-gate 2936*0Sstevel@tonic-gate i = prom_version_check(buf, sizeof (buf), &node); 2937*0Sstevel@tonic-gate 2938*0Sstevel@tonic-gate if (i == PROM_VER64_OK) 2939*0Sstevel@tonic-gate return; 2940*0Sstevel@tonic-gate 2941*0Sstevel@tonic-gate if (i == PROM_VER64_UPGRADE) { 2942*0Sstevel@tonic-gate cmn_err(CE_WARN, drev, "", buf); 2943*0Sstevel@tonic-gate 2944*0Sstevel@tonic-gate #ifdef DEBUG 2945*0Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to continue */ 2946*0Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2947*0Sstevel@tonic-gate return; 2948*0Sstevel@tonic-gate #else 2949*0Sstevel@tonic-gate halt(0); 2950*0Sstevel@tonic-gate #endif 2951*0Sstevel@tonic-gate } 2952*0Sstevel@tonic-gate 2953*0Sstevel@tonic-gate /* 2954*0Sstevel@tonic-gate * The other possibility is that this is a server running 2955*0Sstevel@tonic-gate * good firmware, but down-rev firmware was detected on at 2956*0Sstevel@tonic-gate * least one other cpu board. We just complain if we see 2957*0Sstevel@tonic-gate * that. 2958*0Sstevel@tonic-gate */ 2959*0Sstevel@tonic-gate cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2960*0Sstevel@tonic-gate } 2961*0Sstevel@tonic-gate 2962*0Sstevel@tonic-gate static void 2963*0Sstevel@tonic-gate kpm_init() 2964*0Sstevel@tonic-gate { 2965*0Sstevel@tonic-gate kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 2966*0Sstevel@tonic-gate kpm_pgsz = 1ull << kpm_pgshft; 2967*0Sstevel@tonic-gate kpm_pgoff = kpm_pgsz - 1; 2968*0Sstevel@tonic-gate kpmp2pshft = kpm_pgshft - PAGESHIFT; 2969*0Sstevel@tonic-gate kpmpnpgs = 1 << kpmp2pshft; 2970*0Sstevel@tonic-gate ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 2971*0Sstevel@tonic-gate } 2972*0Sstevel@tonic-gate 2973*0Sstevel@tonic-gate void 2974*0Sstevel@tonic-gate kpm_npages_setup(int memblocks) 2975*0Sstevel@tonic-gate { 2976*0Sstevel@tonic-gate /* 2977*0Sstevel@tonic-gate * npages can be scattered in a maximum of 'memblocks' 2978*0Sstevel@tonic-gate */ 2979*0Sstevel@tonic-gate kpm_npages = ptokpmpr(npages) + memblocks; 2980*0Sstevel@tonic-gate } 2981*0Sstevel@tonic-gate 2982*0Sstevel@tonic-gate /* 2983*0Sstevel@tonic-gate * Must be defined in platform dependent code. 2984*0Sstevel@tonic-gate */ 2985*0Sstevel@tonic-gate extern caddr_t modtext; 2986*0Sstevel@tonic-gate extern size_t modtext_sz; 2987*0Sstevel@tonic-gate extern caddr_t moddata; 2988*0Sstevel@tonic-gate 2989*0Sstevel@tonic-gate #define HEAPTEXT_ARENA(addr) \ 2990*0Sstevel@tonic-gate ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 2991*0Sstevel@tonic-gate (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 2992*0Sstevel@tonic-gate (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 2993*0Sstevel@tonic-gate 2994*0Sstevel@tonic-gate #define HEAPTEXT_OVERSIZED(addr) \ 2995*0Sstevel@tonic-gate ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 2996*0Sstevel@tonic-gate 2997*0Sstevel@tonic-gate vmem_t *texthole_source[HEAPTEXT_NARENAS]; 2998*0Sstevel@tonic-gate vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 2999*0Sstevel@tonic-gate kmutex_t texthole_lock; 3000*0Sstevel@tonic-gate 3001*0Sstevel@tonic-gate char kern_bootargs[OBP_MAXPATHLEN]; 3002*0Sstevel@tonic-gate 3003*0Sstevel@tonic-gate void 3004*0Sstevel@tonic-gate kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3005*0Sstevel@tonic-gate { 3006*0Sstevel@tonic-gate uintptr_t addr, limit; 3007*0Sstevel@tonic-gate 3008*0Sstevel@tonic-gate addr = HEAPTEXT_BASE; 3009*0Sstevel@tonic-gate limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3010*0Sstevel@tonic-gate 3011*0Sstevel@tonic-gate /* 3012*0Sstevel@tonic-gate * Before we initialize the text_arena, we want to punch holes in the 3013*0Sstevel@tonic-gate * underlying heaptext_arena. This guarantees that for any text 3014*0Sstevel@tonic-gate * address we can find a text hole less than HEAPTEXT_MAPPED away. 3015*0Sstevel@tonic-gate */ 3016*0Sstevel@tonic-gate for (; addr + HEAPTEXT_UNMAPPED <= limit; 3017*0Sstevel@tonic-gate addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3018*0Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3019*0Sstevel@tonic-gate 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3020*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3021*0Sstevel@tonic-gate } 3022*0Sstevel@tonic-gate 3023*0Sstevel@tonic-gate /* 3024*0Sstevel@tonic-gate * Allocate one page at the oversize to break up the text region 3025*0Sstevel@tonic-gate * from the oversized region. 3026*0Sstevel@tonic-gate */ 3027*0Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3028*0Sstevel@tonic-gate (void *)limit, (void *)(limit + PAGESIZE), 3029*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3030*0Sstevel@tonic-gate 3031*0Sstevel@tonic-gate *text_arena = vmem_create("module_text", modtext, modtext_sz, 3032*0Sstevel@tonic-gate sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3033*0Sstevel@tonic-gate heaptext_arena, 0, VM_SLEEP); 3034*0Sstevel@tonic-gate *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3035*0Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3036*0Sstevel@tonic-gate } 3037*0Sstevel@tonic-gate 3038*0Sstevel@tonic-gate caddr_t 3039*0Sstevel@tonic-gate kobj_text_alloc(vmem_t *arena, size_t size) 3040*0Sstevel@tonic-gate { 3041*0Sstevel@tonic-gate caddr_t rval, better; 3042*0Sstevel@tonic-gate 3043*0Sstevel@tonic-gate /* 3044*0Sstevel@tonic-gate * First, try a sleeping allocation. 3045*0Sstevel@tonic-gate */ 3046*0Sstevel@tonic-gate rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3047*0Sstevel@tonic-gate 3048*0Sstevel@tonic-gate if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3049*0Sstevel@tonic-gate return (rval); 3050*0Sstevel@tonic-gate 3051*0Sstevel@tonic-gate /* 3052*0Sstevel@tonic-gate * We didn't get the area that we wanted. We're going to try to do an 3053*0Sstevel@tonic-gate * allocation with explicit constraints. 3054*0Sstevel@tonic-gate */ 3055*0Sstevel@tonic-gate better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3056*0Sstevel@tonic-gate (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3057*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT); 3058*0Sstevel@tonic-gate 3059*0Sstevel@tonic-gate if (better != NULL) { 3060*0Sstevel@tonic-gate /* 3061*0Sstevel@tonic-gate * That worked. Free our first attempt and return. 3062*0Sstevel@tonic-gate */ 3063*0Sstevel@tonic-gate vmem_free(arena, rval, size); 3064*0Sstevel@tonic-gate return (better); 3065*0Sstevel@tonic-gate } 3066*0Sstevel@tonic-gate 3067*0Sstevel@tonic-gate /* 3068*0Sstevel@tonic-gate * That didn't work; we'll have to return our first attempt. 3069*0Sstevel@tonic-gate */ 3070*0Sstevel@tonic-gate return (rval); 3071*0Sstevel@tonic-gate } 3072*0Sstevel@tonic-gate 3073*0Sstevel@tonic-gate caddr_t 3074*0Sstevel@tonic-gate kobj_texthole_alloc(caddr_t addr, size_t size) 3075*0Sstevel@tonic-gate { 3076*0Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 3077*0Sstevel@tonic-gate char c[30]; 3078*0Sstevel@tonic-gate uintptr_t base; 3079*0Sstevel@tonic-gate 3080*0Sstevel@tonic-gate if (HEAPTEXT_OVERSIZED(addr)) { 3081*0Sstevel@tonic-gate /* 3082*0Sstevel@tonic-gate * If this is an oversized allocation, there is no text hole 3083*0Sstevel@tonic-gate * available for it; return NULL. 3084*0Sstevel@tonic-gate */ 3085*0Sstevel@tonic-gate return (NULL); 3086*0Sstevel@tonic-gate } 3087*0Sstevel@tonic-gate 3088*0Sstevel@tonic-gate mutex_enter(&texthole_lock); 3089*0Sstevel@tonic-gate 3090*0Sstevel@tonic-gate if (texthole_arena[arena] == NULL) { 3091*0Sstevel@tonic-gate ASSERT(texthole_source[arena] == NULL); 3092*0Sstevel@tonic-gate 3093*0Sstevel@tonic-gate if (arena == 0) { 3094*0Sstevel@tonic-gate texthole_source[0] = vmem_create("module_text_holesrc", 3095*0Sstevel@tonic-gate (void *)(KERNELBASE + MMU_PAGESIZE4M), 3096*0Sstevel@tonic-gate MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3097*0Sstevel@tonic-gate 0, VM_SLEEP); 3098*0Sstevel@tonic-gate } else { 3099*0Sstevel@tonic-gate base = HEAPTEXT_BASE + 3100*0Sstevel@tonic-gate (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3101*0Sstevel@tonic-gate 3102*0Sstevel@tonic-gate (void) snprintf(c, sizeof (c), 3103*0Sstevel@tonic-gate "heaptext_holesrc_%d", arena); 3104*0Sstevel@tonic-gate 3105*0Sstevel@tonic-gate texthole_source[arena] = vmem_create(c, (void *)base, 3106*0Sstevel@tonic-gate HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3107*0Sstevel@tonic-gate 0, VM_SLEEP); 3108*0Sstevel@tonic-gate } 3109*0Sstevel@tonic-gate 3110*0Sstevel@tonic-gate (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3111*0Sstevel@tonic-gate 3112*0Sstevel@tonic-gate texthole_arena[arena] = vmem_create(c, NULL, 0, 3113*0Sstevel@tonic-gate sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3114*0Sstevel@tonic-gate texthole_source[arena], 0, VM_SLEEP); 3115*0Sstevel@tonic-gate } 3116*0Sstevel@tonic-gate 3117*0Sstevel@tonic-gate mutex_exit(&texthole_lock); 3118*0Sstevel@tonic-gate 3119*0Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 3120*0Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3121*0Sstevel@tonic-gate return (vmem_alloc(texthole_arena[arena], size, 3122*0Sstevel@tonic-gate VM_BESTFIT | VM_NOSLEEP)); 3123*0Sstevel@tonic-gate } 3124*0Sstevel@tonic-gate 3125*0Sstevel@tonic-gate void 3126*0Sstevel@tonic-gate kobj_texthole_free(caddr_t addr, size_t size) 3127*0Sstevel@tonic-gate { 3128*0Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr); 3129*0Sstevel@tonic-gate 3130*0Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3131*0Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL); 3132*0Sstevel@tonic-gate vmem_free(texthole_arena[arena], addr, size); 3133*0Sstevel@tonic-gate } 3134