1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/t_lock.h> 31*0Sstevel@tonic-gate #include <sys/param.h> 32*0Sstevel@tonic-gate #include <sys/sysmacros.h> 33*0Sstevel@tonic-gate #include <sys/signal.h> 34*0Sstevel@tonic-gate #include <sys/systm.h> 35*0Sstevel@tonic-gate #include <sys/user.h> 36*0Sstevel@tonic-gate #include <sys/mman.h> 37*0Sstevel@tonic-gate #include <sys/vm.h> 38*0Sstevel@tonic-gate #include <sys/conf.h> 39*0Sstevel@tonic-gate #include <sys/avintr.h> 40*0Sstevel@tonic-gate #include <sys/autoconf.h> 41*0Sstevel@tonic-gate #include <sys/disp.h> 42*0Sstevel@tonic-gate #include <sys/class.h> 43*0Sstevel@tonic-gate #include <sys/bitmap.h> 44*0Sstevel@tonic-gate 45*0Sstevel@tonic-gate #include <sys/privregs.h> 46*0Sstevel@tonic-gate 47*0Sstevel@tonic-gate #include <sys/proc.h> 48*0Sstevel@tonic-gate #include <sys/buf.h> 49*0Sstevel@tonic-gate #include <sys/kmem.h> 50*0Sstevel@tonic-gate #include <sys/kstat.h> 51*0Sstevel@tonic-gate 52*0Sstevel@tonic-gate #include <sys/reboot.h> 53*0Sstevel@tonic-gate #include <sys/uadmin.h> 54*0Sstevel@tonic-gate 55*0Sstevel@tonic-gate #include <sys/cred.h> 56*0Sstevel@tonic-gate #include <sys/vnode.h> 57*0Sstevel@tonic-gate #include <sys/file.h> 58*0Sstevel@tonic-gate 59*0Sstevel@tonic-gate #include <sys/procfs.h> 60*0Sstevel@tonic-gate #include <sys/acct.h> 61*0Sstevel@tonic-gate 62*0Sstevel@tonic-gate #include <sys/vfs.h> 63*0Sstevel@tonic-gate #include <sys/dnlc.h> 64*0Sstevel@tonic-gate #include <sys/var.h> 65*0Sstevel@tonic-gate #include <sys/cmn_err.h> 66*0Sstevel@tonic-gate #include <sys/utsname.h> 67*0Sstevel@tonic-gate #include <sys/debug.h> 68*0Sstevel@tonic-gate #include <sys/kdi.h> 69*0Sstevel@tonic-gate 70*0Sstevel@tonic-gate #include <sys/dumphdr.h> 71*0Sstevel@tonic-gate #include <sys/bootconf.h> 72*0Sstevel@tonic-gate #include <sys/varargs.h> 73*0Sstevel@tonic-gate #include <sys/promif.h> 74*0Sstevel@tonic-gate #include <sys/prom_emul.h> /* for create_prom_prop */ 75*0Sstevel@tonic-gate #include <sys/modctl.h> /* for "procfs" hack */ 76*0Sstevel@tonic-gate 77*0Sstevel@tonic-gate #include <sys/consdev.h> 78*0Sstevel@tonic-gate #include <sys/frame.h> 79*0Sstevel@tonic-gate 80*0Sstevel@tonic-gate #include <sys/sunddi.h> 81*0Sstevel@tonic-gate #include <sys/sunndi.h> 82*0Sstevel@tonic-gate #include <sys/ndi_impldefs.h> 83*0Sstevel@tonic-gate #include <sys/ddidmareq.h> 84*0Sstevel@tonic-gate #include <sys/psw.h> 85*0Sstevel@tonic-gate #include <sys/regset.h> 86*0Sstevel@tonic-gate #include <sys/clock.h> 87*0Sstevel@tonic-gate #include <sys/pte.h> 88*0Sstevel@tonic-gate #include <sys/mmu.h> 89*0Sstevel@tonic-gate #include <sys/tss.h> 90*0Sstevel@tonic-gate #include <sys/stack.h> 91*0Sstevel@tonic-gate #include <sys/trap.h> 92*0Sstevel@tonic-gate #include <sys/pic.h> 93*0Sstevel@tonic-gate #include <sys/fp.h> 94*0Sstevel@tonic-gate #include <vm/anon.h> 95*0Sstevel@tonic-gate #include <vm/as.h> 96*0Sstevel@tonic-gate #include <vm/page.h> 97*0Sstevel@tonic-gate #include <vm/seg.h> 98*0Sstevel@tonic-gate #include <vm/seg_dev.h> 99*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 100*0Sstevel@tonic-gate #include <vm/seg_kpm.h> 101*0Sstevel@tonic-gate #include <vm/seg_map.h> 102*0Sstevel@tonic-gate #include <vm/seg_vn.h> 103*0Sstevel@tonic-gate #include <vm/seg_kp.h> 104*0Sstevel@tonic-gate #include <sys/memnode.h> 105*0Sstevel@tonic-gate #include <vm/vm_dep.h> 106*0Sstevel@tonic-gate #include <sys/swap.h> 107*0Sstevel@tonic-gate #include <sys/thread.h> 108*0Sstevel@tonic-gate #include <sys/sysconf.h> 109*0Sstevel@tonic-gate #include <sys/vm_machparam.h> 110*0Sstevel@tonic-gate #include <sys/archsystm.h> 111*0Sstevel@tonic-gate #include <sys/machsystm.h> 112*0Sstevel@tonic-gate #include <vm/hat.h> 113*0Sstevel@tonic-gate #include <vm/hat_i86.h> 114*0Sstevel@tonic-gate #include <sys/pmem.h> 115*0Sstevel@tonic-gate #include <sys/instance.h> 116*0Sstevel@tonic-gate #include <sys/smp_impldefs.h> 117*0Sstevel@tonic-gate #include <sys/x86_archext.h> 118*0Sstevel@tonic-gate #include <sys/segments.h> 119*0Sstevel@tonic-gate #include <sys/clconf.h> 120*0Sstevel@tonic-gate #include <sys/kobj.h> 121*0Sstevel@tonic-gate #include <sys/kobj_lex.h> 122*0Sstevel@tonic-gate #include <sys/prom_emul.h> 123*0Sstevel@tonic-gate #include <sys/cpc_impl.h> 124*0Sstevel@tonic-gate #include <sys/chip.h> 125*0Sstevel@tonic-gate #include <sys/x86_archext.h> 126*0Sstevel@tonic-gate 127*0Sstevel@tonic-gate extern void debug_enter(char *); 128*0Sstevel@tonic-gate extern void progressbar_init(void); 129*0Sstevel@tonic-gate extern void progressbar_start(void); 130*0Sstevel@tonic-gate 131*0Sstevel@tonic-gate /* 132*0Sstevel@tonic-gate * XXX make declaration below "static" when drivers no longer use this 133*0Sstevel@tonic-gate * interface. 134*0Sstevel@tonic-gate */ 135*0Sstevel@tonic-gate extern caddr_t p0_va; /* Virtual address for accessing physical page 0 */ 136*0Sstevel@tonic-gate 137*0Sstevel@tonic-gate /* 138*0Sstevel@tonic-gate * segkp 139*0Sstevel@tonic-gate */ 140*0Sstevel@tonic-gate extern int segkp_fromheap; 141*0Sstevel@tonic-gate 142*0Sstevel@tonic-gate static void kvm_init(void); 143*0Sstevel@tonic-gate static void startup_init(void); 144*0Sstevel@tonic-gate static void startup_memlist(void); 145*0Sstevel@tonic-gate static void startup_modules(void); 146*0Sstevel@tonic-gate static void startup_bop_gone(void); 147*0Sstevel@tonic-gate static void startup_vm(void); 148*0Sstevel@tonic-gate static void startup_end(void); 149*0Sstevel@tonic-gate 150*0Sstevel@tonic-gate /* 151*0Sstevel@tonic-gate * Declare these as initialized data so we can patch them. 152*0Sstevel@tonic-gate */ 153*0Sstevel@tonic-gate pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 154*0Sstevel@tonic-gate pgcnt_t obp_pages; /* Memory used by PROM for its text and data */ 155*0Sstevel@tonic-gate 156*0Sstevel@tonic-gate char *kobj_file_buf; 157*0Sstevel@tonic-gate int kobj_file_bufsize; /* set in /etc/system */ 158*0Sstevel@tonic-gate 159*0Sstevel@tonic-gate /* Global variables for MP support. Used in mp_startup */ 160*0Sstevel@tonic-gate caddr_t rm_platter_va; 161*0Sstevel@tonic-gate uint32_t rm_platter_pa; 162*0Sstevel@tonic-gate 163*0Sstevel@tonic-gate /* 164*0Sstevel@tonic-gate * Some CPUs have holes in the middle of the 64-bit virtual address range. 165*0Sstevel@tonic-gate */ 166*0Sstevel@tonic-gate uintptr_t hole_start, hole_end; 167*0Sstevel@tonic-gate 168*0Sstevel@tonic-gate /* 169*0Sstevel@tonic-gate * kpm mapping window 170*0Sstevel@tonic-gate */ 171*0Sstevel@tonic-gate caddr_t kpm_vbase; 172*0Sstevel@tonic-gate size_t kpm_size; 173*0Sstevel@tonic-gate static int kpm_desired = 0; /* Do we want to try to use segkpm? */ 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate /* 176*0Sstevel@tonic-gate * VA range that must be preserved for boot until we release all of its 177*0Sstevel@tonic-gate * mappings. 178*0Sstevel@tonic-gate */ 179*0Sstevel@tonic-gate #if defined(__amd64) 180*0Sstevel@tonic-gate static void *kmem_setaside; 181*0Sstevel@tonic-gate #endif 182*0Sstevel@tonic-gate 183*0Sstevel@tonic-gate /* 184*0Sstevel@tonic-gate * Configuration parameters set at boot time. 185*0Sstevel@tonic-gate */ 186*0Sstevel@tonic-gate 187*0Sstevel@tonic-gate caddr_t econtig; /* end of first block of contiguous kernel */ 188*0Sstevel@tonic-gate 189*0Sstevel@tonic-gate struct bootops *bootops = 0; /* passed in from boot */ 190*0Sstevel@tonic-gate struct bootops **bootopsp; 191*0Sstevel@tonic-gate struct boot_syscalls *sysp; /* passed in from boot */ 192*0Sstevel@tonic-gate 193*0Sstevel@tonic-gate char bootblock_fstype[16]; 194*0Sstevel@tonic-gate 195*0Sstevel@tonic-gate char kern_bootargs[OBP_MAXPATHLEN]; 196*0Sstevel@tonic-gate 197*0Sstevel@tonic-gate /* 198*0Sstevel@tonic-gate * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this 199*0Sstevel@tonic-gate * depends on number of BOP_ALLOC calls made and requested size, memory size 200*0Sstevel@tonic-gate * combination and whether boot.bin memory needs to be freed. 201*0Sstevel@tonic-gate */ 202*0Sstevel@tonic-gate #define POSS_NEW_FRAGMENTS 12 203*0Sstevel@tonic-gate 204*0Sstevel@tonic-gate /* 205*0Sstevel@tonic-gate * VM data structures 206*0Sstevel@tonic-gate */ 207*0Sstevel@tonic-gate long page_hashsz; /* Size of page hash table (power of two) */ 208*0Sstevel@tonic-gate struct page *pp_base; /* Base of initial system page struct array */ 209*0Sstevel@tonic-gate struct page **page_hash; /* Page hash table */ 210*0Sstevel@tonic-gate struct seg ktextseg; /* Segment used for kernel executable image */ 211*0Sstevel@tonic-gate struct seg kvalloc; /* Segment used for "valloc" mapping */ 212*0Sstevel@tonic-gate struct seg kpseg; /* Segment used for pageable kernel virt mem */ 213*0Sstevel@tonic-gate struct seg kmapseg; /* Segment used for generic kernel mappings */ 214*0Sstevel@tonic-gate struct seg kdebugseg; /* Segment used for the kernel debugger */ 215*0Sstevel@tonic-gate 216*0Sstevel@tonic-gate struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 217*0Sstevel@tonic-gate struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 218*0Sstevel@tonic-gate 219*0Sstevel@tonic-gate #if defined(__amd64) 220*0Sstevel@tonic-gate struct seg kvseg_core; /* Segment used for the core heap */ 221*0Sstevel@tonic-gate struct seg kpmseg; /* Segment used for physical mapping */ 222*0Sstevel@tonic-gate struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 223*0Sstevel@tonic-gate #else 224*0Sstevel@tonic-gate struct seg *segkpm = NULL; /* Unused on IA32 */ 225*0Sstevel@tonic-gate #endif 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate caddr_t segkp_base; /* Base address of segkp */ 228*0Sstevel@tonic-gate #if defined(__amd64) 229*0Sstevel@tonic-gate pgcnt_t segkpsize = btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 230*0Sstevel@tonic-gate #else 231*0Sstevel@tonic-gate pgcnt_t segkpsize = 0; 232*0Sstevel@tonic-gate #endif 233*0Sstevel@tonic-gate 234*0Sstevel@tonic-gate struct memseg *memseg_base; 235*0Sstevel@tonic-gate struct vnode unused_pages_vp; 236*0Sstevel@tonic-gate 237*0Sstevel@tonic-gate #define FOURGB 0x100000000LL 238*0Sstevel@tonic-gate 239*0Sstevel@tonic-gate struct memlist *memlist; 240*0Sstevel@tonic-gate 241*0Sstevel@tonic-gate caddr_t s_text; /* start of kernel text segment */ 242*0Sstevel@tonic-gate caddr_t e_text; /* end of kernel text segment */ 243*0Sstevel@tonic-gate caddr_t s_data; /* start of kernel data segment */ 244*0Sstevel@tonic-gate caddr_t e_data; /* end of kernel data segment */ 245*0Sstevel@tonic-gate caddr_t modtext; /* start of loadable module text reserved */ 246*0Sstevel@tonic-gate caddr_t e_modtext; /* end of loadable module text reserved */ 247*0Sstevel@tonic-gate caddr_t moddata; /* start of loadable module data reserved */ 248*0Sstevel@tonic-gate caddr_t e_moddata; /* end of loadable module data reserved */ 249*0Sstevel@tonic-gate 250*0Sstevel@tonic-gate struct memlist *phys_install; /* Total installed physical memory */ 251*0Sstevel@tonic-gate struct memlist *phys_avail; /* Total available physical memory */ 252*0Sstevel@tonic-gate 253*0Sstevel@tonic-gate static void memlist_add(uint64_t, uint64_t, struct memlist *, 254*0Sstevel@tonic-gate struct memlist **); 255*0Sstevel@tonic-gate 256*0Sstevel@tonic-gate /* 257*0Sstevel@tonic-gate * kphysm_init returns the number of pages that were processed 258*0Sstevel@tonic-gate */ 259*0Sstevel@tonic-gate static pgcnt_t kphysm_init(page_t *, struct memseg *, pgcnt_t, pgcnt_t); 260*0Sstevel@tonic-gate 261*0Sstevel@tonic-gate #define IO_PROP_SIZE 64 /* device property size */ 262*0Sstevel@tonic-gate 263*0Sstevel@tonic-gate /* 264*0Sstevel@tonic-gate * a couple useful roundup macros 265*0Sstevel@tonic-gate */ 266*0Sstevel@tonic-gate #define ROUND_UP_PAGE(x) \ 267*0Sstevel@tonic-gate ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE)) 268*0Sstevel@tonic-gate #define ROUND_UP_LPAGE(x) \ 269*0Sstevel@tonic-gate ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1])) 270*0Sstevel@tonic-gate #define ROUND_UP_4MEG(x) \ 271*0Sstevel@tonic-gate ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOURMB_PAGESIZE)) 272*0Sstevel@tonic-gate #define ROUND_UP_TOPLEVEL(x) \ 273*0Sstevel@tonic-gate ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level])) 274*0Sstevel@tonic-gate 275*0Sstevel@tonic-gate /* 276*0Sstevel@tonic-gate * 32-bit Kernel's Virtual memory layout. 277*0Sstevel@tonic-gate * +-----------------------+ 278*0Sstevel@tonic-gate * | psm 1-1 map | 279*0Sstevel@tonic-gate * | exec args area | 280*0Sstevel@tonic-gate * 0xFFC00000 -|-----------------------|- ARGSBASE 281*0Sstevel@tonic-gate * | debugger | 282*0Sstevel@tonic-gate * 0xFF800000 -|-----------------------|- SEGDEBUGBASE 283*0Sstevel@tonic-gate * | Kernel Data | 284*0Sstevel@tonic-gate * 0xFEC00000 -|-----------------------| 285*0Sstevel@tonic-gate * | Kernel Text | 286*0Sstevel@tonic-gate * 0xFE800000 -|-----------------------|- KERNEL_TEXT 287*0Sstevel@tonic-gate * | LUFS sinkhole | 288*0Sstevel@tonic-gate * 0xFE000000 -|-----------------------|- lufs_addr 289*0Sstevel@tonic-gate * --- -|-----------------------|- valloc_base + valloc_sz 290*0Sstevel@tonic-gate * | early pp structures | 291*0Sstevel@tonic-gate * | memsegs, memlists, | 292*0Sstevel@tonic-gate * | page hash, etc. | 293*0Sstevel@tonic-gate * --- -|-----------------------|- valloc_base (floating) 294*0Sstevel@tonic-gate * | ptable_va | 295*0Sstevel@tonic-gate * 0xFDFFE000 -|-----------------------|- ekernelheap, ptable_va 296*0Sstevel@tonic-gate * | | (segkp is an arena under the heap) 297*0Sstevel@tonic-gate * | | 298*0Sstevel@tonic-gate * | kvseg | 299*0Sstevel@tonic-gate * | | 300*0Sstevel@tonic-gate * | | 301*0Sstevel@tonic-gate * --- -|-----------------------|- kernelheap (floating) 302*0Sstevel@tonic-gate * | Segkmap | 303*0Sstevel@tonic-gate * 0xC3002000 -|-----------------------|- segkmap_start (floating) 304*0Sstevel@tonic-gate * | Red Zone | 305*0Sstevel@tonic-gate * 0xC3000000 -|-----------------------|- kernelbase / userlimit (floating) 306*0Sstevel@tonic-gate * | | || 307*0Sstevel@tonic-gate * | Shared objects | \/ 308*0Sstevel@tonic-gate * | | 309*0Sstevel@tonic-gate * : : 310*0Sstevel@tonic-gate * | user data | 311*0Sstevel@tonic-gate * |-----------------------| 312*0Sstevel@tonic-gate * | user text | 313*0Sstevel@tonic-gate * 0x08048000 -|-----------------------| 314*0Sstevel@tonic-gate * | user stack | 315*0Sstevel@tonic-gate * : : 316*0Sstevel@tonic-gate * | invalid | 317*0Sstevel@tonic-gate * 0x00000000 +-----------------------+ 318*0Sstevel@tonic-gate * 319*0Sstevel@tonic-gate * 320*0Sstevel@tonic-gate * 64-bit Kernel's Virtual memory layout. (assuming 64 bit app) 321*0Sstevel@tonic-gate * +-----------------------+ 322*0Sstevel@tonic-gate * | psm 1-1 map | 323*0Sstevel@tonic-gate * | exec args area | 324*0Sstevel@tonic-gate * 0xFFFFFFFF.FFC00000 |-----------------------|- ARGSBASE 325*0Sstevel@tonic-gate * | debugger (?) | 326*0Sstevel@tonic-gate * 0xFFFFFFFF.FF800000 |-----------------------|- SEGDEBUGBASE 327*0Sstevel@tonic-gate * | unused | 328*0Sstevel@tonic-gate * +-----------------------+ 329*0Sstevel@tonic-gate * | Kernel Data | 330*0Sstevel@tonic-gate * 0xFFFFFFFF.FBC00000 |-----------------------| 331*0Sstevel@tonic-gate * | Kernel Text | 332*0Sstevel@tonic-gate * 0xFFFFFFFF.FB800000 |-----------------------|- KERNEL_TEXT 333*0Sstevel@tonic-gate * | LUFS sinkhole | 334*0Sstevel@tonic-gate * 0xFFFFFFFF.FB000000 -|-----------------------|- lufs_addr 335*0Sstevel@tonic-gate * --- |-----------------------|- valloc_base + valloc_sz 336*0Sstevel@tonic-gate * | early pp structures | 337*0Sstevel@tonic-gate * | memsegs, memlists, | 338*0Sstevel@tonic-gate * | page hash, etc. | 339*0Sstevel@tonic-gate * --- |-----------------------|- valloc_base 340*0Sstevel@tonic-gate * | ptable_va | 341*0Sstevel@tonic-gate * --- |-----------------------|- ptable_va 342*0Sstevel@tonic-gate * | Core heap | (used for loadable modules) 343*0Sstevel@tonic-gate * 0xFFFFFFFF.C0000000 |-----------------------|- core_base / ekernelheap 344*0Sstevel@tonic-gate * | Kernel | 345*0Sstevel@tonic-gate * | heap | 346*0Sstevel@tonic-gate * 0xFFFFFXXX.XXX00000 |-----------------------|- kernelheap (floating) 347*0Sstevel@tonic-gate * | segkmap | 348*0Sstevel@tonic-gate * 0xFFFFFXXX.XXX00000 |-----------------------|- segkmap_start (floating) 349*0Sstevel@tonic-gate * | device mappings | 350*0Sstevel@tonic-gate * 0xFFFFFXXX.XXX00000 |-----------------------|- toxic_addr (floating) 351*0Sstevel@tonic-gate * | segkp | 352*0Sstevel@tonic-gate * --- |-----------------------|- segkp_base 353*0Sstevel@tonic-gate * | segkpm | 354*0Sstevel@tonic-gate * 0xFFFFFE00.00000000 |-----------------------| 355*0Sstevel@tonic-gate * | Red Zone | 356*0Sstevel@tonic-gate * 0xFFFFFD80.00000000 |-----------------------|- KERNELBASE 357*0Sstevel@tonic-gate * | User stack |- User space memory 358*0Sstevel@tonic-gate * | | 359*0Sstevel@tonic-gate * | shared objects, etc | (grows downwards) 360*0Sstevel@tonic-gate * : : 361*0Sstevel@tonic-gate * | | 362*0Sstevel@tonic-gate * 0xFFFF8000.00000000 |-----------------------| 363*0Sstevel@tonic-gate * | | 364*0Sstevel@tonic-gate * | VA Hole / unused | 365*0Sstevel@tonic-gate * | | 366*0Sstevel@tonic-gate * 0x00008000.00000000 |-----------------------| 367*0Sstevel@tonic-gate * | | 368*0Sstevel@tonic-gate * | | 369*0Sstevel@tonic-gate * : : 370*0Sstevel@tonic-gate * | user heap | (grows upwards) 371*0Sstevel@tonic-gate * | | 372*0Sstevel@tonic-gate * | user data | 373*0Sstevel@tonic-gate * |-----------------------| 374*0Sstevel@tonic-gate * | user text | 375*0Sstevel@tonic-gate * 0x00000000.04000000 |-----------------------| 376*0Sstevel@tonic-gate * | invalid | 377*0Sstevel@tonic-gate * 0x00000000.00000000 +-----------------------+ 378*0Sstevel@tonic-gate * 379*0Sstevel@tonic-gate * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit 380*0Sstevel@tonic-gate * kernel, except that userlimit is raised to 0xfe000000 381*0Sstevel@tonic-gate * 382*0Sstevel@tonic-gate * Floating values: 383*0Sstevel@tonic-gate * 384*0Sstevel@tonic-gate * valloc_base: start of the kernel's memory management/tracking data 385*0Sstevel@tonic-gate * structures. This region contains page_t structures for the lowest 4GB 386*0Sstevel@tonic-gate * of physical memory, memsegs, memlists, and the page hash. 387*0Sstevel@tonic-gate * 388*0Sstevel@tonic-gate * core_base: start of the kernel's "core" heap area on 64-bit systems. 389*0Sstevel@tonic-gate * This area is intended to be used for global data as well as for module 390*0Sstevel@tonic-gate * text/data that does not fit into the nucleus pages. The core heap is 391*0Sstevel@tonic-gate * restricted to a 2GB range, allowing every address within it to be 392*0Sstevel@tonic-gate * accessed using rip-relative addressing 393*0Sstevel@tonic-gate * 394*0Sstevel@tonic-gate * ekernelheap: end of kernelheap and start of segmap. 395*0Sstevel@tonic-gate * 396*0Sstevel@tonic-gate * kernelheap: start of kernel heap. On 32-bit systems, this starts right 397*0Sstevel@tonic-gate * above a red zone that separates the user's address space from the 398*0Sstevel@tonic-gate * kernel's. On 64-bit systems, it sits above segkp and segkpm. 399*0Sstevel@tonic-gate * 400*0Sstevel@tonic-gate * segkmap_start: start of segmap. The length of segmap can be modified 401*0Sstevel@tonic-gate * by changing segmapsize in /etc/system (preferred) or eeprom (deprecated). 402*0Sstevel@tonic-gate * The default length is 16MB on 32-bit systems and 64MB on 64-bit systems. 403*0Sstevel@tonic-gate * 404*0Sstevel@tonic-gate * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be 405*0Sstevel@tonic-gate * decreased by 2X the size required for page_t. This allows the kernel 406*0Sstevel@tonic-gate * heap to grow in size with physical memory. With sizeof(page_t) == 80 407*0Sstevel@tonic-gate * bytes, the following shows the values of kernelbase and kernel heap 408*0Sstevel@tonic-gate * sizes for different memory configurations (assuming default segmap and 409*0Sstevel@tonic-gate * segkp sizes). 410*0Sstevel@tonic-gate * 411*0Sstevel@tonic-gate * mem size for kernelbase kernel heap 412*0Sstevel@tonic-gate * size page_t's size 413*0Sstevel@tonic-gate * ---- --------- ---------- ----------- 414*0Sstevel@tonic-gate * 1gb 0x01400000 0xd1800000 684MB 415*0Sstevel@tonic-gate * 2gb 0x02800000 0xcf000000 704MB 416*0Sstevel@tonic-gate * 4gb 0x05000000 0xca000000 744MB 417*0Sstevel@tonic-gate * 6gb 0x07800000 0xc5000000 784MB 418*0Sstevel@tonic-gate * 8gb 0x0a000000 0xc0000000 824MB 419*0Sstevel@tonic-gate * 16gb 0x14000000 0xac000000 984MB 420*0Sstevel@tonic-gate * 32gb 0x28000000 0x84000000 1304MB 421*0Sstevel@tonic-gate * 64gb 0x50000000 0x34000000 1944MB (*) 422*0Sstevel@tonic-gate * 423*0Sstevel@tonic-gate * kernelbase is less than the abi minimum of 0xc0000000 for memory 424*0Sstevel@tonic-gate * configurations above 8gb. 425*0Sstevel@tonic-gate * 426*0Sstevel@tonic-gate * (*) support for memory configurations above 32gb will require manual tuning 427*0Sstevel@tonic-gate * of kernelbase to balance out the need of user applications. 428*0Sstevel@tonic-gate */ 429*0Sstevel@tonic-gate 430*0Sstevel@tonic-gate void init_intr_threads(struct cpu *); 431*0Sstevel@tonic-gate 432*0Sstevel@tonic-gate /* 433*0Sstevel@tonic-gate * Dummy spl priority masks 434*0Sstevel@tonic-gate */ 435*0Sstevel@tonic-gate static unsigned char dummy_cpu_pri[MAXIPL + 1] = { 436*0Sstevel@tonic-gate 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 437*0Sstevel@tonic-gate 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf 438*0Sstevel@tonic-gate }; 439*0Sstevel@tonic-gate 440*0Sstevel@tonic-gate /* real-time-clock initialization parameters */ 441*0Sstevel@tonic-gate long gmt_lag; /* offset in seconds of gmt to local time */ 442*0Sstevel@tonic-gate extern long process_rtc_config_file(void); 443*0Sstevel@tonic-gate 444*0Sstevel@tonic-gate char *final_kernelheap; 445*0Sstevel@tonic-gate char *boot_kernelheap; 446*0Sstevel@tonic-gate uintptr_t kernelbase; 447*0Sstevel@tonic-gate uintptr_t eprom_kernelbase; 448*0Sstevel@tonic-gate size_t segmapsize; 449*0Sstevel@tonic-gate static uintptr_t segmap_reserved; 450*0Sstevel@tonic-gate uintptr_t segkmap_start; 451*0Sstevel@tonic-gate int segmapfreelists; 452*0Sstevel@tonic-gate pgcnt_t boot_npages; 453*0Sstevel@tonic-gate pgcnt_t npages; 454*0Sstevel@tonic-gate size_t core_size; /* size of "core" heap */ 455*0Sstevel@tonic-gate uintptr_t core_base; /* base address of "core" heap */ 456*0Sstevel@tonic-gate 457*0Sstevel@tonic-gate /* 458*0Sstevel@tonic-gate * List of bootstrap pages. We mark these as allocated in startup. 459*0Sstevel@tonic-gate * release_bootstrap() will free them when we're completely done with 460*0Sstevel@tonic-gate * the bootstrap. 461*0Sstevel@tonic-gate */ 462*0Sstevel@tonic-gate static page_t *bootpages, *rd_pages; 463*0Sstevel@tonic-gate 464*0Sstevel@tonic-gate struct system_hardware system_hardware; 465*0Sstevel@tonic-gate 466*0Sstevel@tonic-gate /* 467*0Sstevel@tonic-gate * Enable some debugging messages concerning memory usage... 468*0Sstevel@tonic-gate * 469*0Sstevel@tonic-gate * XX64 There should only be one print routine once memlist usage between 470*0Sstevel@tonic-gate * vmx and the kernel is cleaned up and there is a single memlist structure 471*0Sstevel@tonic-gate * shared between kernel and boot. 472*0Sstevel@tonic-gate */ 473*0Sstevel@tonic-gate static void 474*0Sstevel@tonic-gate print_boot_memlist(char *title, struct memlist *mp) 475*0Sstevel@tonic-gate { 476*0Sstevel@tonic-gate prom_printf("MEMLIST: %s:\n", title); 477*0Sstevel@tonic-gate while (mp != NULL) { 478*0Sstevel@tonic-gate prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n", 479*0Sstevel@tonic-gate mp->address, mp->size); 480*0Sstevel@tonic-gate mp = mp->next; 481*0Sstevel@tonic-gate } 482*0Sstevel@tonic-gate } 483*0Sstevel@tonic-gate 484*0Sstevel@tonic-gate static void 485*0Sstevel@tonic-gate print_kernel_memlist(char *title, struct memlist *mp) 486*0Sstevel@tonic-gate { 487*0Sstevel@tonic-gate prom_printf("MEMLIST: %s:\n", title); 488*0Sstevel@tonic-gate while (mp != NULL) { 489*0Sstevel@tonic-gate prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n", 490*0Sstevel@tonic-gate mp->address, mp->size); 491*0Sstevel@tonic-gate mp = mp->next; 492*0Sstevel@tonic-gate } 493*0Sstevel@tonic-gate } 494*0Sstevel@tonic-gate 495*0Sstevel@tonic-gate /* 496*0Sstevel@tonic-gate * XX64 need a comment here.. are these just default values, surely 497*0Sstevel@tonic-gate * we read the "cpuid" type information to figure this out. 498*0Sstevel@tonic-gate */ 499*0Sstevel@tonic-gate int l2cache_sz = 0x80000; 500*0Sstevel@tonic-gate int l2cache_linesz = 0x40; 501*0Sstevel@tonic-gate int l2cache_assoc = 1; 502*0Sstevel@tonic-gate 503*0Sstevel@tonic-gate /* 504*0Sstevel@tonic-gate * on 64 bit we use a predifined VA range for mapping devices in the kernel 505*0Sstevel@tonic-gate * on 32 bit the mappings are intermixed in the heap, so we use a bit map 506*0Sstevel@tonic-gate */ 507*0Sstevel@tonic-gate #ifdef __amd64 508*0Sstevel@tonic-gate 509*0Sstevel@tonic-gate vmem_t *device_arena; 510*0Sstevel@tonic-gate uintptr_t toxic_addr = (uintptr_t)NULL; 511*0Sstevel@tonic-gate size_t toxic_size = 1 * 1024 * 1024 * 1024; /* Sparc uses 1 gig too */ 512*0Sstevel@tonic-gate 513*0Sstevel@tonic-gate #else /* __i386 */ 514*0Sstevel@tonic-gate 515*0Sstevel@tonic-gate ulong_t *toxic_bit_map; /* one bit for each 4k of VA in heap_arena */ 516*0Sstevel@tonic-gate size_t toxic_bit_map_len = 0; /* in bits */ 517*0Sstevel@tonic-gate 518*0Sstevel@tonic-gate #endif /* __i386 */ 519*0Sstevel@tonic-gate 520*0Sstevel@tonic-gate /* 521*0Sstevel@tonic-gate * Simple boot time debug facilities 522*0Sstevel@tonic-gate */ 523*0Sstevel@tonic-gate static char *prm_dbg_str[] = { 524*0Sstevel@tonic-gate "%s:%d: '%s' is 0x%x\n", 525*0Sstevel@tonic-gate "%s:%d: '%s' is 0x%llx\n" 526*0Sstevel@tonic-gate }; 527*0Sstevel@tonic-gate 528*0Sstevel@tonic-gate int prom_debug; 529*0Sstevel@tonic-gate 530*0Sstevel@tonic-gate #define PRM_DEBUG(q) if (prom_debug) \ 531*0Sstevel@tonic-gate prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q); 532*0Sstevel@tonic-gate #define PRM_POINT(q) if (prom_debug) \ 533*0Sstevel@tonic-gate prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q); 534*0Sstevel@tonic-gate 535*0Sstevel@tonic-gate /* 536*0Sstevel@tonic-gate * This structure is used to keep track of the intial allocations 537*0Sstevel@tonic-gate * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to 538*0Sstevel@tonic-gate * be >= the number of ADD_TO_ALLOCATIONS() executed in the code. 539*0Sstevel@tonic-gate */ 540*0Sstevel@tonic-gate #define NUM_ALLOCATIONS 7 541*0Sstevel@tonic-gate int num_allocations = 0; 542*0Sstevel@tonic-gate struct { 543*0Sstevel@tonic-gate void **al_ptr; 544*0Sstevel@tonic-gate size_t al_size; 545*0Sstevel@tonic-gate } allocations[NUM_ALLOCATIONS]; 546*0Sstevel@tonic-gate size_t valloc_sz = 0; 547*0Sstevel@tonic-gate uintptr_t valloc_base; 548*0Sstevel@tonic-gate extern uintptr_t ptable_va; 549*0Sstevel@tonic-gate extern size_t ptable_sz; 550*0Sstevel@tonic-gate 551*0Sstevel@tonic-gate #define ADD_TO_ALLOCATIONS(ptr, size) { \ 552*0Sstevel@tonic-gate size = ROUND_UP_PAGE(size); \ 553*0Sstevel@tonic-gate if (num_allocations == NUM_ALLOCATIONS) \ 554*0Sstevel@tonic-gate panic("too many ADD_TO_ALLOCATIONS()"); \ 555*0Sstevel@tonic-gate allocations[num_allocations].al_ptr = (void**)&ptr; \ 556*0Sstevel@tonic-gate allocations[num_allocations].al_size = size; \ 557*0Sstevel@tonic-gate valloc_sz += size; \ 558*0Sstevel@tonic-gate ++num_allocations; \ 559*0Sstevel@tonic-gate } 560*0Sstevel@tonic-gate 561*0Sstevel@tonic-gate static void 562*0Sstevel@tonic-gate perform_allocations(void) 563*0Sstevel@tonic-gate { 564*0Sstevel@tonic-gate caddr_t mem; 565*0Sstevel@tonic-gate int i; 566*0Sstevel@tonic-gate 567*0Sstevel@tonic-gate mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, BO_NO_ALIGN); 568*0Sstevel@tonic-gate if (mem != (caddr_t)valloc_base) 569*0Sstevel@tonic-gate panic("BOP_ALLOC() failed"); 570*0Sstevel@tonic-gate bzero(mem, valloc_sz); 571*0Sstevel@tonic-gate for (i = 0; i < num_allocations; ++i) { 572*0Sstevel@tonic-gate *allocations[i].al_ptr = (void *)mem; 573*0Sstevel@tonic-gate mem += allocations[i].al_size; 574*0Sstevel@tonic-gate } 575*0Sstevel@tonic-gate } 576*0Sstevel@tonic-gate 577*0Sstevel@tonic-gate /* 578*0Sstevel@tonic-gate * Our world looks like this at startup time. 579*0Sstevel@tonic-gate * 580*0Sstevel@tonic-gate * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data 581*0Sstevel@tonic-gate * at 0xfec00000. On a 64-bit OS, kernel text and data are loaded at 582*0Sstevel@tonic-gate * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively. Those 583*0Sstevel@tonic-gate * addresses are fixed in the binary at link time. 584*0Sstevel@tonic-gate * 585*0Sstevel@tonic-gate * On the text page: 586*0Sstevel@tonic-gate * unix/genunix/krtld/module text loads. 587*0Sstevel@tonic-gate * 588*0Sstevel@tonic-gate * On the data page: 589*0Sstevel@tonic-gate * unix/genunix/krtld/module data loads and space for page_t's. 590*0Sstevel@tonic-gate */ 591*0Sstevel@tonic-gate /* 592*0Sstevel@tonic-gate * Machine-dependent startup code 593*0Sstevel@tonic-gate */ 594*0Sstevel@tonic-gate void 595*0Sstevel@tonic-gate startup(void) 596*0Sstevel@tonic-gate { 597*0Sstevel@tonic-gate extern void startup_bios_disk(); 598*0Sstevel@tonic-gate /* 599*0Sstevel@tonic-gate * Make sure that nobody tries to use sekpm until we have 600*0Sstevel@tonic-gate * initialized it properly. 601*0Sstevel@tonic-gate */ 602*0Sstevel@tonic-gate #if defined(__amd64) 603*0Sstevel@tonic-gate kpm_desired = kpm_enable; 604*0Sstevel@tonic-gate #endif 605*0Sstevel@tonic-gate kpm_enable = 0; 606*0Sstevel@tonic-gate 607*0Sstevel@tonic-gate progressbar_init(); 608*0Sstevel@tonic-gate startup_init(); 609*0Sstevel@tonic-gate startup_memlist(); 610*0Sstevel@tonic-gate startup_modules(); 611*0Sstevel@tonic-gate startup_bios_disk(); 612*0Sstevel@tonic-gate startup_bop_gone(); 613*0Sstevel@tonic-gate startup_vm(); 614*0Sstevel@tonic-gate startup_end(); 615*0Sstevel@tonic-gate progressbar_start(); 616*0Sstevel@tonic-gate } 617*0Sstevel@tonic-gate 618*0Sstevel@tonic-gate static void 619*0Sstevel@tonic-gate startup_init() 620*0Sstevel@tonic-gate { 621*0Sstevel@tonic-gate PRM_POINT("startup_init() starting..."); 622*0Sstevel@tonic-gate 623*0Sstevel@tonic-gate /* 624*0Sstevel@tonic-gate * Complete the extraction of cpuid data 625*0Sstevel@tonic-gate */ 626*0Sstevel@tonic-gate cpuid_pass2(CPU); 627*0Sstevel@tonic-gate 628*0Sstevel@tonic-gate (void) check_boot_version(BOP_GETVERSION(bootops)); 629*0Sstevel@tonic-gate 630*0Sstevel@tonic-gate /* 631*0Sstevel@tonic-gate * Check for prom_debug in boot environment 632*0Sstevel@tonic-gate */ 633*0Sstevel@tonic-gate if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) { 634*0Sstevel@tonic-gate ++prom_debug; 635*0Sstevel@tonic-gate PRM_POINT("prom_debug found in boot enviroment"); 636*0Sstevel@tonic-gate } 637*0Sstevel@tonic-gate 638*0Sstevel@tonic-gate /* 639*0Sstevel@tonic-gate * Collect node, cpu and memory configuration information. 640*0Sstevel@tonic-gate */ 641*0Sstevel@tonic-gate get_system_configuration(); 642*0Sstevel@tonic-gate 643*0Sstevel@tonic-gate /* 644*0Sstevel@tonic-gate * Halt if this is an unsupported processor. 645*0Sstevel@tonic-gate */ 646*0Sstevel@tonic-gate if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) { 647*0Sstevel@tonic-gate printf("\n486 processor (\"%s\") detected.\n", 648*0Sstevel@tonic-gate CPU->cpu_brandstr); 649*0Sstevel@tonic-gate halt("This processor is not supported by this release " 650*0Sstevel@tonic-gate "of Solaris."); 651*0Sstevel@tonic-gate } 652*0Sstevel@tonic-gate 653*0Sstevel@tonic-gate /* 654*0Sstevel@tonic-gate * Set up dummy values till psm spl code installed 655*0Sstevel@tonic-gate */ 656*0Sstevel@tonic-gate CPU->cpu_pri_data = dummy_cpu_pri; 657*0Sstevel@tonic-gate 658*0Sstevel@tonic-gate PRM_POINT("startup_init() done"); 659*0Sstevel@tonic-gate } 660*0Sstevel@tonic-gate 661*0Sstevel@tonic-gate /* 662*0Sstevel@tonic-gate * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie. 663*0Sstevel@tonic-gate * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it 664*0Sstevel@tonic-gate * also filters out physical page zero. There is some reliance on the 665*0Sstevel@tonic-gate * boot loader allocating only a few contiguous physical memory chunks. 666*0Sstevel@tonic-gate */ 667*0Sstevel@tonic-gate static void 668*0Sstevel@tonic-gate avail_filter(uint64_t *addr, uint64_t *size) 669*0Sstevel@tonic-gate { 670*0Sstevel@tonic-gate uintptr_t va; 671*0Sstevel@tonic-gate uintptr_t next_va; 672*0Sstevel@tonic-gate pfn_t pfn; 673*0Sstevel@tonic-gate uint64_t pfn_addr; 674*0Sstevel@tonic-gate uint64_t pfn_eaddr; 675*0Sstevel@tonic-gate uint_t prot; 676*0Sstevel@tonic-gate size_t len; 677*0Sstevel@tonic-gate uint_t change; 678*0Sstevel@tonic-gate 679*0Sstevel@tonic-gate if (prom_debug) 680*0Sstevel@tonic-gate prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n", 681*0Sstevel@tonic-gate *addr, *size); 682*0Sstevel@tonic-gate 683*0Sstevel@tonic-gate /* 684*0Sstevel@tonic-gate * page zero is required for BIOS.. never make it available 685*0Sstevel@tonic-gate */ 686*0Sstevel@tonic-gate if (*addr == 0) { 687*0Sstevel@tonic-gate *addr += MMU_PAGESIZE; 688*0Sstevel@tonic-gate *size -= MMU_PAGESIZE; 689*0Sstevel@tonic-gate } 690*0Sstevel@tonic-gate 691*0Sstevel@tonic-gate /* 692*0Sstevel@tonic-gate * First we trim from the front of the range. Since hat_boot_probe() 693*0Sstevel@tonic-gate * walks ranges in virtual order, but addr/size are physical, we need 694*0Sstevel@tonic-gate * to the list until no changes are seen. This deals with the case 695*0Sstevel@tonic-gate * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w 696*0Sstevel@tonic-gate * but w < v. 697*0Sstevel@tonic-gate */ 698*0Sstevel@tonic-gate do { 699*0Sstevel@tonic-gate change = 0; 700*0Sstevel@tonic-gate for (va = KERNEL_TEXT; 701*0Sstevel@tonic-gate *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0; 702*0Sstevel@tonic-gate va = next_va) { 703*0Sstevel@tonic-gate 704*0Sstevel@tonic-gate next_va = va + len; 705*0Sstevel@tonic-gate pfn_addr = ptob((uint64_t)pfn); 706*0Sstevel@tonic-gate pfn_eaddr = pfn_addr + len; 707*0Sstevel@tonic-gate 708*0Sstevel@tonic-gate if (pfn_addr <= *addr && pfn_eaddr > *addr) { 709*0Sstevel@tonic-gate change = 1; 710*0Sstevel@tonic-gate while (*size > 0 && len > 0) { 711*0Sstevel@tonic-gate *addr += MMU_PAGESIZE; 712*0Sstevel@tonic-gate *size -= MMU_PAGESIZE; 713*0Sstevel@tonic-gate len -= MMU_PAGESIZE; 714*0Sstevel@tonic-gate } 715*0Sstevel@tonic-gate } 716*0Sstevel@tonic-gate } 717*0Sstevel@tonic-gate if (change && prom_debug) 718*0Sstevel@tonic-gate prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n", 719*0Sstevel@tonic-gate *addr, *size); 720*0Sstevel@tonic-gate } while (change); 721*0Sstevel@tonic-gate 722*0Sstevel@tonic-gate /* 723*0Sstevel@tonic-gate * Trim pages from the end of the range. 724*0Sstevel@tonic-gate */ 725*0Sstevel@tonic-gate for (va = KERNEL_TEXT; 726*0Sstevel@tonic-gate *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0; 727*0Sstevel@tonic-gate va = next_va) { 728*0Sstevel@tonic-gate 729*0Sstevel@tonic-gate next_va = va + len; 730*0Sstevel@tonic-gate pfn_addr = ptob((uint64_t)pfn); 731*0Sstevel@tonic-gate 732*0Sstevel@tonic-gate if (pfn_addr >= *addr && pfn_addr < *addr + *size) 733*0Sstevel@tonic-gate *size = pfn_addr - *addr; 734*0Sstevel@tonic-gate } 735*0Sstevel@tonic-gate 736*0Sstevel@tonic-gate if (prom_debug) 737*0Sstevel@tonic-gate prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n", 738*0Sstevel@tonic-gate *addr, *size); 739*0Sstevel@tonic-gate } 740*0Sstevel@tonic-gate 741*0Sstevel@tonic-gate static void 742*0Sstevel@tonic-gate kpm_init() 743*0Sstevel@tonic-gate { 744*0Sstevel@tonic-gate struct segkpm_crargs b; 745*0Sstevel@tonic-gate uintptr_t start, end; 746*0Sstevel@tonic-gate struct memlist *pmem; 747*0Sstevel@tonic-gate 748*0Sstevel@tonic-gate /* 749*0Sstevel@tonic-gate * These variables were all designed for sfmmu in which segkpm is 750*0Sstevel@tonic-gate * mapped using a single pagesize - either 8KB or 4MB. On x86, we 751*0Sstevel@tonic-gate * might use 2+ page sizes on a single machine, so none of these 752*0Sstevel@tonic-gate * variables have a single correct value. They are set up as if we 753*0Sstevel@tonic-gate * always use a 4KB pagesize, which should do no harm. In the long 754*0Sstevel@tonic-gate * run, we should get rid of KPM's assumption that only a single 755*0Sstevel@tonic-gate * pagesize is used. 756*0Sstevel@tonic-gate */ 757*0Sstevel@tonic-gate kpm_pgshft = MMU_PAGESHIFT; 758*0Sstevel@tonic-gate kpm_pgsz = MMU_PAGESIZE; 759*0Sstevel@tonic-gate kpm_pgoff = MMU_PAGEOFFSET; 760*0Sstevel@tonic-gate kpmp2pshft = 0; 761*0Sstevel@tonic-gate kpmpnpgs = 1; 762*0Sstevel@tonic-gate ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 763*0Sstevel@tonic-gate 764*0Sstevel@tonic-gate PRM_POINT("about to create segkpm"); 765*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 766*0Sstevel@tonic-gate 767*0Sstevel@tonic-gate if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0) 768*0Sstevel@tonic-gate panic("cannot attach segkpm"); 769*0Sstevel@tonic-gate 770*0Sstevel@tonic-gate b.prot = PROT_READ | PROT_WRITE; 771*0Sstevel@tonic-gate b.nvcolors = 1; 772*0Sstevel@tonic-gate 773*0Sstevel@tonic-gate if (segkpm_create(segkpm, (caddr_t)&b) != 0) 774*0Sstevel@tonic-gate panic("segkpm_create segkpm"); 775*0Sstevel@tonic-gate 776*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 777*0Sstevel@tonic-gate 778*0Sstevel@tonic-gate /* 779*0Sstevel@tonic-gate * Map each of the memsegs into the kpm segment, coalesing adjacent 780*0Sstevel@tonic-gate * memsegs to allow mapping with the largest possible pages. 781*0Sstevel@tonic-gate */ 782*0Sstevel@tonic-gate pmem = phys_install; 783*0Sstevel@tonic-gate start = pmem->address; 784*0Sstevel@tonic-gate end = start + pmem->size; 785*0Sstevel@tonic-gate for (;;) { 786*0Sstevel@tonic-gate if (pmem == NULL || pmem->address > end) { 787*0Sstevel@tonic-gate hat_devload(kas.a_hat, kpm_vbase + start, 788*0Sstevel@tonic-gate end - start, mmu_btop(start), 789*0Sstevel@tonic-gate PROT_READ | PROT_WRITE, 790*0Sstevel@tonic-gate HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 791*0Sstevel@tonic-gate if (pmem == NULL) 792*0Sstevel@tonic-gate break; 793*0Sstevel@tonic-gate start = pmem->address; 794*0Sstevel@tonic-gate } 795*0Sstevel@tonic-gate end = pmem->address + pmem->size; 796*0Sstevel@tonic-gate pmem = pmem->next; 797*0Sstevel@tonic-gate } 798*0Sstevel@tonic-gate } 799*0Sstevel@tonic-gate 800*0Sstevel@tonic-gate /* 801*0Sstevel@tonic-gate * The purpose of startup memlist is to get the system to the 802*0Sstevel@tonic-gate * point where it can use kmem_alloc()'s that operate correctly 803*0Sstevel@tonic-gate * relying on BOP_ALLOC(). This includes allocating page_ts, 804*0Sstevel@tonic-gate * page hash table, vmem initialized, etc. 805*0Sstevel@tonic-gate * 806*0Sstevel@tonic-gate * Boot's versions of physinstalled and physavail are insufficient for 807*0Sstevel@tonic-gate * the kernel's purposes. Specifically we don't know which pages that 808*0Sstevel@tonic-gate * are not in physavail can be reclaimed after boot is gone. 809*0Sstevel@tonic-gate * 810*0Sstevel@tonic-gate * This code solves the problem by dividing the address space 811*0Sstevel@tonic-gate * into 3 regions as it takes over the MMU from the booter. 812*0Sstevel@tonic-gate * 813*0Sstevel@tonic-gate * 1) Any (non-nucleus) pages that are mapped at addresses above KERNEL_TEXT 814*0Sstevel@tonic-gate * can not be used by the kernel. 815*0Sstevel@tonic-gate * 816*0Sstevel@tonic-gate * 2) Any free page that happens to be mapped below kernelbase 817*0Sstevel@tonic-gate * is protected until the boot loader is released, but will then be reclaimed. 818*0Sstevel@tonic-gate * 819*0Sstevel@tonic-gate * 3) Boot shouldn't use any address in the remaining area between kernelbase 820*0Sstevel@tonic-gate * and KERNEL_TEXT. 821*0Sstevel@tonic-gate * 822*0Sstevel@tonic-gate * In the case of multiple mappings to the same page, region 1 has precedence 823*0Sstevel@tonic-gate * over region 2. 824*0Sstevel@tonic-gate */ 825*0Sstevel@tonic-gate static void 826*0Sstevel@tonic-gate startup_memlist(void) 827*0Sstevel@tonic-gate { 828*0Sstevel@tonic-gate size_t memlist_sz; 829*0Sstevel@tonic-gate size_t memseg_sz; 830*0Sstevel@tonic-gate size_t pagehash_sz; 831*0Sstevel@tonic-gate size_t pp_sz; 832*0Sstevel@tonic-gate uintptr_t va; 833*0Sstevel@tonic-gate size_t len; 834*0Sstevel@tonic-gate uint_t prot; 835*0Sstevel@tonic-gate pfn_t pfn; 836*0Sstevel@tonic-gate int memblocks; 837*0Sstevel@tonic-gate caddr_t pagecolor_mem; 838*0Sstevel@tonic-gate size_t pagecolor_memsz; 839*0Sstevel@tonic-gate caddr_t page_ctrs_mem; 840*0Sstevel@tonic-gate size_t page_ctrs_size; 841*0Sstevel@tonic-gate struct memlist *current; 842*0Sstevel@tonic-gate extern void startup_build_mem_nodes(struct memlist *); 843*0Sstevel@tonic-gate 844*0Sstevel@tonic-gate /* XX64 fix these - they should be in include files */ 845*0Sstevel@tonic-gate extern ulong_t cr4_value; 846*0Sstevel@tonic-gate extern size_t page_coloring_init(uint_t, int, int); 847*0Sstevel@tonic-gate extern void page_coloring_setup(caddr_t); 848*0Sstevel@tonic-gate 849*0Sstevel@tonic-gate PRM_POINT("startup_memlist() starting..."); 850*0Sstevel@tonic-gate 851*0Sstevel@tonic-gate /* 852*0Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update. 853*0Sstevel@tonic-gate * For this to work properly, we first have to ask boot for its 854*0Sstevel@tonic-gate * end address. 855*0Sstevel@tonic-gate */ 856*0Sstevel@tonic-gate if (BOP_GETPROPLEN(bootops, "memory-update") == 0) 857*0Sstevel@tonic-gate (void) BOP_GETPROP(bootops, "memory-update", NULL); 858*0Sstevel@tonic-gate 859*0Sstevel@tonic-gate /* 860*0Sstevel@tonic-gate * find if the kernel is mapped on a large page 861*0Sstevel@tonic-gate */ 862*0Sstevel@tonic-gate va = KERNEL_TEXT; 863*0Sstevel@tonic-gate if (hat_boot_probe(&va, &len, &pfn, &prot) == 0) 864*0Sstevel@tonic-gate panic("Couldn't find kernel text boot mapping"); 865*0Sstevel@tonic-gate 866*0Sstevel@tonic-gate /* 867*0Sstevel@tonic-gate * Use leftover large page nucleus text/data space for loadable modules. 868*0Sstevel@tonic-gate * Use at most MODTEXT/MODDATA. 869*0Sstevel@tonic-gate */ 870*0Sstevel@tonic-gate if (len > MMU_PAGESIZE) { 871*0Sstevel@tonic-gate 872*0Sstevel@tonic-gate moddata = (caddr_t)ROUND_UP_PAGE(e_data); 873*0Sstevel@tonic-gate e_moddata = (caddr_t)ROUND_UP_4MEG(e_data); 874*0Sstevel@tonic-gate if (e_moddata - moddata > MODDATA) 875*0Sstevel@tonic-gate e_moddata = moddata + MODDATA; 876*0Sstevel@tonic-gate 877*0Sstevel@tonic-gate modtext = (caddr_t)ROUND_UP_PAGE(e_text); 878*0Sstevel@tonic-gate e_modtext = (caddr_t)ROUND_UP_4MEG(e_text); 879*0Sstevel@tonic-gate if (e_modtext - modtext > MODTEXT) 880*0Sstevel@tonic-gate e_modtext = modtext + MODTEXT; 881*0Sstevel@tonic-gate 882*0Sstevel@tonic-gate 883*0Sstevel@tonic-gate } else { 884*0Sstevel@tonic-gate 885*0Sstevel@tonic-gate PRM_POINT("Kernel NOT loaded on Large Page!"); 886*0Sstevel@tonic-gate e_moddata = moddata = (caddr_t)ROUND_UP_PAGE(e_data); 887*0Sstevel@tonic-gate e_modtext = modtext = (caddr_t)ROUND_UP_PAGE(e_text); 888*0Sstevel@tonic-gate 889*0Sstevel@tonic-gate } 890*0Sstevel@tonic-gate econtig = e_moddata; 891*0Sstevel@tonic-gate 892*0Sstevel@tonic-gate PRM_DEBUG(modtext); 893*0Sstevel@tonic-gate PRM_DEBUG(e_modtext); 894*0Sstevel@tonic-gate PRM_DEBUG(moddata); 895*0Sstevel@tonic-gate PRM_DEBUG(e_moddata); 896*0Sstevel@tonic-gate PRM_DEBUG(econtig); 897*0Sstevel@tonic-gate 898*0Sstevel@tonic-gate /* 899*0Sstevel@tonic-gate * For MP machines cr4_value must be set or the non-boot 900*0Sstevel@tonic-gate * CPUs will not be able to start. 901*0Sstevel@tonic-gate */ 902*0Sstevel@tonic-gate if (x86_feature & X86_LARGEPAGE) 903*0Sstevel@tonic-gate cr4_value = getcr4(); 904*0Sstevel@tonic-gate PRM_DEBUG(cr4_value); 905*0Sstevel@tonic-gate 906*0Sstevel@tonic-gate /* 907*0Sstevel@tonic-gate * Examine the boot loaders physical memory map to find out: 908*0Sstevel@tonic-gate * - total memory in system - physinstalled 909*0Sstevel@tonic-gate * - the max physical address - physmax 910*0Sstevel@tonic-gate * - the number of segments the intsalled memory comes in 911*0Sstevel@tonic-gate */ 912*0Sstevel@tonic-gate if (prom_debug) 913*0Sstevel@tonic-gate print_boot_memlist("boot physinstalled", 914*0Sstevel@tonic-gate bootops->boot_mem->physinstalled); 915*0Sstevel@tonic-gate installed_top_size(bootops->boot_mem->physinstalled, &physmax, 916*0Sstevel@tonic-gate &physinstalled, &memblocks); 917*0Sstevel@tonic-gate PRM_DEBUG(physmax); 918*0Sstevel@tonic-gate PRM_DEBUG(physinstalled); 919*0Sstevel@tonic-gate PRM_DEBUG(memblocks); 920*0Sstevel@tonic-gate 921*0Sstevel@tonic-gate if (prom_debug) 922*0Sstevel@tonic-gate print_boot_memlist("boot physavail", 923*0Sstevel@tonic-gate bootops->boot_mem->physavail); 924*0Sstevel@tonic-gate 925*0Sstevel@tonic-gate /* 926*0Sstevel@tonic-gate * Initialize hat's mmu parameters. 927*0Sstevel@tonic-gate * Check for enforce-prot-exec in boot environment. It's used to 928*0Sstevel@tonic-gate * enable/disable support for the page table entry NX bit. 929*0Sstevel@tonic-gate * The default is to enforce PROT_EXEC on processors that support NX. 930*0Sstevel@tonic-gate * Boot seems to round up the "len", but 8 seems to be big enough. 931*0Sstevel@tonic-gate */ 932*0Sstevel@tonic-gate mmu_init(); 933*0Sstevel@tonic-gate 934*0Sstevel@tonic-gate #ifdef __i386 935*0Sstevel@tonic-gate /* 936*0Sstevel@tonic-gate * physmax is lowered if there is more memory than can be 937*0Sstevel@tonic-gate * physically addressed in 32 bit (PAE/non-PAE) modes. 938*0Sstevel@tonic-gate */ 939*0Sstevel@tonic-gate if (mmu.pae_hat) { 940*0Sstevel@tonic-gate if (PFN_ABOVE64G(physmax)) { 941*0Sstevel@tonic-gate physinstalled -= (physmax - (PFN_64G - 1)); 942*0Sstevel@tonic-gate physmax = PFN_64G - 1; 943*0Sstevel@tonic-gate } 944*0Sstevel@tonic-gate } else { 945*0Sstevel@tonic-gate if (PFN_ABOVE4G(physmax)) { 946*0Sstevel@tonic-gate physinstalled -= (physmax - (PFN_4G - 1)); 947*0Sstevel@tonic-gate physmax = PFN_4G - 1; 948*0Sstevel@tonic-gate } 949*0Sstevel@tonic-gate } 950*0Sstevel@tonic-gate #endif 951*0Sstevel@tonic-gate 952*0Sstevel@tonic-gate startup_build_mem_nodes(bootops->boot_mem->physinstalled); 953*0Sstevel@tonic-gate 954*0Sstevel@tonic-gate if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) { 955*0Sstevel@tonic-gate int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec"); 956*0Sstevel@tonic-gate char value[8]; 957*0Sstevel@tonic-gate 958*0Sstevel@tonic-gate if (len < 8) 959*0Sstevel@tonic-gate (void) BOP_GETPROP(bootops, "enforce-prot-exec", value); 960*0Sstevel@tonic-gate else 961*0Sstevel@tonic-gate (void) strcpy(value, ""); 962*0Sstevel@tonic-gate if (strcmp(value, "off") == 0) 963*0Sstevel@tonic-gate mmu.pt_nx = 0; 964*0Sstevel@tonic-gate } 965*0Sstevel@tonic-gate PRM_DEBUG(mmu.pt_nx); 966*0Sstevel@tonic-gate 967*0Sstevel@tonic-gate /* 968*0Sstevel@tonic-gate * We will need page_t's for every page in the system, except for 969*0Sstevel@tonic-gate * memory mapped at or above above the start of the kernel text segment. 970*0Sstevel@tonic-gate * 971*0Sstevel@tonic-gate * pages above e_modtext are attributed to kernel debugger (obp_pages) 972*0Sstevel@tonic-gate */ 973*0Sstevel@tonic-gate npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */ 974*0Sstevel@tonic-gate obp_pages = 0; 975*0Sstevel@tonic-gate va = KERNEL_TEXT; 976*0Sstevel@tonic-gate while (hat_boot_probe(&va, &len, &pfn, &prot) != 0) { 977*0Sstevel@tonic-gate npages -= len >> MMU_PAGESHIFT; 978*0Sstevel@tonic-gate if (va >= (uintptr_t)e_moddata) 979*0Sstevel@tonic-gate obp_pages += len >> MMU_PAGESHIFT; 980*0Sstevel@tonic-gate va += len; 981*0Sstevel@tonic-gate } 982*0Sstevel@tonic-gate PRM_DEBUG(npages); 983*0Sstevel@tonic-gate PRM_DEBUG(obp_pages); 984*0Sstevel@tonic-gate 985*0Sstevel@tonic-gate /* 986*0Sstevel@tonic-gate * If physmem is patched to be non-zero, use it instead of 987*0Sstevel@tonic-gate * the computed value unless it is larger than the real 988*0Sstevel@tonic-gate * amount of memory on hand. 989*0Sstevel@tonic-gate */ 990*0Sstevel@tonic-gate if (physmem == 0 || physmem > npages) 991*0Sstevel@tonic-gate physmem = npages; 992*0Sstevel@tonic-gate else 993*0Sstevel@tonic-gate npages = physmem; 994*0Sstevel@tonic-gate PRM_DEBUG(physmem); 995*0Sstevel@tonic-gate 996*0Sstevel@tonic-gate /* 997*0Sstevel@tonic-gate * We now compute the sizes of all the initial allocations for 998*0Sstevel@tonic-gate * structures the kernel needs in order do kmem_alloc(). These 999*0Sstevel@tonic-gate * include: 1000*0Sstevel@tonic-gate * memsegs 1001*0Sstevel@tonic-gate * memlists 1002*0Sstevel@tonic-gate * page hash table 1003*0Sstevel@tonic-gate * page_t's 1004*0Sstevel@tonic-gate * page coloring data structs 1005*0Sstevel@tonic-gate */ 1006*0Sstevel@tonic-gate memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS); 1007*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(memseg_base, memseg_sz); 1008*0Sstevel@tonic-gate PRM_DEBUG(memseg_sz); 1009*0Sstevel@tonic-gate 1010*0Sstevel@tonic-gate /* 1011*0Sstevel@tonic-gate * Reserve space for phys_avail/phys_install memlists. 1012*0Sstevel@tonic-gate * There's no real good way to know exactly how much room we'll need, 1013*0Sstevel@tonic-gate * but this should be a good upper bound. 1014*0Sstevel@tonic-gate */ 1015*0Sstevel@tonic-gate memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) * 1016*0Sstevel@tonic-gate (memblocks + POSS_NEW_FRAGMENTS)); 1017*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(memlist, memlist_sz); 1018*0Sstevel@tonic-gate PRM_DEBUG(memlist_sz); 1019*0Sstevel@tonic-gate 1020*0Sstevel@tonic-gate /* 1021*0Sstevel@tonic-gate * The page structure hash table size is a power of 2 1022*0Sstevel@tonic-gate * such that the average hash chain length is PAGE_HASHAVELEN. 1023*0Sstevel@tonic-gate */ 1024*0Sstevel@tonic-gate page_hashsz = npages / PAGE_HASHAVELEN; 1025*0Sstevel@tonic-gate page_hashsz = 1 << highbit(page_hashsz); 1026*0Sstevel@tonic-gate pagehash_sz = sizeof (struct page *) * page_hashsz; 1027*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(page_hash, pagehash_sz); 1028*0Sstevel@tonic-gate PRM_DEBUG(pagehash_sz); 1029*0Sstevel@tonic-gate 1030*0Sstevel@tonic-gate /* 1031*0Sstevel@tonic-gate * Set aside room for the page structures themselves. Note: on 1032*0Sstevel@tonic-gate * 64-bit systems we don't allocate page_t's for every page here. 1033*0Sstevel@tonic-gate * We just allocate enough to map the lowest 4GB of physical 1034*0Sstevel@tonic-gate * memory, minus those pages that are used for the "nucleus" kernel 1035*0Sstevel@tonic-gate * text and data. The remaining pages are allocated once we can 1036*0Sstevel@tonic-gate * map around boot. 1037*0Sstevel@tonic-gate * 1038*0Sstevel@tonic-gate * boot_npages is used to allocate an area big enough for our 1039*0Sstevel@tonic-gate * initial page_t's. kphym_init may use less than that. 1040*0Sstevel@tonic-gate */ 1041*0Sstevel@tonic-gate boot_npages = npages; 1042*0Sstevel@tonic-gate #if defined(__amd64) 1043*0Sstevel@tonic-gate if (npages > mmu_btop(FOURGB - (econtig - s_text))) 1044*0Sstevel@tonic-gate boot_npages = mmu_btop(FOURGB - (econtig - s_text)); 1045*0Sstevel@tonic-gate #endif 1046*0Sstevel@tonic-gate PRM_DEBUG(boot_npages); 1047*0Sstevel@tonic-gate pp_sz = sizeof (struct page) * boot_npages; 1048*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(pp_base, pp_sz); 1049*0Sstevel@tonic-gate PRM_DEBUG(pp_sz); 1050*0Sstevel@tonic-gate 1051*0Sstevel@tonic-gate /* 1052*0Sstevel@tonic-gate * determine l2 cache info and memory size for page coloring 1053*0Sstevel@tonic-gate */ 1054*0Sstevel@tonic-gate (void) getl2cacheinfo(CPU, 1055*0Sstevel@tonic-gate &l2cache_sz, &l2cache_linesz, &l2cache_assoc); 1056*0Sstevel@tonic-gate pagecolor_memsz = 1057*0Sstevel@tonic-gate page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc); 1058*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz); 1059*0Sstevel@tonic-gate PRM_DEBUG(pagecolor_memsz); 1060*0Sstevel@tonic-gate 1061*0Sstevel@tonic-gate page_ctrs_size = page_ctrs_sz(); 1062*0Sstevel@tonic-gate ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size); 1063*0Sstevel@tonic-gate PRM_DEBUG(page_ctrs_size); 1064*0Sstevel@tonic-gate 1065*0Sstevel@tonic-gate /* 1066*0Sstevel@tonic-gate * valloc_base will be below kernel text 1067*0Sstevel@tonic-gate * The extra pages are for the HAT and kmdb to map page tables. 1068*0Sstevel@tonic-gate */ 1069*0Sstevel@tonic-gate valloc_sz = ROUND_UP_LPAGE(valloc_sz); 1070*0Sstevel@tonic-gate valloc_base = KERNEL_TEXT - valloc_sz; 1071*0Sstevel@tonic-gate PRM_DEBUG(valloc_base); 1072*0Sstevel@tonic-gate ptable_va = valloc_base - ptable_sz; 1073*0Sstevel@tonic-gate 1074*0Sstevel@tonic-gate #if defined(__amd64) 1075*0Sstevel@tonic-gate if (eprom_kernelbase && eprom_kernelbase != KERNELBASE) 1076*0Sstevel@tonic-gate cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit " 1077*0Sstevel@tonic-gate "systems."); 1078*0Sstevel@tonic-gate kernelbase = (uintptr_t)KERNELBASE; 1079*0Sstevel@tonic-gate core_base = (uintptr_t)COREHEAP_BASE; 1080*0Sstevel@tonic-gate core_size = ptable_va - core_base; 1081*0Sstevel@tonic-gate #else /* __i386 */ 1082*0Sstevel@tonic-gate /* 1083*0Sstevel@tonic-gate * We configure kernelbase based on: 1084*0Sstevel@tonic-gate * 1085*0Sstevel@tonic-gate * 1. user specified kernelbase via eeprom command. Value cannot exceed 1086*0Sstevel@tonic-gate * KERNELBASE_MAX. we large page align eprom_kernelbase 1087*0Sstevel@tonic-gate * 1088*0Sstevel@tonic-gate * 2. Default to KERNELBASE and adjust to 2X less the size for page_t. 1089*0Sstevel@tonic-gate * On large memory systems we must lower kernelbase to allow 1090*0Sstevel@tonic-gate * enough room for page_t's for all of memory. 1091*0Sstevel@tonic-gate * 1092*0Sstevel@tonic-gate * The value set here, might be changed a little later. 1093*0Sstevel@tonic-gate */ 1094*0Sstevel@tonic-gate if (eprom_kernelbase) { 1095*0Sstevel@tonic-gate kernelbase = eprom_kernelbase & mmu.level_mask[1]; 1096*0Sstevel@tonic-gate if (kernelbase > KERNELBASE_MAX) 1097*0Sstevel@tonic-gate kernelbase = KERNELBASE_MAX; 1098*0Sstevel@tonic-gate } else { 1099*0Sstevel@tonic-gate kernelbase = (uintptr_t)KERNELBASE; 1100*0Sstevel@tonic-gate kernelbase -= ROUND_UP_4MEG(2 * valloc_sz); 1101*0Sstevel@tonic-gate } 1102*0Sstevel@tonic-gate ASSERT((kernelbase & mmu.level_offset[1]) == 0); 1103*0Sstevel@tonic-gate core_base = ptable_va; 1104*0Sstevel@tonic-gate core_size = 0; 1105*0Sstevel@tonic-gate #endif 1106*0Sstevel@tonic-gate 1107*0Sstevel@tonic-gate PRM_DEBUG(kernelbase); 1108*0Sstevel@tonic-gate PRM_DEBUG(core_base); 1109*0Sstevel@tonic-gate PRM_DEBUG(core_size); 1110*0Sstevel@tonic-gate 1111*0Sstevel@tonic-gate /* 1112*0Sstevel@tonic-gate * At this point, we can only use a portion of the kernelheap that 1113*0Sstevel@tonic-gate * will be available after we boot. Both 32-bit and 64-bit systems 1114*0Sstevel@tonic-gate * have this limitation, although the reasons are completely 1115*0Sstevel@tonic-gate * different. 1116*0Sstevel@tonic-gate * 1117*0Sstevel@tonic-gate * On 64-bit systems, the booter only supports allocations in the 1118*0Sstevel@tonic-gate * upper 4GB of memory, so we have to work with a reduced kernel 1119*0Sstevel@tonic-gate * heap until we take over all allocations. The booter also sits 1120*0Sstevel@tonic-gate * in the lower portion of that 4GB range, so we have to raise the 1121*0Sstevel@tonic-gate * bottom of the heap even further. 1122*0Sstevel@tonic-gate * 1123*0Sstevel@tonic-gate * On 32-bit systems we have to leave room to place segmap below 1124*0Sstevel@tonic-gate * the heap. We don't yet know how large segmap will be, so we 1125*0Sstevel@tonic-gate * have to be very conservative. 1126*0Sstevel@tonic-gate */ 1127*0Sstevel@tonic-gate #if defined(__amd64) 1128*0Sstevel@tonic-gate /* 1129*0Sstevel@tonic-gate * XX64: For now, we let boot have the lower 2GB of the top 4GB 1130*0Sstevel@tonic-gate * address range. In the long run, that should be fixed. It's 1131*0Sstevel@tonic-gate * insane for a booter to need 2 2GB address ranges. 1132*0Sstevel@tonic-gate */ 1133*0Sstevel@tonic-gate boot_kernelheap = (caddr_t)(BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE); 1134*0Sstevel@tonic-gate segmap_reserved = 0; 1135*0Sstevel@tonic-gate 1136*0Sstevel@tonic-gate #else /* __i386 */ 1137*0Sstevel@tonic-gate segkp_fromheap = 1; 1138*0Sstevel@tonic-gate segmap_reserved = ROUND_UP_LPAGE(MAX(segmapsize, SEGMAPMAX)); 1139*0Sstevel@tonic-gate boot_kernelheap = (caddr_t)(ROUND_UP_LPAGE(kernelbase) + 1140*0Sstevel@tonic-gate segmap_reserved); 1141*0Sstevel@tonic-gate #endif 1142*0Sstevel@tonic-gate PRM_DEBUG(boot_kernelheap); 1143*0Sstevel@tonic-gate kernelheap = boot_kernelheap; 1144*0Sstevel@tonic-gate ekernelheap = (char *)core_base; 1145*0Sstevel@tonic-gate 1146*0Sstevel@tonic-gate /* 1147*0Sstevel@tonic-gate * If segmap is too large we can push the bottom of the kernel heap 1148*0Sstevel@tonic-gate * higher than the base. Or worse, it could exceed the top of the 1149*0Sstevel@tonic-gate * VA space entirely, causing it to wrap around. 1150*0Sstevel@tonic-gate */ 1151*0Sstevel@tonic-gate if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase) 1152*0Sstevel@tonic-gate panic("too little memory available for kernelheap," 1153*0Sstevel@tonic-gate " use a different kernelbase"); 1154*0Sstevel@tonic-gate 1155*0Sstevel@tonic-gate /* 1156*0Sstevel@tonic-gate * Now that we know the real value of kernelbase, 1157*0Sstevel@tonic-gate * update variables that were initialized with a value of 1158*0Sstevel@tonic-gate * KERNELBASE (in common/conf/param.c). 1159*0Sstevel@tonic-gate * 1160*0Sstevel@tonic-gate * XXX The problem with this sort of hackery is that the 1161*0Sstevel@tonic-gate * compiler just may feel like putting the const declarations 1162*0Sstevel@tonic-gate * (in param.c) into the .text section. Perhaps they should 1163*0Sstevel@tonic-gate * just be declared as variables there? 1164*0Sstevel@tonic-gate */ 1165*0Sstevel@tonic-gate 1166*0Sstevel@tonic-gate #if defined(__amd64) 1167*0Sstevel@tonic-gate ASSERT(_kernelbase == KERNELBASE); 1168*0Sstevel@tonic-gate ASSERT(_userlimit == USERLIMIT); 1169*0Sstevel@tonic-gate /* 1170*0Sstevel@tonic-gate * As one final sanity check, verify that the "red zone" between 1171*0Sstevel@tonic-gate * kernel and userspace is exactly the size we expected. 1172*0Sstevel@tonic-gate */ 1173*0Sstevel@tonic-gate ASSERT(_kernelbase == (_userlimit + (2 * 1024 * 1024))); 1174*0Sstevel@tonic-gate #else 1175*0Sstevel@tonic-gate *(uintptr_t *)&_kernelbase = kernelbase; 1176*0Sstevel@tonic-gate *(uintptr_t *)&_userlimit = kernelbase; 1177*0Sstevel@tonic-gate *(uintptr_t *)&_userlimit32 = _userlimit; 1178*0Sstevel@tonic-gate #endif 1179*0Sstevel@tonic-gate PRM_DEBUG(_kernelbase); 1180*0Sstevel@tonic-gate PRM_DEBUG(_userlimit); 1181*0Sstevel@tonic-gate PRM_DEBUG(_userlimit32); 1182*0Sstevel@tonic-gate 1183*0Sstevel@tonic-gate /* 1184*0Sstevel@tonic-gate * do all the initial allocations 1185*0Sstevel@tonic-gate */ 1186*0Sstevel@tonic-gate perform_allocations(); 1187*0Sstevel@tonic-gate 1188*0Sstevel@tonic-gate /* 1189*0Sstevel@tonic-gate * Initialize the kernel heap. Note 3rd argument must be > 1st. 1190*0Sstevel@tonic-gate */ 1191*0Sstevel@tonic-gate kernelheap_init(kernelheap, ekernelheap, kernelheap + MMU_PAGESIZE, 1192*0Sstevel@tonic-gate (void *)core_base, (void *)ptable_va); 1193*0Sstevel@tonic-gate 1194*0Sstevel@tonic-gate /* 1195*0Sstevel@tonic-gate * Build phys_install and phys_avail in kernel memspace. 1196*0Sstevel@tonic-gate * - phys_install should be all memory in the system. 1197*0Sstevel@tonic-gate * - phys_avail is phys_install minus any memory mapped before this 1198*0Sstevel@tonic-gate * point above KERNEL_TEXT. 1199*0Sstevel@tonic-gate */ 1200*0Sstevel@tonic-gate current = phys_install = memlist; 1201*0Sstevel@tonic-gate copy_memlist_filter(bootops->boot_mem->physinstalled, ¤t, NULL); 1202*0Sstevel@tonic-gate if ((caddr_t)current > (caddr_t)memlist + memlist_sz) 1203*0Sstevel@tonic-gate panic("physinstalled was too big!"); 1204*0Sstevel@tonic-gate if (prom_debug) 1205*0Sstevel@tonic-gate print_kernel_memlist("phys_install", phys_install); 1206*0Sstevel@tonic-gate 1207*0Sstevel@tonic-gate phys_avail = current; 1208*0Sstevel@tonic-gate PRM_POINT("Building phys_avail:\n"); 1209*0Sstevel@tonic-gate copy_memlist_filter(bootops->boot_mem->physinstalled, ¤t, 1210*0Sstevel@tonic-gate avail_filter); 1211*0Sstevel@tonic-gate if ((caddr_t)current > (caddr_t)memlist + memlist_sz) 1212*0Sstevel@tonic-gate panic("physavail was too big!"); 1213*0Sstevel@tonic-gate if (prom_debug) 1214*0Sstevel@tonic-gate print_kernel_memlist("phys_avail", phys_avail); 1215*0Sstevel@tonic-gate 1216*0Sstevel@tonic-gate /* 1217*0Sstevel@tonic-gate * setup page coloring 1218*0Sstevel@tonic-gate */ 1219*0Sstevel@tonic-gate page_coloring_setup(pagecolor_mem); 1220*0Sstevel@tonic-gate page_lock_init(); /* currently a no-op */ 1221*0Sstevel@tonic-gate 1222*0Sstevel@tonic-gate /* 1223*0Sstevel@tonic-gate * free page list counters 1224*0Sstevel@tonic-gate */ 1225*0Sstevel@tonic-gate (void) page_ctrs_alloc(page_ctrs_mem); 1226*0Sstevel@tonic-gate 1227*0Sstevel@tonic-gate /* 1228*0Sstevel@tonic-gate * Initialize the page structures from the memory lists. 1229*0Sstevel@tonic-gate */ 1230*0Sstevel@tonic-gate availrmem_initial = availrmem = freemem = 0; 1231*0Sstevel@tonic-gate PRM_POINT("Calling kphysm_init()..."); 1232*0Sstevel@tonic-gate boot_npages = kphysm_init(pp_base, memseg_base, 0, boot_npages); 1233*0Sstevel@tonic-gate PRM_POINT("kphysm_init() done"); 1234*0Sstevel@tonic-gate PRM_DEBUG(boot_npages); 1235*0Sstevel@tonic-gate 1236*0Sstevel@tonic-gate /* 1237*0Sstevel@tonic-gate * Now that page_t's have been initialized, remove all the 1238*0Sstevel@tonic-gate * initial allocation pages from the kernel free page lists. 1239*0Sstevel@tonic-gate */ 1240*0Sstevel@tonic-gate boot_mapin((caddr_t)valloc_base, valloc_sz); 1241*0Sstevel@tonic-gate 1242*0Sstevel@tonic-gate /* 1243*0Sstevel@tonic-gate * Initialize kernel memory allocator. 1244*0Sstevel@tonic-gate */ 1245*0Sstevel@tonic-gate kmem_init(); 1246*0Sstevel@tonic-gate 1247*0Sstevel@tonic-gate /* 1248*0Sstevel@tonic-gate * print this out early so that we know what's going on 1249*0Sstevel@tonic-gate */ 1250*0Sstevel@tonic-gate cmn_err(CE_CONT, "?features: %b\n", x86_feature, FMT_X86_FEATURE); 1251*0Sstevel@tonic-gate 1252*0Sstevel@tonic-gate /* 1253*0Sstevel@tonic-gate * Initialize bp_mapin(). 1254*0Sstevel@tonic-gate */ 1255*0Sstevel@tonic-gate bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK); 1256*0Sstevel@tonic-gate 1257*0Sstevel@tonic-gate #if defined(__i386) 1258*0Sstevel@tonic-gate if (eprom_kernelbase && (eprom_kernelbase != kernelbase)) 1259*0Sstevel@tonic-gate cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, " 1260*0Sstevel@tonic-gate "System using 0x%lx", 1261*0Sstevel@tonic-gate (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase); 1262*0Sstevel@tonic-gate #endif 1263*0Sstevel@tonic-gate 1264*0Sstevel@tonic-gate #ifdef KERNELBASE_ABI_MIN 1265*0Sstevel@tonic-gate if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) { 1266*0Sstevel@tonic-gate cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not " 1267*0Sstevel@tonic-gate "i386 ABI compliant.", (uintptr_t)kernelbase); 1268*0Sstevel@tonic-gate } 1269*0Sstevel@tonic-gate #endif 1270*0Sstevel@tonic-gate 1271*0Sstevel@tonic-gate PRM_POINT("startup_memlist() done"); 1272*0Sstevel@tonic-gate } 1273*0Sstevel@tonic-gate 1274*0Sstevel@tonic-gate static void 1275*0Sstevel@tonic-gate startup_modules(void) 1276*0Sstevel@tonic-gate { 1277*0Sstevel@tonic-gate unsigned int i; 1278*0Sstevel@tonic-gate extern void impl_setup_ddi(void); 1279*0Sstevel@tonic-gate extern void prom_setup(void); 1280*0Sstevel@tonic-gate 1281*0Sstevel@tonic-gate PRM_POINT("startup_modules() starting..."); 1282*0Sstevel@tonic-gate /* 1283*0Sstevel@tonic-gate * Initialize ten-micro second timer so that drivers will 1284*0Sstevel@tonic-gate * not get short changed in their init phase. This was 1285*0Sstevel@tonic-gate * not getting called until clkinit which, on fast cpu's 1286*0Sstevel@tonic-gate * caused the drv_usecwait to be way too short. 1287*0Sstevel@tonic-gate */ 1288*0Sstevel@tonic-gate microfind(); 1289*0Sstevel@tonic-gate 1290*0Sstevel@tonic-gate /* 1291*0Sstevel@tonic-gate * Read the GMT lag from /etc/rtc_config. 1292*0Sstevel@tonic-gate */ 1293*0Sstevel@tonic-gate gmt_lag = process_rtc_config_file(); 1294*0Sstevel@tonic-gate 1295*0Sstevel@tonic-gate /* 1296*0Sstevel@tonic-gate * Calculate default settings of system parameters based upon 1297*0Sstevel@tonic-gate * maxusers, yet allow to be overridden via the /etc/system file. 1298*0Sstevel@tonic-gate */ 1299*0Sstevel@tonic-gate param_calc(0); 1300*0Sstevel@tonic-gate 1301*0Sstevel@tonic-gate mod_setup(); 1302*0Sstevel@tonic-gate 1303*0Sstevel@tonic-gate /* 1304*0Sstevel@tonic-gate * Setup machine check architecture on P6 1305*0Sstevel@tonic-gate */ 1306*0Sstevel@tonic-gate setup_mca(); 1307*0Sstevel@tonic-gate 1308*0Sstevel@tonic-gate /* 1309*0Sstevel@tonic-gate * Initialize system parameters. 1310*0Sstevel@tonic-gate */ 1311*0Sstevel@tonic-gate param_init(); 1312*0Sstevel@tonic-gate 1313*0Sstevel@tonic-gate /* 1314*0Sstevel@tonic-gate * maxmem is the amount of physical memory we're playing with. 1315*0Sstevel@tonic-gate */ 1316*0Sstevel@tonic-gate maxmem = physmem; 1317*0Sstevel@tonic-gate 1318*0Sstevel@tonic-gate /* 1319*0Sstevel@tonic-gate * Initialize the hat layer. 1320*0Sstevel@tonic-gate */ 1321*0Sstevel@tonic-gate hat_init(); 1322*0Sstevel@tonic-gate 1323*0Sstevel@tonic-gate /* 1324*0Sstevel@tonic-gate * Initialize segment management stuff. 1325*0Sstevel@tonic-gate */ 1326*0Sstevel@tonic-gate seg_init(); 1327*0Sstevel@tonic-gate 1328*0Sstevel@tonic-gate if (modload("fs", "specfs") == -1) 1329*0Sstevel@tonic-gate halt("Can't load specfs"); 1330*0Sstevel@tonic-gate 1331*0Sstevel@tonic-gate if (modload("fs", "devfs") == -1) 1332*0Sstevel@tonic-gate halt("Can't load devfs"); 1333*0Sstevel@tonic-gate 1334*0Sstevel@tonic-gate dispinit(); 1335*0Sstevel@tonic-gate 1336*0Sstevel@tonic-gate /* 1337*0Sstevel@tonic-gate * This is needed here to initialize hw_serial[] for cluster booting. 1338*0Sstevel@tonic-gate */ 1339*0Sstevel@tonic-gate if ((i = modload("misc", "sysinit")) != (unsigned int)-1) 1340*0Sstevel@tonic-gate (void) modunload(i); 1341*0Sstevel@tonic-gate else 1342*0Sstevel@tonic-gate cmn_err(CE_CONT, "sysinit load failed"); 1343*0Sstevel@tonic-gate 1344*0Sstevel@tonic-gate /* Read cluster configuration data. */ 1345*0Sstevel@tonic-gate clconf_init(); 1346*0Sstevel@tonic-gate 1347*0Sstevel@tonic-gate /* 1348*0Sstevel@tonic-gate * Create a kernel device tree. First, create rootnex and 1349*0Sstevel@tonic-gate * then invoke bus specific code to probe devices. 1350*0Sstevel@tonic-gate */ 1351*0Sstevel@tonic-gate setup_ddi(); 1352*0Sstevel@tonic-gate impl_setup_ddi(); 1353*0Sstevel@tonic-gate /* 1354*0Sstevel@tonic-gate * Fake a prom tree such that /dev/openprom continues to work 1355*0Sstevel@tonic-gate */ 1356*0Sstevel@tonic-gate prom_setup(); 1357*0Sstevel@tonic-gate 1358*0Sstevel@tonic-gate /* 1359*0Sstevel@tonic-gate * Load all platform specific modules 1360*0Sstevel@tonic-gate */ 1361*0Sstevel@tonic-gate psm_modload(); 1362*0Sstevel@tonic-gate 1363*0Sstevel@tonic-gate PRM_POINT("startup_modules() done"); 1364*0Sstevel@tonic-gate } 1365*0Sstevel@tonic-gate 1366*0Sstevel@tonic-gate static void 1367*0Sstevel@tonic-gate startup_bop_gone(void) 1368*0Sstevel@tonic-gate { 1369*0Sstevel@tonic-gate PRM_POINT("startup_bop_gone() starting..."); 1370*0Sstevel@tonic-gate 1371*0Sstevel@tonic-gate /* 1372*0Sstevel@tonic-gate * Do final allocations of HAT data structures that need to 1373*0Sstevel@tonic-gate * be allocated before quiescing the boot loader. 1374*0Sstevel@tonic-gate */ 1375*0Sstevel@tonic-gate PRM_POINT("Calling hat_kern_alloc()..."); 1376*0Sstevel@tonic-gate hat_kern_alloc(); 1377*0Sstevel@tonic-gate PRM_POINT("hat_kern_alloc() done"); 1378*0Sstevel@tonic-gate 1379*0Sstevel@tonic-gate /* 1380*0Sstevel@tonic-gate * Setup MTRR (Memory type range registers) 1381*0Sstevel@tonic-gate */ 1382*0Sstevel@tonic-gate setup_mtrr(); 1383*0Sstevel@tonic-gate PRM_POINT("startup_bop_gone() done"); 1384*0Sstevel@tonic-gate } 1385*0Sstevel@tonic-gate 1386*0Sstevel@tonic-gate /* 1387*0Sstevel@tonic-gate * Walk through the pagetables looking for pages mapped in by boot. If the 1388*0Sstevel@tonic-gate * setaside flag is set the pages are expected to be returned to the 1389*0Sstevel@tonic-gate * kernel later in boot, so we add them to the bootpages list. 1390*0Sstevel@tonic-gate */ 1391*0Sstevel@tonic-gate static void 1392*0Sstevel@tonic-gate protect_boot_range(uintptr_t low, uintptr_t high, int setaside) 1393*0Sstevel@tonic-gate { 1394*0Sstevel@tonic-gate uintptr_t va = low; 1395*0Sstevel@tonic-gate size_t len; 1396*0Sstevel@tonic-gate uint_t prot; 1397*0Sstevel@tonic-gate pfn_t pfn; 1398*0Sstevel@tonic-gate page_t *pp; 1399*0Sstevel@tonic-gate pgcnt_t boot_protect_cnt = 0; 1400*0Sstevel@tonic-gate 1401*0Sstevel@tonic-gate while (hat_boot_probe(&va, &len, &pfn, &prot) != 0 && va < high) { 1402*0Sstevel@tonic-gate if (va + len >= high) 1403*0Sstevel@tonic-gate panic("0x%lx byte mapping at 0x%p exceeds boot's " 1404*0Sstevel@tonic-gate "legal range.", len, (void *)va); 1405*0Sstevel@tonic-gate 1406*0Sstevel@tonic-gate while (len > 0) { 1407*0Sstevel@tonic-gate pp = page_numtopp_alloc(pfn); 1408*0Sstevel@tonic-gate if (pp != NULL) { 1409*0Sstevel@tonic-gate if (setaside == 0) 1410*0Sstevel@tonic-gate panic("Unexpected mapping by boot. " 1411*0Sstevel@tonic-gate "addr=%p pfn=%lx\n", 1412*0Sstevel@tonic-gate (void *)va, pfn); 1413*0Sstevel@tonic-gate 1414*0Sstevel@tonic-gate pp->p_next = bootpages; 1415*0Sstevel@tonic-gate bootpages = pp; 1416*0Sstevel@tonic-gate ++boot_protect_cnt; 1417*0Sstevel@tonic-gate } 1418*0Sstevel@tonic-gate 1419*0Sstevel@tonic-gate ++pfn; 1420*0Sstevel@tonic-gate len -= MMU_PAGESIZE; 1421*0Sstevel@tonic-gate va += MMU_PAGESIZE; 1422*0Sstevel@tonic-gate } 1423*0Sstevel@tonic-gate } 1424*0Sstevel@tonic-gate PRM_DEBUG(boot_protect_cnt); 1425*0Sstevel@tonic-gate } 1426*0Sstevel@tonic-gate 1427*0Sstevel@tonic-gate static void 1428*0Sstevel@tonic-gate startup_vm(void) 1429*0Sstevel@tonic-gate { 1430*0Sstevel@tonic-gate struct segmap_crargs a; 1431*0Sstevel@tonic-gate extern void hat_kern_setup(void); 1432*0Sstevel@tonic-gate pgcnt_t pages_left; 1433*0Sstevel@tonic-gate 1434*0Sstevel@tonic-gate PRM_POINT("startup_vm() starting..."); 1435*0Sstevel@tonic-gate 1436*0Sstevel@tonic-gate /* 1437*0Sstevel@tonic-gate * The next two loops are done in distinct steps in order 1438*0Sstevel@tonic-gate * to be sure that any page that is doubly mapped (both above 1439*0Sstevel@tonic-gate * KERNEL_TEXT and below kernelbase) is dealt with correctly. 1440*0Sstevel@tonic-gate * Note this may never happen, but it might someday. 1441*0Sstevel@tonic-gate */ 1442*0Sstevel@tonic-gate 1443*0Sstevel@tonic-gate bootpages = NULL; 1444*0Sstevel@tonic-gate PRM_POINT("Protecting boot pages"); 1445*0Sstevel@tonic-gate /* 1446*0Sstevel@tonic-gate * Protect any pages mapped above KERNEL_TEXT that somehow have 1447*0Sstevel@tonic-gate * page_t's. This can only happen if something weird allocated 1448*0Sstevel@tonic-gate * in this range (like kadb/kmdb). 1449*0Sstevel@tonic-gate */ 1450*0Sstevel@tonic-gate protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0); 1451*0Sstevel@tonic-gate 1452*0Sstevel@tonic-gate /* 1453*0Sstevel@tonic-gate * Before we can take over memory allocation/mapping from the boot 1454*0Sstevel@tonic-gate * loader we must remove from our free page lists any boot pages that 1455*0Sstevel@tonic-gate * will stay mapped until release_bootstrap(). 1456*0Sstevel@tonic-gate */ 1457*0Sstevel@tonic-gate protect_boot_range(0, kernelbase, 1); 1458*0Sstevel@tonic-gate #if defined(__amd64) 1459*0Sstevel@tonic-gate protect_boot_range(BOOT_DOUBLEMAP_BASE, 1460*0Sstevel@tonic-gate BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE, 0); 1461*0Sstevel@tonic-gate #endif 1462*0Sstevel@tonic-gate 1463*0Sstevel@tonic-gate /* 1464*0Sstevel@tonic-gate * Copy in boot's page tables, set up extra page tables for the kernel, 1465*0Sstevel@tonic-gate * and switch to the kernel's context. 1466*0Sstevel@tonic-gate */ 1467*0Sstevel@tonic-gate PRM_POINT("Calling hat_kern_setup()..."); 1468*0Sstevel@tonic-gate hat_kern_setup(); 1469*0Sstevel@tonic-gate 1470*0Sstevel@tonic-gate /* 1471*0Sstevel@tonic-gate * It is no longer safe to call BOP_ALLOC(), so make sure we don't. 1472*0Sstevel@tonic-gate */ 1473*0Sstevel@tonic-gate bootops->bsys_alloc = NULL; 1474*0Sstevel@tonic-gate PRM_POINT("hat_kern_setup() done"); 1475*0Sstevel@tonic-gate 1476*0Sstevel@tonic-gate hat_cpu_online(CPU); 1477*0Sstevel@tonic-gate 1478*0Sstevel@tonic-gate /* 1479*0Sstevel@tonic-gate * Before we call kvm_init(), we need to establish the final size 1480*0Sstevel@tonic-gate * of the kernel's heap. So, we need to figure out how much space 1481*0Sstevel@tonic-gate * to set aside for segkp, segkpm, and segmap. 1482*0Sstevel@tonic-gate */ 1483*0Sstevel@tonic-gate final_kernelheap = (caddr_t)ROUND_UP_LPAGE(kernelbase); 1484*0Sstevel@tonic-gate #if defined(__amd64) 1485*0Sstevel@tonic-gate if (kpm_desired) { 1486*0Sstevel@tonic-gate /* 1487*0Sstevel@tonic-gate * Segkpm appears at the bottom of the kernel's address 1488*0Sstevel@tonic-gate * range. To detect accidental overruns of the user 1489*0Sstevel@tonic-gate * address space, we leave a "red zone" of unmapped memory 1490*0Sstevel@tonic-gate * between kernelbase and the beginning of segkpm. 1491*0Sstevel@tonic-gate */ 1492*0Sstevel@tonic-gate kpm_vbase = final_kernelheap + KERNEL_REDZONE_SIZE; 1493*0Sstevel@tonic-gate kpm_size = mmu_ptob(physmax); 1494*0Sstevel@tonic-gate PRM_DEBUG(kpm_vbase); 1495*0Sstevel@tonic-gate PRM_DEBUG(kpm_size); 1496*0Sstevel@tonic-gate final_kernelheap = 1497*0Sstevel@tonic-gate (caddr_t)ROUND_UP_TOPLEVEL(kpm_vbase + kpm_size); 1498*0Sstevel@tonic-gate } 1499*0Sstevel@tonic-gate 1500*0Sstevel@tonic-gate if (!segkp_fromheap) { 1501*0Sstevel@tonic-gate size_t sz = mmu_ptob(segkpsize); 1502*0Sstevel@tonic-gate 1503*0Sstevel@tonic-gate /* 1504*0Sstevel@tonic-gate * determine size of segkp and adjust the bottom of the 1505*0Sstevel@tonic-gate * kernel's heap. 1506*0Sstevel@tonic-gate */ 1507*0Sstevel@tonic-gate if (sz < SEGKPMINSIZE || sz > SEGKPMAXSIZE) { 1508*0Sstevel@tonic-gate sz = SEGKPDEFSIZE; 1509*0Sstevel@tonic-gate cmn_err(CE_WARN, "!Illegal value for segkpsize. " 1510*0Sstevel@tonic-gate "segkpsize has been reset to %ld pages", 1511*0Sstevel@tonic-gate mmu_btop(sz)); 1512*0Sstevel@tonic-gate } 1513*0Sstevel@tonic-gate sz = MIN(sz, MAX(SEGKPMINSIZE, mmu_ptob(physmem))); 1514*0Sstevel@tonic-gate 1515*0Sstevel@tonic-gate segkpsize = mmu_btop(ROUND_UP_LPAGE(sz)); 1516*0Sstevel@tonic-gate segkp_base = final_kernelheap; 1517*0Sstevel@tonic-gate PRM_DEBUG(segkpsize); 1518*0Sstevel@tonic-gate PRM_DEBUG(segkp_base); 1519*0Sstevel@tonic-gate final_kernelheap = segkp_base + mmu_ptob(segkpsize); 1520*0Sstevel@tonic-gate PRM_DEBUG(final_kernelheap); 1521*0Sstevel@tonic-gate } 1522*0Sstevel@tonic-gate 1523*0Sstevel@tonic-gate /* 1524*0Sstevel@tonic-gate * put the range of VA for device mappings next 1525*0Sstevel@tonic-gate */ 1526*0Sstevel@tonic-gate toxic_addr = (uintptr_t)final_kernelheap; 1527*0Sstevel@tonic-gate PRM_DEBUG(toxic_addr); 1528*0Sstevel@tonic-gate final_kernelheap = (char *)toxic_addr + toxic_size; 1529*0Sstevel@tonic-gate #endif 1530*0Sstevel@tonic-gate PRM_DEBUG(final_kernelheap); 1531*0Sstevel@tonic-gate ASSERT(final_kernelheap < boot_kernelheap); 1532*0Sstevel@tonic-gate 1533*0Sstevel@tonic-gate /* 1534*0Sstevel@tonic-gate * Users can change segmapsize through eeprom or /etc/system. 1535*0Sstevel@tonic-gate * If the variable is tuned through eeprom, there is no upper 1536*0Sstevel@tonic-gate * bound on the size of segmap. If it is tuned through 1537*0Sstevel@tonic-gate * /etc/system on 32-bit systems, it must be no larger than we 1538*0Sstevel@tonic-gate * planned for in startup_memlist(). 1539*0Sstevel@tonic-gate */ 1540*0Sstevel@tonic-gate segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT); 1541*0Sstevel@tonic-gate segkmap_start = ROUND_UP_LPAGE((uintptr_t)final_kernelheap); 1542*0Sstevel@tonic-gate 1543*0Sstevel@tonic-gate #if defined(__i386) 1544*0Sstevel@tonic-gate if (segmapsize > segmap_reserved) { 1545*0Sstevel@tonic-gate cmn_err(CE_NOTE, "!segmapsize may not be set > 0x%lx in " 1546*0Sstevel@tonic-gate "/etc/system. Use eeprom.", (long)SEGMAPMAX); 1547*0Sstevel@tonic-gate segmapsize = segmap_reserved; 1548*0Sstevel@tonic-gate } 1549*0Sstevel@tonic-gate /* 1550*0Sstevel@tonic-gate * 32-bit systems don't have segkpm or segkp, so segmap appears at 1551*0Sstevel@tonic-gate * the bottom of the kernel's address range. Set aside space for a 1552*0Sstevel@tonic-gate * red zone just below the start of segmap. 1553*0Sstevel@tonic-gate */ 1554*0Sstevel@tonic-gate segkmap_start += KERNEL_REDZONE_SIZE; 1555*0Sstevel@tonic-gate segmapsize -= KERNEL_REDZONE_SIZE; 1556*0Sstevel@tonic-gate #endif 1557*0Sstevel@tonic-gate final_kernelheap = (char *)(segkmap_start + segmapsize); 1558*0Sstevel@tonic-gate 1559*0Sstevel@tonic-gate PRM_DEBUG(segkmap_start); 1560*0Sstevel@tonic-gate PRM_DEBUG(segmapsize); 1561*0Sstevel@tonic-gate PRM_DEBUG(final_kernelheap); 1562*0Sstevel@tonic-gate 1563*0Sstevel@tonic-gate /* 1564*0Sstevel@tonic-gate * Initialize VM system 1565*0Sstevel@tonic-gate */ 1566*0Sstevel@tonic-gate PRM_POINT("Calling kvm_init()..."); 1567*0Sstevel@tonic-gate kvm_init(); 1568*0Sstevel@tonic-gate PRM_POINT("kvm_init() done"); 1569*0Sstevel@tonic-gate 1570*0Sstevel@tonic-gate /* 1571*0Sstevel@tonic-gate * Tell kmdb that the VM system is now working 1572*0Sstevel@tonic-gate */ 1573*0Sstevel@tonic-gate if (boothowto & RB_DEBUG) 1574*0Sstevel@tonic-gate kdi_dvec_vmready(); 1575*0Sstevel@tonic-gate 1576*0Sstevel@tonic-gate /* 1577*0Sstevel@tonic-gate * Mangle the brand string etc. 1578*0Sstevel@tonic-gate */ 1579*0Sstevel@tonic-gate cpuid_pass3(CPU); 1580*0Sstevel@tonic-gate 1581*0Sstevel@tonic-gate PRM_DEBUG(final_kernelheap); 1582*0Sstevel@tonic-gate 1583*0Sstevel@tonic-gate /* 1584*0Sstevel@tonic-gate * Now that we can use memory outside the top 4GB (on 64-bit 1585*0Sstevel@tonic-gate * systems) and we know the size of segmap, we can set the final 1586*0Sstevel@tonic-gate * size of the kernel's heap. Note: on 64-bit systems we still 1587*0Sstevel@tonic-gate * can't touch anything in the bottom half of the top 4GB range 1588*0Sstevel@tonic-gate * because boot still has pages mapped there. 1589*0Sstevel@tonic-gate */ 1590*0Sstevel@tonic-gate if (final_kernelheap < boot_kernelheap) { 1591*0Sstevel@tonic-gate kernelheap_extend(final_kernelheap, boot_kernelheap); 1592*0Sstevel@tonic-gate #if defined(__amd64) 1593*0Sstevel@tonic-gate kmem_setaside = vmem_xalloc(heap_arena, BOOT_DOUBLEMAP_SIZE, 1594*0Sstevel@tonic-gate MMU_PAGESIZE, 0, 0, (void *)(BOOT_DOUBLEMAP_BASE), 1595*0Sstevel@tonic-gate (void *)(BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE), 1596*0Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1597*0Sstevel@tonic-gate PRM_DEBUG(kmem_setaside); 1598*0Sstevel@tonic-gate if (kmem_setaside == NULL) 1599*0Sstevel@tonic-gate panic("Could not protect boot's memory"); 1600*0Sstevel@tonic-gate #endif 1601*0Sstevel@tonic-gate } 1602*0Sstevel@tonic-gate /* 1603*0Sstevel@tonic-gate * Now that the kernel heap may have grown significantly, we need 1604*0Sstevel@tonic-gate * to make all the remaining page_t's available to back that memory. 1605*0Sstevel@tonic-gate * 1606*0Sstevel@tonic-gate * XX64 this should probably wait till after release boot-strap too. 1607*0Sstevel@tonic-gate */ 1608*0Sstevel@tonic-gate pages_left = npages - boot_npages; 1609*0Sstevel@tonic-gate if (pages_left > 0) { 1610*0Sstevel@tonic-gate PRM_DEBUG(pages_left); 1611*0Sstevel@tonic-gate (void) kphysm_init(NULL, memseg_base, boot_npages, pages_left); 1612*0Sstevel@tonic-gate } 1613*0Sstevel@tonic-gate 1614*0Sstevel@tonic-gate #if defined(__amd64) 1615*0Sstevel@tonic-gate 1616*0Sstevel@tonic-gate /* 1617*0Sstevel@tonic-gate * Create the device arena for toxic (to dtrace/kmdb) mappings. 1618*0Sstevel@tonic-gate */ 1619*0Sstevel@tonic-gate device_arena = vmem_create("device", (void *)toxic_addr, 1620*0Sstevel@tonic-gate toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); 1621*0Sstevel@tonic-gate 1622*0Sstevel@tonic-gate #else /* __i386 */ 1623*0Sstevel@tonic-gate 1624*0Sstevel@tonic-gate /* 1625*0Sstevel@tonic-gate * allocate the bit map that tracks toxic pages 1626*0Sstevel@tonic-gate */ 1627*0Sstevel@tonic-gate toxic_bit_map_len = btop((ulong_t)(ptable_va - kernelbase)); 1628*0Sstevel@tonic-gate PRM_DEBUG(toxic_bit_map_len); 1629*0Sstevel@tonic-gate toxic_bit_map = 1630*0Sstevel@tonic-gate kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP); 1631*0Sstevel@tonic-gate ASSERT(toxic_bit_map != NULL); 1632*0Sstevel@tonic-gate PRM_DEBUG(toxic_bit_map); 1633*0Sstevel@tonic-gate 1634*0Sstevel@tonic-gate #endif /* __i386 */ 1635*0Sstevel@tonic-gate 1636*0Sstevel@tonic-gate 1637*0Sstevel@tonic-gate /* 1638*0Sstevel@tonic-gate * Now that we've got more VA, as well as the ability to allocate from 1639*0Sstevel@tonic-gate * it, tell the debugger. 1640*0Sstevel@tonic-gate */ 1641*0Sstevel@tonic-gate if (boothowto & RB_DEBUG) 1642*0Sstevel@tonic-gate kdi_dvec_memavail(); 1643*0Sstevel@tonic-gate 1644*0Sstevel@tonic-gate /* 1645*0Sstevel@tonic-gate * The following code installs a special page fault handler (#pf) 1646*0Sstevel@tonic-gate * to work around a pentium bug. 1647*0Sstevel@tonic-gate */ 1648*0Sstevel@tonic-gate #if !defined(__amd64) 1649*0Sstevel@tonic-gate if (x86_type == X86_TYPE_P5) { 1650*0Sstevel@tonic-gate gate_desc_t *newidt; 1651*0Sstevel@tonic-gate desctbr_t newidt_r; 1652*0Sstevel@tonic-gate 1653*0Sstevel@tonic-gate if ((newidt = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP)) == NULL) 1654*0Sstevel@tonic-gate panic("failed to install pentium_pftrap"); 1655*0Sstevel@tonic-gate 1656*0Sstevel@tonic-gate bcopy(idt0, newidt, sizeof (idt0)); 1657*0Sstevel@tonic-gate set_gatesegd(&newidt[T_PGFLT], &pentium_pftrap, 1658*0Sstevel@tonic-gate KCS_SEL, 0, SDT_SYSIGT, SEL_KPL); 1659*0Sstevel@tonic-gate 1660*0Sstevel@tonic-gate (void) as_setprot(&kas, (caddr_t)newidt, MMU_PAGESIZE, 1661*0Sstevel@tonic-gate PROT_READ|PROT_EXEC); 1662*0Sstevel@tonic-gate 1663*0Sstevel@tonic-gate newidt_r.dtr_limit = sizeof (idt0) - 1; 1664*0Sstevel@tonic-gate newidt_r.dtr_base = (uintptr_t)newidt; 1665*0Sstevel@tonic-gate CPU->cpu_idt = newidt; 1666*0Sstevel@tonic-gate wr_idtr(&newidt_r); 1667*0Sstevel@tonic-gate } 1668*0Sstevel@tonic-gate #endif /* !__amd64 */ 1669*0Sstevel@tonic-gate 1670*0Sstevel@tonic-gate /* 1671*0Sstevel@tonic-gate * Map page pfn=0 for drivers, such as kd, that need to pick up 1672*0Sstevel@tonic-gate * parameters left there by controllers/BIOS. 1673*0Sstevel@tonic-gate */ 1674*0Sstevel@tonic-gate PRM_POINT("setup up p0_va"); 1675*0Sstevel@tonic-gate p0_va = i86devmap(0, 1, PROT_READ); 1676*0Sstevel@tonic-gate PRM_DEBUG(p0_va); 1677*0Sstevel@tonic-gate 1678*0Sstevel@tonic-gate /* 1679*0Sstevel@tonic-gate * If the following is true, someone has patched phsymem to be less 1680*0Sstevel@tonic-gate * than the number of pages that the system actually has. Remove 1681*0Sstevel@tonic-gate * pages until system memory is limited to the requested amount. 1682*0Sstevel@tonic-gate * Since we have allocated page structures for all pages, we 1683*0Sstevel@tonic-gate * correct the amount of memory we want to remove by the size of 1684*0Sstevel@tonic-gate * the memory used to hold page structures for the non-used pages. 1685*0Sstevel@tonic-gate */ 1686*0Sstevel@tonic-gate if (physmem < npages) { 1687*0Sstevel@tonic-gate uint_t diff; 1688*0Sstevel@tonic-gate offset_t off; 1689*0Sstevel@tonic-gate struct page *pp; 1690*0Sstevel@tonic-gate caddr_t rand_vaddr; 1691*0Sstevel@tonic-gate struct seg kseg; 1692*0Sstevel@tonic-gate 1693*0Sstevel@tonic-gate cmn_err(CE_WARN, "limiting physmem to %lu pages", physmem); 1694*0Sstevel@tonic-gate 1695*0Sstevel@tonic-gate off = 0; 1696*0Sstevel@tonic-gate diff = npages - physmem; 1697*0Sstevel@tonic-gate diff -= mmu_btopr(diff * sizeof (struct page)); 1698*0Sstevel@tonic-gate kseg.s_as = &kas; 1699*0Sstevel@tonic-gate while (diff--) { 1700*0Sstevel@tonic-gate rand_vaddr = (caddr_t) 1701*0Sstevel@tonic-gate (((uintptr_t)&unused_pages_vp >> 7) ^ 1702*0Sstevel@tonic-gate (uintptr_t)((u_offset_t)off >> MMU_PAGESHIFT)); 1703*0Sstevel@tonic-gate pp = page_create_va(&unused_pages_vp, off, MMU_PAGESIZE, 1704*0Sstevel@tonic-gate PG_WAIT | PG_EXCL, &kseg, rand_vaddr); 1705*0Sstevel@tonic-gate if (pp == NULL) { 1706*0Sstevel@tonic-gate panic("limited physmem too much!"); 1707*0Sstevel@tonic-gate /*NOTREACHED*/ 1708*0Sstevel@tonic-gate } 1709*0Sstevel@tonic-gate page_io_unlock(pp); 1710*0Sstevel@tonic-gate page_downgrade(pp); 1711*0Sstevel@tonic-gate availrmem--; 1712*0Sstevel@tonic-gate off += MMU_PAGESIZE; 1713*0Sstevel@tonic-gate } 1714*0Sstevel@tonic-gate } 1715*0Sstevel@tonic-gate 1716*0Sstevel@tonic-gate cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n", 1717*0Sstevel@tonic-gate physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled)); 1718*0Sstevel@tonic-gate 1719*0Sstevel@tonic-gate PRM_POINT("Calling hat_init_finish()..."); 1720*0Sstevel@tonic-gate hat_init_finish(); 1721*0Sstevel@tonic-gate PRM_POINT("hat_init_finish() done"); 1722*0Sstevel@tonic-gate 1723*0Sstevel@tonic-gate /* 1724*0Sstevel@tonic-gate * Initialize the segkp segment type. 1725*0Sstevel@tonic-gate */ 1726*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 1727*0Sstevel@tonic-gate if (!segkp_fromheap) { 1728*0Sstevel@tonic-gate if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize), 1729*0Sstevel@tonic-gate segkp) < 0) { 1730*0Sstevel@tonic-gate panic("startup: cannot attach segkp"); 1731*0Sstevel@tonic-gate /*NOTREACHED*/ 1732*0Sstevel@tonic-gate } 1733*0Sstevel@tonic-gate } else { 1734*0Sstevel@tonic-gate /* 1735*0Sstevel@tonic-gate * For 32 bit x86 systems, we will have segkp under the heap. 1736*0Sstevel@tonic-gate * There will not be a segkp segment. We do, however, need 1737*0Sstevel@tonic-gate * to fill in the seg structure. 1738*0Sstevel@tonic-gate */ 1739*0Sstevel@tonic-gate segkp->s_as = &kas; 1740*0Sstevel@tonic-gate } 1741*0Sstevel@tonic-gate if (segkp_create(segkp) != 0) { 1742*0Sstevel@tonic-gate panic("startup: segkp_create failed"); 1743*0Sstevel@tonic-gate /*NOTREACHED*/ 1744*0Sstevel@tonic-gate } 1745*0Sstevel@tonic-gate PRM_DEBUG(segkp); 1746*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 1747*0Sstevel@tonic-gate 1748*0Sstevel@tonic-gate /* 1749*0Sstevel@tonic-gate * kpm segment 1750*0Sstevel@tonic-gate */ 1751*0Sstevel@tonic-gate segmap_kpm = 0; 1752*0Sstevel@tonic-gate if (kpm_desired) { 1753*0Sstevel@tonic-gate kpm_init(); 1754*0Sstevel@tonic-gate kpm_enable = 1; 1755*0Sstevel@tonic-gate } 1756*0Sstevel@tonic-gate 1757*0Sstevel@tonic-gate /* 1758*0Sstevel@tonic-gate * Now create segmap segment. 1759*0Sstevel@tonic-gate */ 1760*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 1761*0Sstevel@tonic-gate if (seg_attach(&kas, (caddr_t)segkmap_start, segmapsize, segkmap) < 0) { 1762*0Sstevel@tonic-gate panic("cannot attach segkmap"); 1763*0Sstevel@tonic-gate /*NOTREACHED*/ 1764*0Sstevel@tonic-gate } 1765*0Sstevel@tonic-gate PRM_DEBUG(segkmap); 1766*0Sstevel@tonic-gate 1767*0Sstevel@tonic-gate /* 1768*0Sstevel@tonic-gate * The 64 bit HAT permanently maps only segmap's page tables. 1769*0Sstevel@tonic-gate * The 32 bit HAT maps the heap's page tables too. 1770*0Sstevel@tonic-gate */ 1771*0Sstevel@tonic-gate #if defined(__amd64) 1772*0Sstevel@tonic-gate hat_kmap_init(segkmap_start, segmapsize); 1773*0Sstevel@tonic-gate #else /* __i386 */ 1774*0Sstevel@tonic-gate ASSERT(segkmap_start + segmapsize == (uintptr_t)final_kernelheap); 1775*0Sstevel@tonic-gate hat_kmap_init(segkmap_start, (uintptr_t)ekernelheap - segkmap_start); 1776*0Sstevel@tonic-gate #endif /* __i386 */ 1777*0Sstevel@tonic-gate 1778*0Sstevel@tonic-gate a.prot = PROT_READ | PROT_WRITE; 1779*0Sstevel@tonic-gate a.shmsize = 0; 1780*0Sstevel@tonic-gate a.nfreelist = segmapfreelists; 1781*0Sstevel@tonic-gate 1782*0Sstevel@tonic-gate if (segmap_create(segkmap, (caddr_t)&a) != 0) 1783*0Sstevel@tonic-gate panic("segmap_create segkmap"); 1784*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 1785*0Sstevel@tonic-gate 1786*0Sstevel@tonic-gate setup_vaddr_for_ppcopy(CPU); 1787*0Sstevel@tonic-gate 1788*0Sstevel@tonic-gate segdev_init(); 1789*0Sstevel@tonic-gate pmem_init(); 1790*0Sstevel@tonic-gate PRM_POINT("startup_vm() done"); 1791*0Sstevel@tonic-gate } 1792*0Sstevel@tonic-gate 1793*0Sstevel@tonic-gate static void 1794*0Sstevel@tonic-gate startup_end(void) 1795*0Sstevel@tonic-gate { 1796*0Sstevel@tonic-gate extern void setx86isalist(void); 1797*0Sstevel@tonic-gate 1798*0Sstevel@tonic-gate PRM_POINT("startup_end() starting..."); 1799*0Sstevel@tonic-gate 1800*0Sstevel@tonic-gate /* 1801*0Sstevel@tonic-gate * Perform tasks that get done after most of the VM 1802*0Sstevel@tonic-gate * initialization has been done but before the clock 1803*0Sstevel@tonic-gate * and other devices get started. 1804*0Sstevel@tonic-gate */ 1805*0Sstevel@tonic-gate kern_setup1(); 1806*0Sstevel@tonic-gate 1807*0Sstevel@tonic-gate /* 1808*0Sstevel@tonic-gate * Perform CPC initialization for this CPU. 1809*0Sstevel@tonic-gate */ 1810*0Sstevel@tonic-gate kcpc_hw_init(CPU); 1811*0Sstevel@tonic-gate 1812*0Sstevel@tonic-gate #if defined(__amd64) 1813*0Sstevel@tonic-gate /* 1814*0Sstevel@tonic-gate * Validate support for syscall/sysret 1815*0Sstevel@tonic-gate * XX64 -- include SSE, SSE2, etc. here too? 1816*0Sstevel@tonic-gate */ 1817*0Sstevel@tonic-gate if ((x86_feature & X86_ASYSC) == 0) { 1818*0Sstevel@tonic-gate cmn_err(CE_WARN, 1819*0Sstevel@tonic-gate "cpu%d does not support syscall/sysret", CPU->cpu_id); 1820*0Sstevel@tonic-gate } 1821*0Sstevel@tonic-gate #endif 1822*0Sstevel@tonic-gate /* 1823*0Sstevel@tonic-gate * Configure the system. 1824*0Sstevel@tonic-gate */ 1825*0Sstevel@tonic-gate PRM_POINT("Calling configure()..."); 1826*0Sstevel@tonic-gate configure(); /* set up devices */ 1827*0Sstevel@tonic-gate PRM_POINT("configure() done"); 1828*0Sstevel@tonic-gate 1829*0Sstevel@tonic-gate /* 1830*0Sstevel@tonic-gate * Set the isa_list string to the defined instruction sets we 1831*0Sstevel@tonic-gate * support. 1832*0Sstevel@tonic-gate */ 1833*0Sstevel@tonic-gate setx86isalist(); 1834*0Sstevel@tonic-gate init_intr_threads(CPU); 1835*0Sstevel@tonic-gate psm_install(); 1836*0Sstevel@tonic-gate 1837*0Sstevel@tonic-gate /* 1838*0Sstevel@tonic-gate * We're done with bootops. We don't unmap the bootstrap yet because 1839*0Sstevel@tonic-gate * we're still using bootsvcs. 1840*0Sstevel@tonic-gate */ 1841*0Sstevel@tonic-gate PRM_POINT("zeroing out bootops"); 1842*0Sstevel@tonic-gate *bootopsp = (struct bootops *)0; 1843*0Sstevel@tonic-gate bootops = (struct bootops *)NULL; 1844*0Sstevel@tonic-gate 1845*0Sstevel@tonic-gate PRM_POINT("Enabling interrupts"); 1846*0Sstevel@tonic-gate (*picinitf)(); 1847*0Sstevel@tonic-gate sti(); 1848*0Sstevel@tonic-gate 1849*0Sstevel@tonic-gate (void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1, 1850*0Sstevel@tonic-gate "softlevel1", NULL, NULL); /* XXX to be moved later */ 1851*0Sstevel@tonic-gate 1852*0Sstevel@tonic-gate PRM_POINT("startup_end() done"); 1853*0Sstevel@tonic-gate } 1854*0Sstevel@tonic-gate 1855*0Sstevel@tonic-gate extern char hw_serial[]; 1856*0Sstevel@tonic-gate char *_hs1107 = hw_serial; 1857*0Sstevel@tonic-gate ulong_t _bdhs34; 1858*0Sstevel@tonic-gate 1859*0Sstevel@tonic-gate void 1860*0Sstevel@tonic-gate post_startup(void) 1861*0Sstevel@tonic-gate { 1862*0Sstevel@tonic-gate extern void memscrub_init(void); 1863*0Sstevel@tonic-gate 1864*0Sstevel@tonic-gate /* 1865*0Sstevel@tonic-gate * Set the system wide, processor-specific flags to be passed 1866*0Sstevel@tonic-gate * to userland via the aux vector for performance hints and 1867*0Sstevel@tonic-gate * instruction set extensions. 1868*0Sstevel@tonic-gate */ 1869*0Sstevel@tonic-gate bind_hwcap(); 1870*0Sstevel@tonic-gate 1871*0Sstevel@tonic-gate /* 1872*0Sstevel@tonic-gate * Startup memory scrubber. 1873*0Sstevel@tonic-gate */ 1874*0Sstevel@tonic-gate (void) memscrub_init(); 1875*0Sstevel@tonic-gate 1876*0Sstevel@tonic-gate /* 1877*0Sstevel@tonic-gate * Perform forceloading tasks for /etc/system. 1878*0Sstevel@tonic-gate */ 1879*0Sstevel@tonic-gate (void) mod_sysctl(SYS_FORCELOAD, NULL); 1880*0Sstevel@tonic-gate 1881*0Sstevel@tonic-gate /* 1882*0Sstevel@tonic-gate * complete mmu initialization, now that kernel and critical 1883*0Sstevel@tonic-gate * modules have been loaded. 1884*0Sstevel@tonic-gate */ 1885*0Sstevel@tonic-gate (void) post_startup_mmu_initialization(); 1886*0Sstevel@tonic-gate 1887*0Sstevel@tonic-gate /* 1888*0Sstevel@tonic-gate * ON4.0: Force /proc module in until clock interrupt handle fixed 1889*0Sstevel@tonic-gate * ON4.0: This must be fixed or restated in /etc/systems. 1890*0Sstevel@tonic-gate */ 1891*0Sstevel@tonic-gate (void) modload("fs", "procfs"); 1892*0Sstevel@tonic-gate 1893*0Sstevel@tonic-gate #if defined(__i386) 1894*0Sstevel@tonic-gate /* 1895*0Sstevel@tonic-gate * Check for required functional Floating Point hardware, 1896*0Sstevel@tonic-gate * unless FP hardware explicitly disabled. 1897*0Sstevel@tonic-gate */ 1898*0Sstevel@tonic-gate if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO)) 1899*0Sstevel@tonic-gate halt("No working FP hardware found"); 1900*0Sstevel@tonic-gate #endif 1901*0Sstevel@tonic-gate 1902*0Sstevel@tonic-gate maxmem = freemem; 1903*0Sstevel@tonic-gate 1904*0Sstevel@tonic-gate add_cpunode2devtree(CPU->cpu_id, CPU->cpu_m.mcpu_cpi); 1905*0Sstevel@tonic-gate 1906*0Sstevel@tonic-gate /* 1907*0Sstevel@tonic-gate * Perform the formal initialization of the boot chip, 1908*0Sstevel@tonic-gate * and associate the boot cpu with it. 1909*0Sstevel@tonic-gate * This must be done after the cpu node for CPU has been 1910*0Sstevel@tonic-gate * added to the device tree, when the necessary probing to 1911*0Sstevel@tonic-gate * know the chip type and chip "id" is performed. 1912*0Sstevel@tonic-gate */ 1913*0Sstevel@tonic-gate chip_cpu_init(CPU); 1914*0Sstevel@tonic-gate chip_cpu_assign(CPU); 1915*0Sstevel@tonic-gate } 1916*0Sstevel@tonic-gate 1917*0Sstevel@tonic-gate static int 1918*0Sstevel@tonic-gate pp_in_ramdisk(page_t *pp) 1919*0Sstevel@tonic-gate { 1920*0Sstevel@tonic-gate extern uint64_t ramdisk_start, ramdisk_end; 1921*0Sstevel@tonic-gate 1922*0Sstevel@tonic-gate return ((pp->p_pagenum >= btop(ramdisk_start)) && 1923*0Sstevel@tonic-gate (pp->p_pagenum < btopr(ramdisk_end))); 1924*0Sstevel@tonic-gate } 1925*0Sstevel@tonic-gate 1926*0Sstevel@tonic-gate void 1927*0Sstevel@tonic-gate release_bootstrap(void) 1928*0Sstevel@tonic-gate { 1929*0Sstevel@tonic-gate int root_is_ramdisk; 1930*0Sstevel@tonic-gate pfn_t pfn; 1931*0Sstevel@tonic-gate page_t *pp; 1932*0Sstevel@tonic-gate extern void kobj_boot_unmountroot(void); 1933*0Sstevel@tonic-gate extern dev_t rootdev; 1934*0Sstevel@tonic-gate 1935*0Sstevel@tonic-gate /* unmount boot ramdisk and release kmem usage */ 1936*0Sstevel@tonic-gate kobj_boot_unmountroot(); 1937*0Sstevel@tonic-gate 1938*0Sstevel@tonic-gate /* 1939*0Sstevel@tonic-gate * We're finished using the boot loader so free its pages. 1940*0Sstevel@tonic-gate */ 1941*0Sstevel@tonic-gate PRM_POINT("Unmapping lower boot pages"); 1942*0Sstevel@tonic-gate clear_boot_mappings(0, kernelbase); 1943*0Sstevel@tonic-gate #if defined(__amd64) 1944*0Sstevel@tonic-gate PRM_POINT("Unmapping upper boot pages"); 1945*0Sstevel@tonic-gate clear_boot_mappings(BOOT_DOUBLEMAP_BASE, 1946*0Sstevel@tonic-gate BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE); 1947*0Sstevel@tonic-gate #endif 1948*0Sstevel@tonic-gate 1949*0Sstevel@tonic-gate /* 1950*0Sstevel@tonic-gate * If root isn't on ramdisk, destroy the hardcoded 1951*0Sstevel@tonic-gate * ramdisk node now and release the memory. Else, 1952*0Sstevel@tonic-gate * ramdisk memory is kept in rd_pages. 1953*0Sstevel@tonic-gate */ 1954*0Sstevel@tonic-gate root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk")); 1955*0Sstevel@tonic-gate if (!root_is_ramdisk) { 1956*0Sstevel@tonic-gate dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0); 1957*0Sstevel@tonic-gate ASSERT(dip && ddi_get_parent(dip) == ddi_root_node()); 1958*0Sstevel@tonic-gate ndi_rele_devi(dip); /* held from ddi_find_devinfo */ 1959*0Sstevel@tonic-gate (void) ddi_remove_child(dip, 0); 1960*0Sstevel@tonic-gate } 1961*0Sstevel@tonic-gate 1962*0Sstevel@tonic-gate PRM_POINT("Releasing boot pages"); 1963*0Sstevel@tonic-gate while (bootpages) { 1964*0Sstevel@tonic-gate pp = bootpages; 1965*0Sstevel@tonic-gate bootpages = pp->p_next; 1966*0Sstevel@tonic-gate if (root_is_ramdisk && pp_in_ramdisk(pp)) { 1967*0Sstevel@tonic-gate pp->p_next = rd_pages; 1968*0Sstevel@tonic-gate rd_pages = pp; 1969*0Sstevel@tonic-gate continue; 1970*0Sstevel@tonic-gate } 1971*0Sstevel@tonic-gate pp->p_next = (struct page *)0; 1972*0Sstevel@tonic-gate page_free(pp, 1); 1973*0Sstevel@tonic-gate } 1974*0Sstevel@tonic-gate 1975*0Sstevel@tonic-gate /* 1976*0Sstevel@tonic-gate * Find 1 page below 1 MB so that other processors can boot up. 1977*0Sstevel@tonic-gate * Make sure it has a kernel VA as well as a 1:1 mapping. 1978*0Sstevel@tonic-gate * We should have just free'd one up. 1979*0Sstevel@tonic-gate */ 1980*0Sstevel@tonic-gate if (use_mp) { 1981*0Sstevel@tonic-gate for (pfn = 1; pfn < btop(1*1024*1024); pfn++) { 1982*0Sstevel@tonic-gate if (page_numtopp_alloc(pfn) == NULL) 1983*0Sstevel@tonic-gate continue; 1984*0Sstevel@tonic-gate rm_platter_va = i86devmap(pfn, 1, 1985*0Sstevel@tonic-gate PROT_READ | PROT_WRITE | PROT_EXEC); 1986*0Sstevel@tonic-gate rm_platter_pa = ptob(pfn); 1987*0Sstevel@tonic-gate hat_devload(kas.a_hat, 1988*0Sstevel@tonic-gate (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 1989*0Sstevel@tonic-gate pfn, PROT_READ | PROT_WRITE | PROT_EXEC, 1990*0Sstevel@tonic-gate HAT_LOAD_NOCONSIST); 1991*0Sstevel@tonic-gate break; 1992*0Sstevel@tonic-gate } 1993*0Sstevel@tonic-gate if (pfn == btop(1*1024*1024)) 1994*0Sstevel@tonic-gate panic("No page available for starting " 1995*0Sstevel@tonic-gate "other processors"); 1996*0Sstevel@tonic-gate } 1997*0Sstevel@tonic-gate 1998*0Sstevel@tonic-gate #if defined(__amd64) 1999*0Sstevel@tonic-gate PRM_POINT("Returning boot's VA space to kernel heap"); 2000*0Sstevel@tonic-gate if (kmem_setaside != NULL) 2001*0Sstevel@tonic-gate vmem_free(heap_arena, kmem_setaside, BOOT_DOUBLEMAP_SIZE); 2002*0Sstevel@tonic-gate #endif 2003*0Sstevel@tonic-gate } 2004*0Sstevel@tonic-gate 2005*0Sstevel@tonic-gate /* 2006*0Sstevel@tonic-gate * Initialize the platform-specific parts of a page_t. 2007*0Sstevel@tonic-gate */ 2008*0Sstevel@tonic-gate void 2009*0Sstevel@tonic-gate add_physmem_cb(page_t *pp, pfn_t pnum) 2010*0Sstevel@tonic-gate { 2011*0Sstevel@tonic-gate pp->p_pagenum = pnum; 2012*0Sstevel@tonic-gate pp->p_mapping = NULL; 2013*0Sstevel@tonic-gate pp->p_embed = 0; 2014*0Sstevel@tonic-gate pp->p_share = 0; 2015*0Sstevel@tonic-gate pp->p_mlentry = 0; 2016*0Sstevel@tonic-gate } 2017*0Sstevel@tonic-gate 2018*0Sstevel@tonic-gate /* 2019*0Sstevel@tonic-gate * kphysm_init() initializes physical memory. 2020*0Sstevel@tonic-gate */ 2021*0Sstevel@tonic-gate static pgcnt_t 2022*0Sstevel@tonic-gate kphysm_init( 2023*0Sstevel@tonic-gate page_t *inpp, 2024*0Sstevel@tonic-gate struct memseg *memsegp, 2025*0Sstevel@tonic-gate pgcnt_t start, 2026*0Sstevel@tonic-gate pgcnt_t npages) 2027*0Sstevel@tonic-gate { 2028*0Sstevel@tonic-gate struct memlist *pmem; 2029*0Sstevel@tonic-gate struct memseg *cur_memseg; 2030*0Sstevel@tonic-gate struct memseg **memsegpp; 2031*0Sstevel@tonic-gate pfn_t base_pfn; 2032*0Sstevel@tonic-gate pgcnt_t num; 2033*0Sstevel@tonic-gate pgcnt_t total_skipped = 0; 2034*0Sstevel@tonic-gate pgcnt_t skipping = 0; 2035*0Sstevel@tonic-gate pgcnt_t pages_done = 0; 2036*0Sstevel@tonic-gate pgcnt_t largepgcnt; 2037*0Sstevel@tonic-gate uint64_t addr; 2038*0Sstevel@tonic-gate uint64_t size; 2039*0Sstevel@tonic-gate page_t *pp = inpp; 2040*0Sstevel@tonic-gate int dobreak = 0; 2041*0Sstevel@tonic-gate extern pfn_t ddiphysmin; 2042*0Sstevel@tonic-gate 2043*0Sstevel@tonic-gate ASSERT(page_hash != NULL && page_hashsz != 0); 2044*0Sstevel@tonic-gate 2045*0Sstevel@tonic-gate for (cur_memseg = memsegp; cur_memseg->pages != NULL; cur_memseg++); 2046*0Sstevel@tonic-gate ASSERT(cur_memseg == memsegp || start > 0); 2047*0Sstevel@tonic-gate 2048*0Sstevel@tonic-gate for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2049*0Sstevel@tonic-gate /* 2050*0Sstevel@tonic-gate * In a 32 bit kernel can't use higher memory if we're 2051*0Sstevel@tonic-gate * not booting in PAE mode. This check takes care of that. 2052*0Sstevel@tonic-gate */ 2053*0Sstevel@tonic-gate addr = pmem->address; 2054*0Sstevel@tonic-gate size = pmem->size; 2055*0Sstevel@tonic-gate if (btop(addr) > physmax) 2056*0Sstevel@tonic-gate continue; 2057*0Sstevel@tonic-gate 2058*0Sstevel@tonic-gate /* 2059*0Sstevel@tonic-gate * align addr and size - they may not be at page boundaries 2060*0Sstevel@tonic-gate */ 2061*0Sstevel@tonic-gate if ((addr & MMU_PAGEOFFSET) != 0) { 2062*0Sstevel@tonic-gate addr += MMU_PAGEOFFSET; 2063*0Sstevel@tonic-gate addr &= ~(uint64_t)MMU_PAGEOFFSET; 2064*0Sstevel@tonic-gate size -= addr - pmem->address; 2065*0Sstevel@tonic-gate } 2066*0Sstevel@tonic-gate 2067*0Sstevel@tonic-gate /* only process pages below physmax */ 2068*0Sstevel@tonic-gate if (btop(addr + size) > physmax) 2069*0Sstevel@tonic-gate size = ptob(physmax - btop(addr)); 2070*0Sstevel@tonic-gate 2071*0Sstevel@tonic-gate num = btop(size); 2072*0Sstevel@tonic-gate if (num == 0) 2073*0Sstevel@tonic-gate continue; 2074*0Sstevel@tonic-gate 2075*0Sstevel@tonic-gate if (total_skipped < start) { 2076*0Sstevel@tonic-gate if (start - total_skipped > num) { 2077*0Sstevel@tonic-gate total_skipped += num; 2078*0Sstevel@tonic-gate continue; 2079*0Sstevel@tonic-gate } 2080*0Sstevel@tonic-gate skipping = start - total_skipped; 2081*0Sstevel@tonic-gate num -= skipping; 2082*0Sstevel@tonic-gate addr += (MMU_PAGESIZE * skipping); 2083*0Sstevel@tonic-gate total_skipped = start; 2084*0Sstevel@tonic-gate } 2085*0Sstevel@tonic-gate if (num == 0) 2086*0Sstevel@tonic-gate continue; 2087*0Sstevel@tonic-gate 2088*0Sstevel@tonic-gate if (num > npages) 2089*0Sstevel@tonic-gate num = npages; 2090*0Sstevel@tonic-gate 2091*0Sstevel@tonic-gate npages -= num; 2092*0Sstevel@tonic-gate pages_done += num; 2093*0Sstevel@tonic-gate base_pfn = btop(addr); 2094*0Sstevel@tonic-gate 2095*0Sstevel@tonic-gate /* 2096*0Sstevel@tonic-gate * If the caller didn't provide space for the page 2097*0Sstevel@tonic-gate * structures, carve them out of the memseg they will 2098*0Sstevel@tonic-gate * represent. 2099*0Sstevel@tonic-gate */ 2100*0Sstevel@tonic-gate if (pp == NULL) { 2101*0Sstevel@tonic-gate pgcnt_t pp_pgs; 2102*0Sstevel@tonic-gate 2103*0Sstevel@tonic-gate if (num <= 1) 2104*0Sstevel@tonic-gate continue; 2105*0Sstevel@tonic-gate 2106*0Sstevel@tonic-gate /* 2107*0Sstevel@tonic-gate * Compute how many of the pages we need to use for 2108*0Sstevel@tonic-gate * page_ts 2109*0Sstevel@tonic-gate */ 2110*0Sstevel@tonic-gate pp_pgs = (num * sizeof (page_t)) / MMU_PAGESIZE + 1; 2111*0Sstevel@tonic-gate while (mmu_ptob(pp_pgs - 1) / sizeof (page_t) >= 2112*0Sstevel@tonic-gate num - pp_pgs + 1) 2113*0Sstevel@tonic-gate --pp_pgs; 2114*0Sstevel@tonic-gate PRM_DEBUG(pp_pgs); 2115*0Sstevel@tonic-gate 2116*0Sstevel@tonic-gate pp = vmem_alloc(heap_arena, mmu_ptob(pp_pgs), 2117*0Sstevel@tonic-gate VM_NOSLEEP); 2118*0Sstevel@tonic-gate if (pp == NULL) { 2119*0Sstevel@tonic-gate cmn_err(CE_WARN, "Unable to add %ld pages to " 2120*0Sstevel@tonic-gate "the system.", num); 2121*0Sstevel@tonic-gate continue; 2122*0Sstevel@tonic-gate } 2123*0Sstevel@tonic-gate 2124*0Sstevel@tonic-gate hat_devload(kas.a_hat, (void *)pp, mmu_ptob(pp_pgs), 2125*0Sstevel@tonic-gate base_pfn, PROT_READ | PROT_WRITE | HAT_UNORDERED_OK, 2126*0Sstevel@tonic-gate HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 2127*0Sstevel@tonic-gate bzero(pp, mmu_ptob(pp_pgs)); 2128*0Sstevel@tonic-gate num -= pp_pgs; 2129*0Sstevel@tonic-gate base_pfn += pp_pgs; 2130*0Sstevel@tonic-gate } 2131*0Sstevel@tonic-gate 2132*0Sstevel@tonic-gate if (prom_debug) 2133*0Sstevel@tonic-gate prom_printf("MEMSEG addr=0x%" PRIx64 2134*0Sstevel@tonic-gate " pgs=0x%lx pfn 0x%lx-0x%lx\n", 2135*0Sstevel@tonic-gate addr, num, base_pfn, base_pfn + num); 2136*0Sstevel@tonic-gate 2137*0Sstevel@tonic-gate /* 2138*0Sstevel@tonic-gate * drop pages below ddiphysmin to simplify ddi memory 2139*0Sstevel@tonic-gate * allocation with non-zero addr_lo requests. 2140*0Sstevel@tonic-gate */ 2141*0Sstevel@tonic-gate if (base_pfn < ddiphysmin) { 2142*0Sstevel@tonic-gate if (base_pfn + num <= ddiphysmin) { 2143*0Sstevel@tonic-gate /* drop entire range below ddiphysmin */ 2144*0Sstevel@tonic-gate continue; 2145*0Sstevel@tonic-gate } 2146*0Sstevel@tonic-gate /* adjust range to ddiphysmin */ 2147*0Sstevel@tonic-gate pp += (ddiphysmin - base_pfn); 2148*0Sstevel@tonic-gate num -= (ddiphysmin - base_pfn); 2149*0Sstevel@tonic-gate base_pfn = ddiphysmin; 2150*0Sstevel@tonic-gate } 2151*0Sstevel@tonic-gate /* 2152*0Sstevel@tonic-gate * Build the memsegs entry 2153*0Sstevel@tonic-gate */ 2154*0Sstevel@tonic-gate cur_memseg->pages = pp; 2155*0Sstevel@tonic-gate cur_memseg->epages = pp + num; 2156*0Sstevel@tonic-gate cur_memseg->pages_base = base_pfn; 2157*0Sstevel@tonic-gate cur_memseg->pages_end = base_pfn + num; 2158*0Sstevel@tonic-gate 2159*0Sstevel@tonic-gate /* 2160*0Sstevel@tonic-gate * insert in memseg list in decreasing pfn range order. 2161*0Sstevel@tonic-gate * Low memory is typically more fragmented such that this 2162*0Sstevel@tonic-gate * ordering keeps the larger ranges at the front of the list 2163*0Sstevel@tonic-gate * for code that searches memseg. 2164*0Sstevel@tonic-gate */ 2165*0Sstevel@tonic-gate memsegpp = &memsegs; 2166*0Sstevel@tonic-gate for (;;) { 2167*0Sstevel@tonic-gate if (*memsegpp == NULL) { 2168*0Sstevel@tonic-gate /* empty memsegs */ 2169*0Sstevel@tonic-gate memsegs = cur_memseg; 2170*0Sstevel@tonic-gate break; 2171*0Sstevel@tonic-gate } 2172*0Sstevel@tonic-gate /* check for continuity with start of memsegpp */ 2173*0Sstevel@tonic-gate if (cur_memseg->pages_end == (*memsegpp)->pages_base) { 2174*0Sstevel@tonic-gate if (cur_memseg->epages == (*memsegpp)->pages) { 2175*0Sstevel@tonic-gate /* 2176*0Sstevel@tonic-gate * contiguous pfn and page_t's. Merge 2177*0Sstevel@tonic-gate * cur_memseg into *memsegpp. Drop 2178*0Sstevel@tonic-gate * cur_memseg 2179*0Sstevel@tonic-gate */ 2180*0Sstevel@tonic-gate (*memsegpp)->pages_base = 2181*0Sstevel@tonic-gate cur_memseg->pages_base; 2182*0Sstevel@tonic-gate (*memsegpp)->pages = 2183*0Sstevel@tonic-gate cur_memseg->pages; 2184*0Sstevel@tonic-gate /* 2185*0Sstevel@tonic-gate * check if contiguous with the end of 2186*0Sstevel@tonic-gate * the next memseg. 2187*0Sstevel@tonic-gate */ 2188*0Sstevel@tonic-gate if ((*memsegpp)->next && 2189*0Sstevel@tonic-gate ((*memsegpp)->pages_base == 2190*0Sstevel@tonic-gate (*memsegpp)->next->pages_end)) { 2191*0Sstevel@tonic-gate cur_memseg = *memsegpp; 2192*0Sstevel@tonic-gate memsegpp = &((*memsegpp)->next); 2193*0Sstevel@tonic-gate dobreak = 1; 2194*0Sstevel@tonic-gate } else { 2195*0Sstevel@tonic-gate break; 2196*0Sstevel@tonic-gate } 2197*0Sstevel@tonic-gate } else { 2198*0Sstevel@tonic-gate /* 2199*0Sstevel@tonic-gate * contiguous pfn but not page_t's. 2200*0Sstevel@tonic-gate * drop last pfn/page_t in cur_memseg 2201*0Sstevel@tonic-gate * to prevent creation of large pages 2202*0Sstevel@tonic-gate * with noncontiguous page_t's if not 2203*0Sstevel@tonic-gate * aligned to largest page boundary. 2204*0Sstevel@tonic-gate */ 2205*0Sstevel@tonic-gate largepgcnt = page_get_pagecnt( 2206*0Sstevel@tonic-gate page_num_pagesizes() - 1); 2207*0Sstevel@tonic-gate 2208*0Sstevel@tonic-gate if (cur_memseg->pages_end & 2209*0Sstevel@tonic-gate (largepgcnt - 1)) { 2210*0Sstevel@tonic-gate num--; 2211*0Sstevel@tonic-gate cur_memseg->epages--; 2212*0Sstevel@tonic-gate cur_memseg->pages_end--; 2213*0Sstevel@tonic-gate } 2214*0Sstevel@tonic-gate } 2215*0Sstevel@tonic-gate } 2216*0Sstevel@tonic-gate 2217*0Sstevel@tonic-gate /* check for continuity with end of memsegpp */ 2218*0Sstevel@tonic-gate if (cur_memseg->pages_base == (*memsegpp)->pages_end) { 2219*0Sstevel@tonic-gate if (cur_memseg->pages == (*memsegpp)->epages) { 2220*0Sstevel@tonic-gate /* 2221*0Sstevel@tonic-gate * contiguous pfn and page_t's. Merge 2222*0Sstevel@tonic-gate * cur_memseg into *memsegpp. Drop 2223*0Sstevel@tonic-gate * cur_memseg. 2224*0Sstevel@tonic-gate */ 2225*0Sstevel@tonic-gate if (dobreak) { 2226*0Sstevel@tonic-gate /* merge previously done */ 2227*0Sstevel@tonic-gate cur_memseg->pages = 2228*0Sstevel@tonic-gate (*memsegpp)->pages; 2229*0Sstevel@tonic-gate cur_memseg->pages_base = 2230*0Sstevel@tonic-gate (*memsegpp)->pages_base; 2231*0Sstevel@tonic-gate cur_memseg->next = 2232*0Sstevel@tonic-gate (*memsegpp)->next; 2233*0Sstevel@tonic-gate } else { 2234*0Sstevel@tonic-gate (*memsegpp)->pages_end = 2235*0Sstevel@tonic-gate cur_memseg->pages_end; 2236*0Sstevel@tonic-gate (*memsegpp)->epages = 2237*0Sstevel@tonic-gate cur_memseg->epages; 2238*0Sstevel@tonic-gate } 2239*0Sstevel@tonic-gate break; 2240*0Sstevel@tonic-gate } 2241*0Sstevel@tonic-gate /* 2242*0Sstevel@tonic-gate * contiguous pfn but not page_t's. 2243*0Sstevel@tonic-gate * drop first pfn/page_t in cur_memseg 2244*0Sstevel@tonic-gate * to prevent creation of large pages 2245*0Sstevel@tonic-gate * with noncontiguous page_t's if not 2246*0Sstevel@tonic-gate * aligned to largest page boundary. 2247*0Sstevel@tonic-gate */ 2248*0Sstevel@tonic-gate largepgcnt = page_get_pagecnt( 2249*0Sstevel@tonic-gate page_num_pagesizes() - 1); 2250*0Sstevel@tonic-gate if (base_pfn & (largepgcnt - 1)) { 2251*0Sstevel@tonic-gate num--; 2252*0Sstevel@tonic-gate base_pfn++; 2253*0Sstevel@tonic-gate cur_memseg->pages++; 2254*0Sstevel@tonic-gate cur_memseg->pages_base++; 2255*0Sstevel@tonic-gate pp = cur_memseg->pages; 2256*0Sstevel@tonic-gate } 2257*0Sstevel@tonic-gate if (dobreak) 2258*0Sstevel@tonic-gate break; 2259*0Sstevel@tonic-gate } 2260*0Sstevel@tonic-gate 2261*0Sstevel@tonic-gate if (cur_memseg->pages_base >= 2262*0Sstevel@tonic-gate (*memsegpp)->pages_end) { 2263*0Sstevel@tonic-gate cur_memseg->next = *memsegpp; 2264*0Sstevel@tonic-gate *memsegpp = cur_memseg; 2265*0Sstevel@tonic-gate break; 2266*0Sstevel@tonic-gate } 2267*0Sstevel@tonic-gate if ((*memsegpp)->next == NULL) { 2268*0Sstevel@tonic-gate cur_memseg->next = NULL; 2269*0Sstevel@tonic-gate (*memsegpp)->next = cur_memseg; 2270*0Sstevel@tonic-gate break; 2271*0Sstevel@tonic-gate } 2272*0Sstevel@tonic-gate memsegpp = &((*memsegpp)->next); 2273*0Sstevel@tonic-gate ASSERT(*memsegpp != NULL); 2274*0Sstevel@tonic-gate } 2275*0Sstevel@tonic-gate 2276*0Sstevel@tonic-gate /* 2277*0Sstevel@tonic-gate * add_physmem() initializes the PSM part of the page 2278*0Sstevel@tonic-gate * struct by calling the PSM back with add_physmem_cb(). 2279*0Sstevel@tonic-gate * In addition it coalesces pages into larger pages as 2280*0Sstevel@tonic-gate * it initializes them. 2281*0Sstevel@tonic-gate */ 2282*0Sstevel@tonic-gate add_physmem(pp, num, base_pfn); 2283*0Sstevel@tonic-gate cur_memseg++; 2284*0Sstevel@tonic-gate availrmem_initial += num; 2285*0Sstevel@tonic-gate availrmem += num; 2286*0Sstevel@tonic-gate 2287*0Sstevel@tonic-gate /* 2288*0Sstevel@tonic-gate * If the caller provided the page frames to us, then 2289*0Sstevel@tonic-gate * advance in that list. Otherwise, prepare to allocate 2290*0Sstevel@tonic-gate * our own page frames for the next memseg. 2291*0Sstevel@tonic-gate */ 2292*0Sstevel@tonic-gate pp = (inpp == NULL) ? NULL : pp + num; 2293*0Sstevel@tonic-gate } 2294*0Sstevel@tonic-gate 2295*0Sstevel@tonic-gate PRM_DEBUG(availrmem_initial); 2296*0Sstevel@tonic-gate PRM_DEBUG(availrmem); 2297*0Sstevel@tonic-gate PRM_DEBUG(freemem); 2298*0Sstevel@tonic-gate build_pfn_hash(); 2299*0Sstevel@tonic-gate return (pages_done); 2300*0Sstevel@tonic-gate } 2301*0Sstevel@tonic-gate 2302*0Sstevel@tonic-gate /* 2303*0Sstevel@tonic-gate * Kernel VM initialization. 2304*0Sstevel@tonic-gate */ 2305*0Sstevel@tonic-gate static void 2306*0Sstevel@tonic-gate kvm_init(void) 2307*0Sstevel@tonic-gate { 2308*0Sstevel@tonic-gate #ifdef DEBUG 2309*0Sstevel@tonic-gate extern void _start(); 2310*0Sstevel@tonic-gate 2311*0Sstevel@tonic-gate ASSERT((caddr_t)_start == s_text); 2312*0Sstevel@tonic-gate #endif 2313*0Sstevel@tonic-gate ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0); 2314*0Sstevel@tonic-gate 2315*0Sstevel@tonic-gate /* 2316*0Sstevel@tonic-gate * Put the kernel segments in kernel address space. 2317*0Sstevel@tonic-gate */ 2318*0Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER); 2319*0Sstevel@tonic-gate as_avlinit(&kas); 2320*0Sstevel@tonic-gate 2321*0Sstevel@tonic-gate (void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg); 2322*0Sstevel@tonic-gate (void) segkmem_create(&ktextseg); 2323*0Sstevel@tonic-gate 2324*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc); 2325*0Sstevel@tonic-gate (void) segkmem_create(&kvalloc); 2326*0Sstevel@tonic-gate 2327*0Sstevel@tonic-gate /* 2328*0Sstevel@tonic-gate * We're about to map out /boot. This is the beginning of the 2329*0Sstevel@tonic-gate * system resource management transition. We can no longer 2330*0Sstevel@tonic-gate * call into /boot for I/O or memory allocations. 2331*0Sstevel@tonic-gate * 2332*0Sstevel@tonic-gate * XX64 - Is this still correct with kernelheap_extend() being called 2333*0Sstevel@tonic-gate * later than this???? 2334*0Sstevel@tonic-gate */ 2335*0Sstevel@tonic-gate (void) seg_attach(&kas, final_kernelheap, 2336*0Sstevel@tonic-gate ekernelheap - final_kernelheap, &kvseg); 2337*0Sstevel@tonic-gate (void) segkmem_create(&kvseg); 2338*0Sstevel@tonic-gate 2339*0Sstevel@tonic-gate #if defined(__amd64) 2340*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)core_base, core_size, &kvseg_core); 2341*0Sstevel@tonic-gate (void) segkmem_create(&kvseg_core); 2342*0Sstevel@tonic-gate #endif 2343*0Sstevel@tonic-gate 2344*0Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2345*0Sstevel@tonic-gate &kdebugseg); 2346*0Sstevel@tonic-gate (void) segkmem_create(&kdebugseg); 2347*0Sstevel@tonic-gate 2348*0Sstevel@tonic-gate rw_exit(&kas.a_lock); 2349*0Sstevel@tonic-gate 2350*0Sstevel@tonic-gate /* 2351*0Sstevel@tonic-gate * Ensure that the red zone at kernelbase is never accessible. 2352*0Sstevel@tonic-gate */ 2353*0Sstevel@tonic-gate (void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0); 2354*0Sstevel@tonic-gate 2355*0Sstevel@tonic-gate /* 2356*0Sstevel@tonic-gate * Make the text writable so that it can be hot patched by DTrace. 2357*0Sstevel@tonic-gate */ 2358*0Sstevel@tonic-gate (void) as_setprot(&kas, s_text, e_modtext - s_text, 2359*0Sstevel@tonic-gate PROT_READ | PROT_WRITE | PROT_EXEC); 2360*0Sstevel@tonic-gate 2361*0Sstevel@tonic-gate /* 2362*0Sstevel@tonic-gate * Make data writable until end. 2363*0Sstevel@tonic-gate */ 2364*0Sstevel@tonic-gate (void) as_setprot(&kas, s_data, e_moddata - s_data, 2365*0Sstevel@tonic-gate PROT_READ | PROT_WRITE | PROT_EXEC); 2366*0Sstevel@tonic-gate } 2367*0Sstevel@tonic-gate 2368*0Sstevel@tonic-gate /* 2369*0Sstevel@tonic-gate * These are MTTR registers supported by P6 2370*0Sstevel@tonic-gate */ 2371*0Sstevel@tonic-gate static struct mtrrvar mtrrphys_arr[MAX_MTRRVAR]; 2372*0Sstevel@tonic-gate static uint64_t mtrr64k, mtrr16k1, mtrr16k2; 2373*0Sstevel@tonic-gate static uint64_t mtrr4k1, mtrr4k2, mtrr4k3; 2374*0Sstevel@tonic-gate static uint64_t mtrr4k4, mtrr4k5, mtrr4k6; 2375*0Sstevel@tonic-gate static uint64_t mtrr4k7, mtrr4k8, mtrrcap; 2376*0Sstevel@tonic-gate uint64_t mtrrdef, pat_attr_reg; 2377*0Sstevel@tonic-gate 2378*0Sstevel@tonic-gate /* 2379*0Sstevel@tonic-gate * Disable reprogramming of MTRRs by default. 2380*0Sstevel@tonic-gate */ 2381*0Sstevel@tonic-gate int enable_relaxed_mtrr = 0; 2382*0Sstevel@tonic-gate 2383*0Sstevel@tonic-gate /* 2384*0Sstevel@tonic-gate * These must serve for Pentium, Pentium Pro (P6/Pentium II/Pentium III) 2385*0Sstevel@tonic-gate * and Pentium 4, and yes, they are named 0, 1, 2, 4, 3 in ascending 2386*0Sstevel@tonic-gate * address order (starting from 0x400). The Pentium 4 only implements 2387*0Sstevel@tonic-gate * 4 sets, and while they are named 0-3 in the doc, the corresponding 2388*0Sstevel@tonic-gate * names for P6 are 0,1,2,4. So define these arrays in address order 2389*0Sstevel@tonic-gate * so that they work for both pre-Pentium4 and Pentium 4 processors. 2390*0Sstevel@tonic-gate */ 2391*0Sstevel@tonic-gate 2392*0Sstevel@tonic-gate static uint_t mci_ctl[] = {REG_MC0_CTL, REG_MC1_CTL, REG_MC2_CTL, 2393*0Sstevel@tonic-gate REG_MC4_CTL, REG_MC3_CTL}; 2394*0Sstevel@tonic-gate static uint_t mci_status[] = {REG_MC0_STATUS, REG_MC1_STATUS, REG_MC2_STATUS, 2395*0Sstevel@tonic-gate REG_MC4_STATUS, REG_MC3_STATUS}; 2396*0Sstevel@tonic-gate static uint_t mci_addr[] = {REG_MC0_ADDR, REG_MC1_ADDR, REG_MC2_ADDR, 2397*0Sstevel@tonic-gate REG_MC4_ADDR, REG_MC3_ADDR}; 2398*0Sstevel@tonic-gate static int mca_cnt; 2399*0Sstevel@tonic-gate 2400*0Sstevel@tonic-gate 2401*0Sstevel@tonic-gate void 2402*0Sstevel@tonic-gate setup_mca() 2403*0Sstevel@tonic-gate { 2404*0Sstevel@tonic-gate int i; 2405*0Sstevel@tonic-gate uint64_t allzeros; 2406*0Sstevel@tonic-gate uint64_t allones; 2407*0Sstevel@tonic-gate uint64_t mca_cap; 2408*0Sstevel@tonic-gate 2409*0Sstevel@tonic-gate if (!(x86_feature & X86_MCA)) 2410*0Sstevel@tonic-gate return; 2411*0Sstevel@tonic-gate (void) rdmsr(REG_MCG_CAP, &mca_cap); 2412*0Sstevel@tonic-gate allones = 0xffffffffffffffffULL; 2413*0Sstevel@tonic-gate if (mca_cap & MCG_CAP_CTL_P) 2414*0Sstevel@tonic-gate (void) wrmsr(REG_MCG_CTL, &allones); 2415*0Sstevel@tonic-gate mca_cnt = mca_cap & MCG_CAP_COUNT_MASK; 2416*0Sstevel@tonic-gate if (mca_cnt > P6_MCG_CAP_COUNT) 2417*0Sstevel@tonic-gate mca_cnt = P6_MCG_CAP_COUNT; 2418*0Sstevel@tonic-gate for (i = 1; i < mca_cnt; i++) 2419*0Sstevel@tonic-gate (void) wrmsr(mci_ctl[i], &allones); 2420*0Sstevel@tonic-gate allzeros = 0; 2421*0Sstevel@tonic-gate for (i = 0; i < mca_cnt; i++) 2422*0Sstevel@tonic-gate (void) wrmsr(mci_status[i], &allzeros); 2423*0Sstevel@tonic-gate setcr4(getcr4() | CR4_MCE); 2424*0Sstevel@tonic-gate 2425*0Sstevel@tonic-gate } 2426*0Sstevel@tonic-gate 2427*0Sstevel@tonic-gate int 2428*0Sstevel@tonic-gate mca_exception(struct regs *rp) 2429*0Sstevel@tonic-gate { 2430*0Sstevel@tonic-gate uint64_t status, addr; 2431*0Sstevel@tonic-gate uint64_t allzeros; 2432*0Sstevel@tonic-gate uint64_t buf; 2433*0Sstevel@tonic-gate int i, ret = 1, errcode, mserrcode; 2434*0Sstevel@tonic-gate 2435*0Sstevel@tonic-gate allzeros = 0; 2436*0Sstevel@tonic-gate (void) rdmsr(REG_MCG_STATUS, &buf); 2437*0Sstevel@tonic-gate status = buf; 2438*0Sstevel@tonic-gate if (status & MCG_STATUS_RIPV) 2439*0Sstevel@tonic-gate ret = 0; 2440*0Sstevel@tonic-gate if (status & MCG_STATUS_EIPV) 2441*0Sstevel@tonic-gate cmn_err(CE_WARN, "MCE at 0x%lx", rp->r_pc); 2442*0Sstevel@tonic-gate (void) wrmsr(REG_MCG_STATUS, &allzeros); 2443*0Sstevel@tonic-gate for (i = 0; i < mca_cnt; i++) { 2444*0Sstevel@tonic-gate (void) rdmsr(mci_status[i], &buf); 2445*0Sstevel@tonic-gate status = buf; 2446*0Sstevel@tonic-gate /* 2447*0Sstevel@tonic-gate * If status register not valid skip this bank 2448*0Sstevel@tonic-gate */ 2449*0Sstevel@tonic-gate if (!(status & MCI_STATUS_VAL)) 2450*0Sstevel@tonic-gate continue; 2451*0Sstevel@tonic-gate errcode = status & MCI_STATUS_ERRCODE; 2452*0Sstevel@tonic-gate mserrcode = (status >> MSERRCODE_SHFT) & MCI_STATUS_ERRCODE; 2453*0Sstevel@tonic-gate if (status & MCI_STATUS_ADDRV) { 2454*0Sstevel@tonic-gate /* 2455*0Sstevel@tonic-gate * If mci_addr contains the address where 2456*0Sstevel@tonic-gate * error occurred, display the address 2457*0Sstevel@tonic-gate */ 2458*0Sstevel@tonic-gate (void) rdmsr(mci_addr[i], &buf); 2459*0Sstevel@tonic-gate addr = buf; 2460*0Sstevel@tonic-gate cmn_err(CE_WARN, "MCE: Bank %d: error code 0x%x:"\ 2461*0Sstevel@tonic-gate "addr = 0x%" PRIx64 ", model errcode = 0x%x", i, 2462*0Sstevel@tonic-gate errcode, addr, mserrcode); 2463*0Sstevel@tonic-gate } else { 2464*0Sstevel@tonic-gate cmn_err(CE_WARN, 2465*0Sstevel@tonic-gate "MCE: Bank %d: error code 0x%x, mserrcode = 0x%x", 2466*0Sstevel@tonic-gate i, errcode, mserrcode); 2467*0Sstevel@tonic-gate } 2468*0Sstevel@tonic-gate (void) wrmsr(mci_status[i], &allzeros); 2469*0Sstevel@tonic-gate } 2470*0Sstevel@tonic-gate return (ret); 2471*0Sstevel@tonic-gate } 2472*0Sstevel@tonic-gate 2473*0Sstevel@tonic-gate void 2474*0Sstevel@tonic-gate setup_mtrr() 2475*0Sstevel@tonic-gate { 2476*0Sstevel@tonic-gate int i, ecx; 2477*0Sstevel@tonic-gate int vcnt; 2478*0Sstevel@tonic-gate struct mtrrvar *mtrrphys; 2479*0Sstevel@tonic-gate 2480*0Sstevel@tonic-gate if (!(x86_feature & X86_MTRR)) 2481*0Sstevel@tonic-gate return; 2482*0Sstevel@tonic-gate 2483*0Sstevel@tonic-gate (void) rdmsr(REG_MTRRCAP, &mtrrcap); 2484*0Sstevel@tonic-gate (void) rdmsr(REG_MTRRDEF, &mtrrdef); 2485*0Sstevel@tonic-gate if (mtrrcap & MTRRCAP_FIX) { 2486*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR64K, &mtrr64k); 2487*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR16K1, &mtrr16k1); 2488*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR16K2, &mtrr16k2); 2489*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K1, &mtrr4k1); 2490*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K2, &mtrr4k2); 2491*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K3, &mtrr4k3); 2492*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K4, &mtrr4k4); 2493*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K5, &mtrr4k5); 2494*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K6, &mtrr4k6); 2495*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K7, &mtrr4k7); 2496*0Sstevel@tonic-gate (void) rdmsr(REG_MTRR4K8, &mtrr4k8); 2497*0Sstevel@tonic-gate } 2498*0Sstevel@tonic-gate if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR) 2499*0Sstevel@tonic-gate vcnt = MAX_MTRRVAR; 2500*0Sstevel@tonic-gate 2501*0Sstevel@tonic-gate for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr; 2502*0Sstevel@tonic-gate i < vcnt - 1; i++, ecx += 2, mtrrphys++) { 2503*0Sstevel@tonic-gate (void) rdmsr(ecx, &mtrrphys->mtrrphys_base); 2504*0Sstevel@tonic-gate (void) rdmsr(ecx + 1, &mtrrphys->mtrrphys_mask); 2505*0Sstevel@tonic-gate if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) { 2506*0Sstevel@tonic-gate mtrrphys->mtrrphys_mask &= ~MTRRPHYSMASK_V; 2507*0Sstevel@tonic-gate } 2508*0Sstevel@tonic-gate } 2509*0Sstevel@tonic-gate if (x86_feature & X86_PAT) { 2510*0Sstevel@tonic-gate if (enable_relaxed_mtrr) 2511*0Sstevel@tonic-gate mtrrdef = MTRR_TYPE_WB|MTRRDEF_FE|MTRRDEF_E; 2512*0Sstevel@tonic-gate pat_attr_reg = PAT_DEFAULT_ATTRIBUTE; 2513*0Sstevel@tonic-gate } 2514*0Sstevel@tonic-gate 2515*0Sstevel@tonic-gate mtrr_sync(); 2516*0Sstevel@tonic-gate } 2517*0Sstevel@tonic-gate 2518*0Sstevel@tonic-gate /* 2519*0Sstevel@tonic-gate * Sync current cpu mtrr with the incore copy of mtrr. 2520*0Sstevel@tonic-gate * This function has to be invoked with interrupts disabled 2521*0Sstevel@tonic-gate * Currently we do not capture other cpu's. This is invoked on cpu0 2522*0Sstevel@tonic-gate * just after reading /etc/system. 2523*0Sstevel@tonic-gate * On other cpu's its invoked from mp_startup(). 2524*0Sstevel@tonic-gate */ 2525*0Sstevel@tonic-gate void 2526*0Sstevel@tonic-gate mtrr_sync() 2527*0Sstevel@tonic-gate { 2528*0Sstevel@tonic-gate uint64_t my_mtrrdef; 2529*0Sstevel@tonic-gate uint_t crvalue, cr0_orig; 2530*0Sstevel@tonic-gate int vcnt, i, ecx; 2531*0Sstevel@tonic-gate struct mtrrvar *mtrrphys; 2532*0Sstevel@tonic-gate 2533*0Sstevel@tonic-gate cr0_orig = crvalue = getcr0(); 2534*0Sstevel@tonic-gate crvalue |= CR0_CD; 2535*0Sstevel@tonic-gate crvalue &= ~CR0_NW; 2536*0Sstevel@tonic-gate setcr0(crvalue); 2537*0Sstevel@tonic-gate invalidate_cache(); 2538*0Sstevel@tonic-gate setcr3(getcr3()); 2539*0Sstevel@tonic-gate 2540*0Sstevel@tonic-gate if (x86_feature & X86_PAT) { 2541*0Sstevel@tonic-gate (void) wrmsr(REG_MTRRPAT, &pat_attr_reg); 2542*0Sstevel@tonic-gate } 2543*0Sstevel@tonic-gate (void) rdmsr(REG_MTRRDEF, &my_mtrrdef); 2544*0Sstevel@tonic-gate my_mtrrdef &= ~MTRRDEF_E; 2545*0Sstevel@tonic-gate (void) wrmsr(REG_MTRRDEF, &my_mtrrdef); 2546*0Sstevel@tonic-gate if (mtrrcap & MTRRCAP_FIX) { 2547*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR64K, &mtrr64k); 2548*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR16K1, &mtrr16k1); 2549*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR16K2, &mtrr16k2); 2550*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K1, &mtrr4k1); 2551*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K2, &mtrr4k2); 2552*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K3, &mtrr4k3); 2553*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K4, &mtrr4k4); 2554*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K5, &mtrr4k5); 2555*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K6, &mtrr4k6); 2556*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K7, &mtrr4k7); 2557*0Sstevel@tonic-gate (void) wrmsr(REG_MTRR4K8, &mtrr4k8); 2558*0Sstevel@tonic-gate } 2559*0Sstevel@tonic-gate if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR) 2560*0Sstevel@tonic-gate vcnt = MAX_MTRRVAR; 2561*0Sstevel@tonic-gate for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr; 2562*0Sstevel@tonic-gate i < vcnt - 1; i++, ecx += 2, mtrrphys++) { 2563*0Sstevel@tonic-gate (void) wrmsr(ecx, &mtrrphys->mtrrphys_base); 2564*0Sstevel@tonic-gate (void) wrmsr(ecx + 1, &mtrrphys->mtrrphys_mask); 2565*0Sstevel@tonic-gate } 2566*0Sstevel@tonic-gate (void) wrmsr(REG_MTRRDEF, &mtrrdef); 2567*0Sstevel@tonic-gate setcr3(getcr3()); 2568*0Sstevel@tonic-gate invalidate_cache(); 2569*0Sstevel@tonic-gate setcr0(cr0_orig); 2570*0Sstevel@tonic-gate } 2571*0Sstevel@tonic-gate 2572*0Sstevel@tonic-gate /* 2573*0Sstevel@tonic-gate * resync mtrr so that BIOS is happy. Called from mdboot 2574*0Sstevel@tonic-gate */ 2575*0Sstevel@tonic-gate void 2576*0Sstevel@tonic-gate mtrr_resync() 2577*0Sstevel@tonic-gate { 2578*0Sstevel@tonic-gate if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) { 2579*0Sstevel@tonic-gate /* 2580*0Sstevel@tonic-gate * We could have changed the default mtrr definition. 2581*0Sstevel@tonic-gate * Put it back to uncached which is what it is at power on 2582*0Sstevel@tonic-gate */ 2583*0Sstevel@tonic-gate mtrrdef = MTRR_TYPE_UC|MTRRDEF_FE|MTRRDEF_E; 2584*0Sstevel@tonic-gate mtrr_sync(); 2585*0Sstevel@tonic-gate } 2586*0Sstevel@tonic-gate } 2587*0Sstevel@tonic-gate 2588*0Sstevel@tonic-gate void 2589*0Sstevel@tonic-gate get_system_configuration() 2590*0Sstevel@tonic-gate { 2591*0Sstevel@tonic-gate char prop[32]; 2592*0Sstevel@tonic-gate u_longlong_t nodes_ll, cpus_pernode_ll, lvalue; 2593*0Sstevel@tonic-gate 2594*0Sstevel@tonic-gate if (((BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop)) || 2595*0Sstevel@tonic-gate (BOP_GETPROP(bootops, "nodes", prop) < 0) || 2596*0Sstevel@tonic-gate (kobj_getvalue(prop, &nodes_ll) == -1) || 2597*0Sstevel@tonic-gate (nodes_ll > MAXNODES)) || 2598*0Sstevel@tonic-gate ((BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop)) || 2599*0Sstevel@tonic-gate (BOP_GETPROP(bootops, "cpus_pernode", prop) < 0) || 2600*0Sstevel@tonic-gate (kobj_getvalue(prop, &cpus_pernode_ll) == -1))) { 2601*0Sstevel@tonic-gate 2602*0Sstevel@tonic-gate system_hardware.hd_nodes = 1; 2603*0Sstevel@tonic-gate system_hardware.hd_cpus_per_node = 0; 2604*0Sstevel@tonic-gate } else { 2605*0Sstevel@tonic-gate system_hardware.hd_nodes = (int)nodes_ll; 2606*0Sstevel@tonic-gate system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll; 2607*0Sstevel@tonic-gate } 2608*0Sstevel@tonic-gate if ((BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop)) || 2609*0Sstevel@tonic-gate (BOP_GETPROP(bootops, "kernelbase", prop) < 0) || 2610*0Sstevel@tonic-gate (kobj_getvalue(prop, &lvalue) == -1)) 2611*0Sstevel@tonic-gate eprom_kernelbase = NULL; 2612*0Sstevel@tonic-gate else 2613*0Sstevel@tonic-gate eprom_kernelbase = (uintptr_t)lvalue; 2614*0Sstevel@tonic-gate 2615*0Sstevel@tonic-gate if ((BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop)) || 2616*0Sstevel@tonic-gate (BOP_GETPROP(bootops, "segmapsize", prop) < 0) || 2617*0Sstevel@tonic-gate (kobj_getvalue(prop, &lvalue) == -1)) { 2618*0Sstevel@tonic-gate segmapsize = SEGMAPDEFAULT; 2619*0Sstevel@tonic-gate } else { 2620*0Sstevel@tonic-gate segmapsize = (uintptr_t)lvalue; 2621*0Sstevel@tonic-gate } 2622*0Sstevel@tonic-gate 2623*0Sstevel@tonic-gate if ((BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop)) || 2624*0Sstevel@tonic-gate (BOP_GETPROP(bootops, "segmapfreelists", prop) < 0) || 2625*0Sstevel@tonic-gate (kobj_getvalue(prop, &lvalue) == -1)) { 2626*0Sstevel@tonic-gate segmapfreelists = 0; /* use segmap driver default */ 2627*0Sstevel@tonic-gate } else { 2628*0Sstevel@tonic-gate segmapfreelists = (int)lvalue; 2629*0Sstevel@tonic-gate } 2630*0Sstevel@tonic-gate } 2631*0Sstevel@tonic-gate 2632*0Sstevel@tonic-gate /* 2633*0Sstevel@tonic-gate * Add to a memory list. 2634*0Sstevel@tonic-gate * start = start of new memory segment 2635*0Sstevel@tonic-gate * len = length of new memory segment in bytes 2636*0Sstevel@tonic-gate * new = pointer to a new struct memlist 2637*0Sstevel@tonic-gate * memlistp = memory list to which to add segment. 2638*0Sstevel@tonic-gate */ 2639*0Sstevel@tonic-gate static void 2640*0Sstevel@tonic-gate memlist_add( 2641*0Sstevel@tonic-gate uint64_t start, 2642*0Sstevel@tonic-gate uint64_t len, 2643*0Sstevel@tonic-gate struct memlist *new, 2644*0Sstevel@tonic-gate struct memlist **memlistp) 2645*0Sstevel@tonic-gate { 2646*0Sstevel@tonic-gate struct memlist *cur; 2647*0Sstevel@tonic-gate uint64_t end = start + len; 2648*0Sstevel@tonic-gate 2649*0Sstevel@tonic-gate new->address = start; 2650*0Sstevel@tonic-gate new->size = len; 2651*0Sstevel@tonic-gate 2652*0Sstevel@tonic-gate cur = *memlistp; 2653*0Sstevel@tonic-gate 2654*0Sstevel@tonic-gate while (cur) { 2655*0Sstevel@tonic-gate if (cur->address >= end) { 2656*0Sstevel@tonic-gate new->next = cur; 2657*0Sstevel@tonic-gate *memlistp = new; 2658*0Sstevel@tonic-gate new->prev = cur->prev; 2659*0Sstevel@tonic-gate cur->prev = new; 2660*0Sstevel@tonic-gate return; 2661*0Sstevel@tonic-gate } 2662*0Sstevel@tonic-gate ASSERT(cur->address + cur->size <= start); 2663*0Sstevel@tonic-gate if (cur->next == NULL) { 2664*0Sstevel@tonic-gate cur->next = new; 2665*0Sstevel@tonic-gate new->prev = cur; 2666*0Sstevel@tonic-gate new->next = NULL; 2667*0Sstevel@tonic-gate return; 2668*0Sstevel@tonic-gate } 2669*0Sstevel@tonic-gate memlistp = &cur->next; 2670*0Sstevel@tonic-gate cur = cur->next; 2671*0Sstevel@tonic-gate } 2672*0Sstevel@tonic-gate } 2673*0Sstevel@tonic-gate 2674*0Sstevel@tonic-gate void 2675*0Sstevel@tonic-gate kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 2676*0Sstevel@tonic-gate { 2677*0Sstevel@tonic-gate size_t tsize = e_modtext - modtext; 2678*0Sstevel@tonic-gate size_t dsize = e_moddata - moddata; 2679*0Sstevel@tonic-gate 2680*0Sstevel@tonic-gate *text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize, 2681*0Sstevel@tonic-gate 1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP); 2682*0Sstevel@tonic-gate *data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize, 2683*0Sstevel@tonic-gate 1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 2684*0Sstevel@tonic-gate } 2685*0Sstevel@tonic-gate 2686*0Sstevel@tonic-gate caddr_t 2687*0Sstevel@tonic-gate kobj_text_alloc(vmem_t *arena, size_t size) 2688*0Sstevel@tonic-gate { 2689*0Sstevel@tonic-gate return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT)); 2690*0Sstevel@tonic-gate } 2691*0Sstevel@tonic-gate 2692*0Sstevel@tonic-gate /*ARGSUSED*/ 2693*0Sstevel@tonic-gate caddr_t 2694*0Sstevel@tonic-gate kobj_texthole_alloc(caddr_t addr, size_t size) 2695*0Sstevel@tonic-gate { 2696*0Sstevel@tonic-gate panic("unexpected call to kobj_texthole_alloc()"); 2697*0Sstevel@tonic-gate /*NOTREACHED*/ 2698*0Sstevel@tonic-gate return (0); 2699*0Sstevel@tonic-gate } 2700*0Sstevel@tonic-gate 2701*0Sstevel@tonic-gate /*ARGSUSED*/ 2702*0Sstevel@tonic-gate void 2703*0Sstevel@tonic-gate kobj_texthole_free(caddr_t addr, size_t size) 2704*0Sstevel@tonic-gate { 2705*0Sstevel@tonic-gate panic("unexpected call to kobj_texthole_free()"); 2706*0Sstevel@tonic-gate } 2707*0Sstevel@tonic-gate 2708*0Sstevel@tonic-gate /* 2709*0Sstevel@tonic-gate * This is called just after configure() in startup(). 2710*0Sstevel@tonic-gate * 2711*0Sstevel@tonic-gate * The ISALIST concept is a bit hopeless on Intel, because 2712*0Sstevel@tonic-gate * there's no guarantee of an ever-more-capable processor 2713*0Sstevel@tonic-gate * given that various parts of the instruction set may appear 2714*0Sstevel@tonic-gate * and disappear between different implementations. 2715*0Sstevel@tonic-gate * 2716*0Sstevel@tonic-gate * While it would be possible to correct it and even enhance 2717*0Sstevel@tonic-gate * it somewhat, the explicit hardware capability bitmask allows 2718*0Sstevel@tonic-gate * more flexibility. 2719*0Sstevel@tonic-gate * 2720*0Sstevel@tonic-gate * So, we just leave this alone. 2721*0Sstevel@tonic-gate */ 2722*0Sstevel@tonic-gate void 2723*0Sstevel@tonic-gate setx86isalist(void) 2724*0Sstevel@tonic-gate { 2725*0Sstevel@tonic-gate char *tp; 2726*0Sstevel@tonic-gate size_t len; 2727*0Sstevel@tonic-gate extern char *isa_list; 2728*0Sstevel@tonic-gate 2729*0Sstevel@tonic-gate #define TBUFSIZE 1024 2730*0Sstevel@tonic-gate 2731*0Sstevel@tonic-gate tp = kmem_alloc(TBUFSIZE, KM_SLEEP); 2732*0Sstevel@tonic-gate *tp = '\0'; 2733*0Sstevel@tonic-gate 2734*0Sstevel@tonic-gate #if defined(__amd64) 2735*0Sstevel@tonic-gate (void) strcpy(tp, "amd64 "); 2736*0Sstevel@tonic-gate #endif 2737*0Sstevel@tonic-gate 2738*0Sstevel@tonic-gate switch (x86_vendor) { 2739*0Sstevel@tonic-gate case X86_VENDOR_Intel: 2740*0Sstevel@tonic-gate case X86_VENDOR_AMD: 2741*0Sstevel@tonic-gate case X86_VENDOR_TM: 2742*0Sstevel@tonic-gate if (x86_feature & X86_CMOV) { 2743*0Sstevel@tonic-gate /* 2744*0Sstevel@tonic-gate * Pentium Pro or later 2745*0Sstevel@tonic-gate */ 2746*0Sstevel@tonic-gate (void) strcat(tp, "pentium_pro"); 2747*0Sstevel@tonic-gate (void) strcat(tp, x86_feature & X86_MMX ? 2748*0Sstevel@tonic-gate "+mmx pentium_pro " : " "); 2749*0Sstevel@tonic-gate } 2750*0Sstevel@tonic-gate /*FALLTHROUGH*/ 2751*0Sstevel@tonic-gate case X86_VENDOR_Cyrix: 2752*0Sstevel@tonic-gate /* 2753*0Sstevel@tonic-gate * The Cyrix 6x86 does not have any Pentium features 2754*0Sstevel@tonic-gate * accessible while not at privilege level 0. 2755*0Sstevel@tonic-gate */ 2756*0Sstevel@tonic-gate if (x86_feature & X86_CPUID) { 2757*0Sstevel@tonic-gate (void) strcat(tp, "pentium"); 2758*0Sstevel@tonic-gate (void) strcat(tp, x86_feature & X86_MMX ? 2759*0Sstevel@tonic-gate "+mmx pentium " : " "); 2760*0Sstevel@tonic-gate } 2761*0Sstevel@tonic-gate break; 2762*0Sstevel@tonic-gate default: 2763*0Sstevel@tonic-gate break; 2764*0Sstevel@tonic-gate } 2765*0Sstevel@tonic-gate (void) strcat(tp, "i486 i386 i86"); 2766*0Sstevel@tonic-gate len = strlen(tp) + 1; /* account for NULL at end of string */ 2767*0Sstevel@tonic-gate isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp); 2768*0Sstevel@tonic-gate kmem_free(tp, TBUFSIZE); 2769*0Sstevel@tonic-gate 2770*0Sstevel@tonic-gate #undef TBUFSIZE 2771*0Sstevel@tonic-gate } 2772*0Sstevel@tonic-gate 2773*0Sstevel@tonic-gate 2774*0Sstevel@tonic-gate #ifdef __amd64 2775*0Sstevel@tonic-gate 2776*0Sstevel@tonic-gate void * 2777*0Sstevel@tonic-gate device_arena_alloc(size_t size, int vm_flag) 2778*0Sstevel@tonic-gate { 2779*0Sstevel@tonic-gate return (vmem_alloc(device_arena, size, vm_flag)); 2780*0Sstevel@tonic-gate } 2781*0Sstevel@tonic-gate 2782*0Sstevel@tonic-gate void 2783*0Sstevel@tonic-gate device_arena_free(void *vaddr, size_t size) 2784*0Sstevel@tonic-gate { 2785*0Sstevel@tonic-gate vmem_free(device_arena, vaddr, size); 2786*0Sstevel@tonic-gate } 2787*0Sstevel@tonic-gate 2788*0Sstevel@tonic-gate #else 2789*0Sstevel@tonic-gate 2790*0Sstevel@tonic-gate void * 2791*0Sstevel@tonic-gate device_arena_alloc(size_t size, int vm_flag) 2792*0Sstevel@tonic-gate { 2793*0Sstevel@tonic-gate caddr_t vaddr; 2794*0Sstevel@tonic-gate uintptr_t v; 2795*0Sstevel@tonic-gate size_t start; 2796*0Sstevel@tonic-gate size_t end; 2797*0Sstevel@tonic-gate 2798*0Sstevel@tonic-gate vaddr = vmem_alloc(heap_arena, size, vm_flag); 2799*0Sstevel@tonic-gate if (vaddr == NULL) 2800*0Sstevel@tonic-gate return (NULL); 2801*0Sstevel@tonic-gate 2802*0Sstevel@tonic-gate v = (uintptr_t)vaddr; 2803*0Sstevel@tonic-gate ASSERT(v >= kernelbase); 2804*0Sstevel@tonic-gate ASSERT(v + size <= ptable_va); 2805*0Sstevel@tonic-gate 2806*0Sstevel@tonic-gate start = btop(v - kernelbase); 2807*0Sstevel@tonic-gate end = btop(v + size - 1 - kernelbase); 2808*0Sstevel@tonic-gate ASSERT(start < toxic_bit_map_len); 2809*0Sstevel@tonic-gate ASSERT(end < toxic_bit_map_len); 2810*0Sstevel@tonic-gate 2811*0Sstevel@tonic-gate while (start <= end) { 2812*0Sstevel@tonic-gate BT_ATOMIC_SET(toxic_bit_map, start); 2813*0Sstevel@tonic-gate ++start; 2814*0Sstevel@tonic-gate } 2815*0Sstevel@tonic-gate return (vaddr); 2816*0Sstevel@tonic-gate } 2817*0Sstevel@tonic-gate 2818*0Sstevel@tonic-gate void 2819*0Sstevel@tonic-gate device_arena_free(void *vaddr, size_t size) 2820*0Sstevel@tonic-gate { 2821*0Sstevel@tonic-gate uintptr_t v = (uintptr_t)vaddr; 2822*0Sstevel@tonic-gate size_t start; 2823*0Sstevel@tonic-gate size_t end; 2824*0Sstevel@tonic-gate 2825*0Sstevel@tonic-gate ASSERT(v >= kernelbase); 2826*0Sstevel@tonic-gate ASSERT(v + size <= ptable_va); 2827*0Sstevel@tonic-gate 2828*0Sstevel@tonic-gate start = btop(v - kernelbase); 2829*0Sstevel@tonic-gate end = btop(v + size - 1 - kernelbase); 2830*0Sstevel@tonic-gate ASSERT(start < toxic_bit_map_len); 2831*0Sstevel@tonic-gate ASSERT(end < toxic_bit_map_len); 2832*0Sstevel@tonic-gate 2833*0Sstevel@tonic-gate while (start <= end) { 2834*0Sstevel@tonic-gate ASSERT(BT_TEST(toxic_bit_map, start) != 0); 2835*0Sstevel@tonic-gate BT_ATOMIC_CLEAR(toxic_bit_map, start); 2836*0Sstevel@tonic-gate ++start; 2837*0Sstevel@tonic-gate } 2838*0Sstevel@tonic-gate vmem_free(heap_arena, vaddr, size); 2839*0Sstevel@tonic-gate } 2840*0Sstevel@tonic-gate 2841*0Sstevel@tonic-gate /* 2842*0Sstevel@tonic-gate * returns 1st address in range that is in device arena, or NULL 2843*0Sstevel@tonic-gate * if len is not NULL it returns the length of the toxic range 2844*0Sstevel@tonic-gate */ 2845*0Sstevel@tonic-gate void * 2846*0Sstevel@tonic-gate device_arena_contains(void *vaddr, size_t size, size_t *len) 2847*0Sstevel@tonic-gate { 2848*0Sstevel@tonic-gate uintptr_t v = (uintptr_t)vaddr; 2849*0Sstevel@tonic-gate uintptr_t eaddr = v + size; 2850*0Sstevel@tonic-gate size_t start; 2851*0Sstevel@tonic-gate size_t end; 2852*0Sstevel@tonic-gate 2853*0Sstevel@tonic-gate /* 2854*0Sstevel@tonic-gate * if called very early by kmdb, just return NULL 2855*0Sstevel@tonic-gate */ 2856*0Sstevel@tonic-gate if (toxic_bit_map == NULL) 2857*0Sstevel@tonic-gate return (NULL); 2858*0Sstevel@tonic-gate 2859*0Sstevel@tonic-gate /* 2860*0Sstevel@tonic-gate * First check if we're completely outside the bitmap range. 2861*0Sstevel@tonic-gate */ 2862*0Sstevel@tonic-gate if (v >= ptable_va || eaddr < kernelbase) 2863*0Sstevel@tonic-gate return (NULL); 2864*0Sstevel@tonic-gate 2865*0Sstevel@tonic-gate /* 2866*0Sstevel@tonic-gate * Trim ends of search to look at only what the bitmap covers. 2867*0Sstevel@tonic-gate */ 2868*0Sstevel@tonic-gate if (v < kernelbase) 2869*0Sstevel@tonic-gate v = kernelbase; 2870*0Sstevel@tonic-gate start = btop(v - kernelbase); 2871*0Sstevel@tonic-gate end = btop(eaddr - kernelbase); 2872*0Sstevel@tonic-gate if (end >= toxic_bit_map_len) 2873*0Sstevel@tonic-gate end = toxic_bit_map_len; 2874*0Sstevel@tonic-gate 2875*0Sstevel@tonic-gate if (bt_range(toxic_bit_map, &start, &end, end) == 0) 2876*0Sstevel@tonic-gate return (NULL); 2877*0Sstevel@tonic-gate 2878*0Sstevel@tonic-gate v = kernelbase + ptob(start); 2879*0Sstevel@tonic-gate if (len != NULL) 2880*0Sstevel@tonic-gate *len = ptob(end - start); 2881*0Sstevel@tonic-gate return ((void *)v); 2882*0Sstevel@tonic-gate } 2883*0Sstevel@tonic-gate 2884*0Sstevel@tonic-gate #endif 2885