10Sstevel@tonic-gate /*
20Sstevel@tonic-gate * CDDL HEADER START
30Sstevel@tonic-gate *
40Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51253Slq150181 * Common Development and Distribution License (the "License").
61455Sandrei * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate *
80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate * See the License for the specific language governing permissions
110Sstevel@tonic-gate * and limitations under the License.
120Sstevel@tonic-gate *
130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate *
190Sstevel@tonic-gate * CDDL HEADER END
200Sstevel@tonic-gate */
211253Slq150181
220Sstevel@tonic-gate /*
2312230SFrank.Rival@oracle.com * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
240Sstevel@tonic-gate */
250Sstevel@tonic-gate
260Sstevel@tonic-gate #include <sys/machsystm.h>
270Sstevel@tonic-gate #include <sys/archsystm.h>
280Sstevel@tonic-gate #include <sys/vm.h>
290Sstevel@tonic-gate #include <sys/cpu.h>
300Sstevel@tonic-gate #include <sys/atomic.h>
310Sstevel@tonic-gate #include <sys/reboot.h>
320Sstevel@tonic-gate #include <sys/kdi.h>
330Sstevel@tonic-gate #include <sys/bootconf.h>
340Sstevel@tonic-gate #include <sys/memlist_plat.h>
350Sstevel@tonic-gate #include <sys/memlist_impl.h>
360Sstevel@tonic-gate #include <sys/prom_plat.h>
370Sstevel@tonic-gate #include <sys/prom_isa.h>
380Sstevel@tonic-gate #include <sys/autoconf.h>
390Sstevel@tonic-gate #include <sys/ivintr.h>
400Sstevel@tonic-gate #include <sys/fpu/fpusystm.h>
410Sstevel@tonic-gate #include <sys/iommutsb.h>
420Sstevel@tonic-gate #include <vm/vm_dep.h>
430Sstevel@tonic-gate #include <vm/seg_dev.h>
440Sstevel@tonic-gate #include <vm/seg_kmem.h>
450Sstevel@tonic-gate #include <vm/seg_kpm.h>
460Sstevel@tonic-gate #include <vm/seg_map.h>
470Sstevel@tonic-gate #include <vm/seg_kp.h>
480Sstevel@tonic-gate #include <sys/sysconf.h>
490Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
500Sstevel@tonic-gate #include <sys/kobj.h>
510Sstevel@tonic-gate #include <sys/sun4asi.h>
520Sstevel@tonic-gate #include <sys/clconf.h>
530Sstevel@tonic-gate #include <sys/platform_module.h>
540Sstevel@tonic-gate #include <sys/panic.h>
550Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h>
560Sstevel@tonic-gate #include <sys/clock.h>
570Sstevel@tonic-gate #include <sys/cmn_err.h>
58*13145SDave.Plauger@Sun.COM #include <sys/dumphdr.h>
590Sstevel@tonic-gate #include <sys/promif.h>
600Sstevel@tonic-gate #include <sys/prom_debug.h>
610Sstevel@tonic-gate #include <sys/traptrace.h>
620Sstevel@tonic-gate #include <sys/memnode.h>
630Sstevel@tonic-gate #include <sys/mem_cage.h>
642241Shuah #include <sys/mmu.h>
6512230SFrank.Rival@oracle.com #include <sys/swap.h>
660Sstevel@tonic-gate
670Sstevel@tonic-gate extern void setup_trap_table(void);
684050Sjb145095 extern int cpu_intrq_setup(struct cpu *);
690Sstevel@tonic-gate extern void cpu_intrq_register(struct cpu *);
700Sstevel@tonic-gate extern void contig_mem_init(void);
714204Sha137994 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
720Sstevel@tonic-gate extern void mach_dump_buffer_init(void);
730Sstevel@tonic-gate extern void mach_descrip_init(void);
741991Sheppo extern void mach_descrip_startup_fini(void);
750Sstevel@tonic-gate extern void mach_memscrub(void);
760Sstevel@tonic-gate extern void mach_fpras(void);
770Sstevel@tonic-gate extern void mach_cpu_halt_idle(void);
780Sstevel@tonic-gate extern void mach_hw_copy_limit(void);
791991Sheppo extern void load_mach_drivers(void);
800Sstevel@tonic-gate extern void load_tod_module(void);
810Sstevel@tonic-gate #pragma weak load_tod_module
820Sstevel@tonic-gate
830Sstevel@tonic-gate extern int ndata_alloc_mmfsa(struct memlist *ndata);
840Sstevel@tonic-gate #pragma weak ndata_alloc_mmfsa
850Sstevel@tonic-gate
861991Sheppo extern void cif_init(void);
871991Sheppo #pragma weak cif_init
881991Sheppo
890Sstevel@tonic-gate extern void parse_idprom(void);
900Sstevel@tonic-gate extern void add_vx_handler(char *, int, void (*)(cell_t *));
910Sstevel@tonic-gate extern void mem_config_init(void);
920Sstevel@tonic-gate extern void memseg_remap_init(void);
930Sstevel@tonic-gate
942296Sae112802 extern void mach_kpm_init(void);
956880Sdv142724 extern void pcf_init();
964878Sblakej extern int size_pse_array(pgcnt_t, int);
978906SEric.Saxe@Sun.COM extern void pg_init();
982296Sae112802
990Sstevel@tonic-gate /*
1000Sstevel@tonic-gate * External Data:
1010Sstevel@tonic-gate */
1020Sstevel@tonic-gate extern int vac_size; /* cache size in bytes */
1030Sstevel@tonic-gate extern uint_t vac_mask; /* VAC alignment consistency mask */
1040Sstevel@tonic-gate extern uint_t vac_colors;
1050Sstevel@tonic-gate
1060Sstevel@tonic-gate /*
1070Sstevel@tonic-gate * Global Data Definitions:
1080Sstevel@tonic-gate */
1090Sstevel@tonic-gate
1100Sstevel@tonic-gate /*
1110Sstevel@tonic-gate * XXX - Don't port this to new architectures
1120Sstevel@tonic-gate * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
1130Sstevel@tonic-gate * 'romp' has no use with a prom with an IEEE 1275 client interface.
1140Sstevel@tonic-gate * The driver doesn't use the value, but it depends on the symbol.
1150Sstevel@tonic-gate */
1160Sstevel@tonic-gate void *romp; /* veritas driver won't load without romp 4154976 */
1170Sstevel@tonic-gate /*
1180Sstevel@tonic-gate * Declare these as initialized data so we can patch them.
1190Sstevel@tonic-gate */
1200Sstevel@tonic-gate pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */
1210Sstevel@tonic-gate pgcnt_t segkpsize =
1220Sstevel@tonic-gate btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
1239281SPrakash.Sangappa@Sun.COM uint_t segmap_percent = 6; /* Size of segmap segment */
1240Sstevel@tonic-gate
1250Sstevel@tonic-gate int use_cache = 1; /* cache not reliable (605 bugs) with MP */
1260Sstevel@tonic-gate int vac_copyback = 1;
1270Sstevel@tonic-gate char *cache_mode = NULL;
1280Sstevel@tonic-gate int use_mix = 1;
1290Sstevel@tonic-gate int prom_debug = 0;
1300Sstevel@tonic-gate
1310Sstevel@tonic-gate caddr_t boot_tba; /* %tba at boot - used by kmdb */
1320Sstevel@tonic-gate uint_t tba_taken_over = 0;
1330Sstevel@tonic-gate
1340Sstevel@tonic-gate caddr_t s_text; /* start of kernel text segment */
1350Sstevel@tonic-gate caddr_t e_text; /* end of kernel text segment */
1360Sstevel@tonic-gate caddr_t s_data; /* start of kernel data segment */
1370Sstevel@tonic-gate caddr_t e_data; /* end of kernel data segment */
1380Sstevel@tonic-gate
1390Sstevel@tonic-gate caddr_t modtext; /* beginning of module text */
1400Sstevel@tonic-gate size_t modtext_sz; /* size of module text */
1410Sstevel@tonic-gate caddr_t moddata; /* beginning of module data reserve */
1420Sstevel@tonic-gate caddr_t e_moddata; /* end of module data reserve */
1430Sstevel@tonic-gate
1440Sstevel@tonic-gate /*
1450Sstevel@tonic-gate * End of first block of contiguous kernel in 32-bit virtual address space
1460Sstevel@tonic-gate */
1470Sstevel@tonic-gate caddr_t econtig32; /* end of first blk of contiguous kernel */
1480Sstevel@tonic-gate
1490Sstevel@tonic-gate caddr_t ncbase; /* beginning of non-cached segment */
1500Sstevel@tonic-gate caddr_t ncend; /* end of non-cached segment */
1510Sstevel@tonic-gate
1520Sstevel@tonic-gate size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */
1530Sstevel@tonic-gate caddr_t nalloc_base; /* beginning of nucleus allocation */
1540Sstevel@tonic-gate caddr_t nalloc_end; /* end of nucleus allocatable memory */
1550Sstevel@tonic-gate caddr_t valloc_base; /* beginning of kvalloc segment */
1560Sstevel@tonic-gate
1570Sstevel@tonic-gate caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */
1580Sstevel@tonic-gate caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */
1595648Ssetje size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */
1603764Sdp78419 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */
1613764Sdp78419 int kmem64_szc; /* page size code */
1623764Sdp78419 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */
1630Sstevel@tonic-gate
1642296Sae112802 uintptr_t shm_alignment; /* VAC address consistency modulus */
1650Sstevel@tonic-gate struct memlist *phys_install; /* Total installed physical memory */
1660Sstevel@tonic-gate struct memlist *phys_avail; /* Available (unreserved) physical memory */
1670Sstevel@tonic-gate struct memlist *virt_avail; /* Available (unmapped?) virtual memory */
1685648Ssetje struct memlist *nopp_list; /* pages with no backing page structs */
1690Sstevel@tonic-gate struct memlist ndata; /* memlist of nucleus allocatable memory */
1700Sstevel@tonic-gate int memexp_flag; /* memory expansion card flag */
1710Sstevel@tonic-gate uint64_t ecache_flushaddr; /* physical address used for flushing E$ */
1720Sstevel@tonic-gate pgcnt_t obp_pages; /* Physical pages used by OBP */
1730Sstevel@tonic-gate
1740Sstevel@tonic-gate /*
1750Sstevel@tonic-gate * VM data structures
1760Sstevel@tonic-gate */
1770Sstevel@tonic-gate long page_hashsz; /* Size of page hash table (power of two) */
17812230SFrank.Rival@oracle.com unsigned int page_hashsz_shift; /* log2(page_hashsz) */
1790Sstevel@tonic-gate struct page *pp_base; /* Base of system page struct array */
1800Sstevel@tonic-gate size_t pp_sz; /* Size in bytes of page struct array */
1810Sstevel@tonic-gate struct page **page_hash; /* Page hash table */
1824878Sblakej pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */
1834878Sblakej size_t pse_table_size; /* Number of mutexes in pse_mutex[] */
1844878Sblakej int pse_shift; /* log2(pse_table_size) */
1850Sstevel@tonic-gate struct seg ktextseg; /* Segment used for kernel executable image */
1860Sstevel@tonic-gate struct seg kvalloc; /* Segment used for "valloc" mapping */
1870Sstevel@tonic-gate struct seg kpseg; /* Segment used for pageable kernel virt mem */
1880Sstevel@tonic-gate struct seg ktexthole; /* Segment used for nucleus text hole */
1890Sstevel@tonic-gate struct seg kmapseg; /* Segment used for generic kernel mappings */
1900Sstevel@tonic-gate struct seg kpmseg; /* Segment used for physical mapping */
1910Sstevel@tonic-gate struct seg kdebugseg; /* Segment used for the kernel debugger */
1920Sstevel@tonic-gate
1935648Ssetje void *kpm_pp_base; /* Base of system kpm_page array */
1940Sstevel@tonic-gate size_t kpm_pp_sz; /* Size of system kpm_page array */
1950Sstevel@tonic-gate pgcnt_t kpm_npages; /* How many kpm pages are managed */
1960Sstevel@tonic-gate
1970Sstevel@tonic-gate struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
1980Sstevel@tonic-gate struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */
1990Sstevel@tonic-gate struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */
2000Sstevel@tonic-gate
2013290Sjohansen int segzio_fromheap = 0; /* zio allocations occur from heap */
2023290Sjohansen caddr_t segzio_base; /* Base address of segzio */
2033290Sjohansen pgcnt_t segziosize = 0; /* size of zio segment in pages */
2043290Sjohansen
2050Sstevel@tonic-gate /*
20610106SJason.Beloro@Sun.COM * A static DR page_t VA map is reserved that can map the page structures
20710106SJason.Beloro@Sun.COM * for a domain's entire RA space. The pages that backs this space are
20810106SJason.Beloro@Sun.COM * dynamically allocated and need not be physically contiguous. The DR
20910106SJason.Beloro@Sun.COM * map size is derived from KPM size.
21010106SJason.Beloro@Sun.COM */
21110106SJason.Beloro@Sun.COM int ppvm_enable = 0; /* Static virtual map for page structs */
21210106SJason.Beloro@Sun.COM page_t *ppvm_base; /* Base of page struct map */
21310106SJason.Beloro@Sun.COM pgcnt_t ppvm_size = 0; /* Size of page struct map */
21410106SJason.Beloro@Sun.COM
21510106SJason.Beloro@Sun.COM /*
2160Sstevel@tonic-gate * debugger pages (if allocated)
2170Sstevel@tonic-gate */
2180Sstevel@tonic-gate struct vnode kdebugvp;
2190Sstevel@tonic-gate
2200Sstevel@tonic-gate /*
2213446Smrj * VA range available to the debugger
2223446Smrj */
2233446Smrj const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
2243446Smrj const size_t kdi_segdebugsize = SEGDEBUGSIZE;
2253446Smrj
2263446Smrj /*
2270Sstevel@tonic-gate * Segment for relocated kernel structures in 64-bit large RAM kernels
2280Sstevel@tonic-gate */
2290Sstevel@tonic-gate struct seg kmem64;
2300Sstevel@tonic-gate
2315648Ssetje struct memseg *memseg_free;
2325648Ssetje
2330Sstevel@tonic-gate struct vnode unused_pages_vp;
2340Sstevel@tonic-gate
2350Sstevel@tonic-gate /*
2360Sstevel@tonic-gate * VM data structures allocated early during boot.
2370Sstevel@tonic-gate */
2380Sstevel@tonic-gate size_t pagehash_sz;
2390Sstevel@tonic-gate uint64_t memlist_sz;
2400Sstevel@tonic-gate
2410Sstevel@tonic-gate char tbr_wr_addr_inited = 0;
2420Sstevel@tonic-gate
2435358Sjc25722 caddr_t mpo_heap32_buf = NULL;
2445358Sjc25722 size_t mpo_heap32_bufsz = 0;
2450Sstevel@tonic-gate
2460Sstevel@tonic-gate /*
2470Sstevel@tonic-gate * Static Routines:
2480Sstevel@tonic-gate */
2495648Ssetje static int ndata_alloc_memseg(struct memlist *, size_t);
2505648Ssetje static void memlist_new(uint64_t, uint64_t, struct memlist **);
2515648Ssetje static void memlist_add(uint64_t, uint64_t,
2525648Ssetje struct memlist **, struct memlist **);
2535648Ssetje static void kphysm_init(void);
2540Sstevel@tonic-gate static void kvm_init(void);
2555648Ssetje static void install_kmem64_tte(void);
2560Sstevel@tonic-gate
2570Sstevel@tonic-gate static void startup_init(void);
2580Sstevel@tonic-gate static void startup_memlist(void);
2590Sstevel@tonic-gate static void startup_modules(void);
2600Sstevel@tonic-gate static void startup_bop_gone(void);
2610Sstevel@tonic-gate static void startup_vm(void);
2620Sstevel@tonic-gate static void startup_end(void);
2630Sstevel@tonic-gate static void setup_cage_params(void);
2641253Slq150181 static void startup_create_io_node(void);
2650Sstevel@tonic-gate
2660Sstevel@tonic-gate static pgcnt_t npages;
2670Sstevel@tonic-gate static struct memlist *memlist;
2680Sstevel@tonic-gate void *memlist_end;
2690Sstevel@tonic-gate
2700Sstevel@tonic-gate static pgcnt_t bop_alloc_pages;
2710Sstevel@tonic-gate static caddr_t hblk_base;
2720Sstevel@tonic-gate uint_t hblk_alloc_dynamic = 0;
2730Sstevel@tonic-gate uint_t hblk1_min = H1MIN;
2740Sstevel@tonic-gate
2750Sstevel@tonic-gate
2760Sstevel@tonic-gate /*
2770Sstevel@tonic-gate * Hooks for unsupported platforms and down-rev firmware
2780Sstevel@tonic-gate */
2790Sstevel@tonic-gate int iam_positron(void);
2800Sstevel@tonic-gate #pragma weak iam_positron
2810Sstevel@tonic-gate static void do_prom_version_check(void);
2820Sstevel@tonic-gate
2830Sstevel@tonic-gate /*
2840Sstevel@tonic-gate * After receiving a thermal interrupt, this is the number of seconds
2850Sstevel@tonic-gate * to delay before shutting off the system, assuming
2860Sstevel@tonic-gate * shutdown fails. Use /etc/system to change the delay if this isn't
2870Sstevel@tonic-gate * large enough.
2880Sstevel@tonic-gate */
2890Sstevel@tonic-gate int thermal_powerdown_delay = 1200;
2900Sstevel@tonic-gate
2910Sstevel@tonic-gate /*
2920Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed
2930Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel.
2940Sstevel@tonic-gate */
2950Sstevel@tonic-gate int page_relocate_ready = 0;
2960Sstevel@tonic-gate
2970Sstevel@tonic-gate /*
2987218Ssvemuri * Indicate if kmem64 allocation was done in small chunks
2997218Ssvemuri */
3007218Ssvemuri int kmem64_smchunks = 0;
3017218Ssvemuri
3027218Ssvemuri /*
3030Sstevel@tonic-gate * Enable some debugging messages concerning memory usage...
3040Sstevel@tonic-gate */
3050Sstevel@tonic-gate #ifdef DEBUGGING_MEM
3060Sstevel@tonic-gate static int debugging_mem;
3070Sstevel@tonic-gate static void
printmemlist(char * title,struct memlist * list)3080Sstevel@tonic-gate printmemlist(char *title, struct memlist *list)
3090Sstevel@tonic-gate {
3100Sstevel@tonic-gate if (!debugging_mem)
3110Sstevel@tonic-gate return;
3120Sstevel@tonic-gate
3130Sstevel@tonic-gate printf("%s\n", title);
3140Sstevel@tonic-gate
3150Sstevel@tonic-gate while (list) {
3160Sstevel@tonic-gate prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
31711474SJonathan.Adams@Sun.COM (uint32_t)(list->ml_address >> 32),
31811474SJonathan.Adams@Sun.COM (uint32_t)list->ml_address,
31911474SJonathan.Adams@Sun.COM (uint32_t)(list->ml_size >> 32),
32011474SJonathan.Adams@Sun.COM (uint32_t)(list->ml_size));
32111474SJonathan.Adams@Sun.COM list = list->ml_next;
3220Sstevel@tonic-gate }
3230Sstevel@tonic-gate }
3240Sstevel@tonic-gate
3250Sstevel@tonic-gate void
printmemseg(struct memseg * memseg)3260Sstevel@tonic-gate printmemseg(struct memseg *memseg)
3270Sstevel@tonic-gate {
3280Sstevel@tonic-gate if (!debugging_mem)
3290Sstevel@tonic-gate return;
3300Sstevel@tonic-gate
3310Sstevel@tonic-gate printf("memseg\n");
3320Sstevel@tonic-gate
3330Sstevel@tonic-gate while (memseg) {
3340Sstevel@tonic-gate prom_printf("\tpage = 0x%p, epage = 0x%p, "
3350Sstevel@tonic-gate "pfn = 0x%x, epfn = 0x%x\n",
3360Sstevel@tonic-gate memseg->pages, memseg->epages,
3370Sstevel@tonic-gate memseg->pages_base, memseg->pages_end);
3380Sstevel@tonic-gate memseg = memseg->next;
3390Sstevel@tonic-gate }
3400Sstevel@tonic-gate }
3410Sstevel@tonic-gate
3420Sstevel@tonic-gate #define debug_pause(str) halt((str))
3430Sstevel@tonic-gate #define MPRINTF(str) if (debugging_mem) prom_printf((str))
3440Sstevel@tonic-gate #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))
3450Sstevel@tonic-gate #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))
3460Sstevel@tonic-gate #define MPRINTF3(str, a, b, c) \
3470Sstevel@tonic-gate if (debugging_mem) prom_printf((str), (a), (b), (c))
3480Sstevel@tonic-gate #else /* DEBUGGING_MEM */
3490Sstevel@tonic-gate #define MPRINTF(str)
3500Sstevel@tonic-gate #define MPRINTF1(str, a)
3510Sstevel@tonic-gate #define MPRINTF2(str, a, b)
3520Sstevel@tonic-gate #define MPRINTF3(str, a, b, c)
3530Sstevel@tonic-gate #endif /* DEBUGGING_MEM */
3540Sstevel@tonic-gate
3550Sstevel@tonic-gate
3560Sstevel@tonic-gate /*
3570Sstevel@tonic-gate *
3580Sstevel@tonic-gate * Kernel's Virtual Memory Layout.
3590Sstevel@tonic-gate * /-----------------------\
3600Sstevel@tonic-gate * 0xFFFFFFFF.FFFFFFFF -| |-
3610Sstevel@tonic-gate * | OBP's virtual page |
3620Sstevel@tonic-gate * | tables |
3630Sstevel@tonic-gate * 0xFFFFFFFC.00000000 -|-----------------------|-
3640Sstevel@tonic-gate * : :
3650Sstevel@tonic-gate * : :
3663290Sjohansen * -|-----------------------|-
3673290Sjohansen * | segzio | (base and size vary)
3680Sstevel@tonic-gate * 0xFFFFFE00.00000000 -|-----------------------|-
3690Sstevel@tonic-gate * | | Ultrasparc I/II support
3700Sstevel@tonic-gate * | segkpm segment | up to 2TB of physical
3710Sstevel@tonic-gate * | (64-bit kernel ONLY) | memory, VAC has 2 colors
3720Sstevel@tonic-gate * | |
3730Sstevel@tonic-gate * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment
3740Sstevel@tonic-gate * : :
3750Sstevel@tonic-gate * : :
3760Sstevel@tonic-gate * 0xFFFFF810.00000000 -|-----------------------|- hole_end
3770Sstevel@tonic-gate * | | ^
3780Sstevel@tonic-gate * | UltraSPARC I/II call | |
3790Sstevel@tonic-gate * | bug requires an extra | |
3800Sstevel@tonic-gate * | 4 GB of space between | |
3810Sstevel@tonic-gate * | hole and used RAM | |
3820Sstevel@tonic-gate * | | |
3830Sstevel@tonic-gate * 0xFFFFF800.00000000 -|-----------------------|- |
3840Sstevel@tonic-gate * | | |
3850Sstevel@tonic-gate * | Virtual Address Hole | UltraSPARC
3860Sstevel@tonic-gate * | on UltraSPARC I/II | I/II * ONLY *
3870Sstevel@tonic-gate * | | |
3880Sstevel@tonic-gate * 0x00000800.00000000 -|-----------------------|- |
3890Sstevel@tonic-gate * | | |
3900Sstevel@tonic-gate * | UltraSPARC I/II call | |
3910Sstevel@tonic-gate * | bug requires an extra | |
3920Sstevel@tonic-gate * | 4 GB of space between | |
3930Sstevel@tonic-gate * | hole and used RAM | |
3940Sstevel@tonic-gate * | | v
3950Sstevel@tonic-gate * 0x000007FF.00000000 -|-----------------------|- hole_start -----
3960Sstevel@tonic-gate * : : ^
3970Sstevel@tonic-gate * : : |
3985648Ssetje * |-----------------------| |
3995648Ssetje * | | |
4005648Ssetje * | ecache flush area | |
4015648Ssetje * | (twice largest e$) | |
4025648Ssetje * | | |
4033764Sdp78419 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ |
4043764Sdp78419 * | overmapped area | alignend_end |
4053764Sdp78419 * | (kmem64_alignsize | |
4063764Sdp78419 * | boundary) | |
4070Sstevel@tonic-gate * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end |
4080Sstevel@tonic-gate * | | |
4090Sstevel@tonic-gate * | 64-bit kernel ONLY | |
4100Sstevel@tonic-gate * | | |
4110Sstevel@tonic-gate * | kmem64 segment | |
4120Sstevel@tonic-gate * | | |
4130Sstevel@tonic-gate * | (Relocated extra HME | Approximately
4140Sstevel@tonic-gate * | block allocations, | 1 TB of virtual
4150Sstevel@tonic-gate * | memnode freelists, | address space
4160Sstevel@tonic-gate * | HME hash buckets, | |
4170Sstevel@tonic-gate * | mml_table, kpmp_table,| |
4180Sstevel@tonic-gate * | page_t array and | |
4190Sstevel@tonic-gate * | hashblock pool to | |
4200Sstevel@tonic-gate * | avoid hard-coded | |
4210Sstevel@tonic-gate * | 32-bit vaddr | |
4220Sstevel@tonic-gate * | limitations) | |
4230Sstevel@tonic-gate * | | v
4240Sstevel@tonic-gate * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base)
4250Sstevel@tonic-gate * | |
4260Sstevel@tonic-gate * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB)
4270Sstevel@tonic-gate * | |
4280Sstevel@tonic-gate * 0x00000300.00000000 -|-----------------------|- SYSBASE
4290Sstevel@tonic-gate * : :
4300Sstevel@tonic-gate * : :
4310Sstevel@tonic-gate * -|-----------------------|-
4320Sstevel@tonic-gate * | |
4330Sstevel@tonic-gate * | segmap segment | SEGMAPSIZE (1/8th physmem,
4340Sstevel@tonic-gate * | | 256G MAX)
4350Sstevel@tonic-gate * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE
4360Sstevel@tonic-gate * : :
4370Sstevel@tonic-gate * : :
4380Sstevel@tonic-gate * -|-----------------------|-
4390Sstevel@tonic-gate * | |
4400Sstevel@tonic-gate * | segkp | SEGKPSIZE (2GB)
4410Sstevel@tonic-gate * | |
4420Sstevel@tonic-gate * | |
4430Sstevel@tonic-gate * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE
4440Sstevel@tonic-gate * | |
4450Sstevel@tonic-gate * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE
4460Sstevel@tonic-gate * | | (SEGKPBASE - 0x400000)
4470Sstevel@tonic-gate * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE
4480Sstevel@tonic-gate * | | (MEMSCRUBBASE - NCARGS)
4490Sstevel@tonic-gate * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE
4500Sstevel@tonic-gate * | | (ARGSBASE - PPMAPSIZE)
4510Sstevel@tonic-gate * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE
4520Sstevel@tonic-gate * | |
4530Sstevel@tonic-gate * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE
4540Sstevel@tonic-gate * | |
4550Sstevel@tonic-gate * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE
4560Sstevel@tonic-gate * : :
4570Sstevel@tonic-gate * : :
4580Sstevel@tonic-gate * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR
4590Sstevel@tonic-gate * | |
4600Sstevel@tonic-gate * | OBP |
4610Sstevel@tonic-gate * | |
4620Sstevel@tonic-gate * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR
4630Sstevel@tonic-gate * | kmdb |
4640Sstevel@tonic-gate * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE
4650Sstevel@tonic-gate * : :
4660Sstevel@tonic-gate * : :
4670Sstevel@tonic-gate * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32
4680Sstevel@tonic-gate * | |
4690Sstevel@tonic-gate * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 =
4700Sstevel@tonic-gate * | | ~64MB)
47111444SGangadhar.M@Sun.COM * -|-----------------------|
47211444SGangadhar.M@Sun.COM * | IVSIZE |
47311444SGangadhar.M@Sun.COM * 0x00000000.70004000 -|-----------------------|
47411234SGangadhar.M@Sun.COM * | panicbuf |
47511444SGangadhar.M@Sun.COM * 0x00000000.70002000 -|-----------------------|
47611444SGangadhar.M@Sun.COM * | PAGESIZE |
4775648Ssetje * 0x00000000.70000000 -|-----------------------|- SYSBASE32
4785648Ssetje * | boot-time |
4795648Ssetje * | temporary space |
4805648Ssetje * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE
4810Sstevel@tonic-gate * : :
4820Sstevel@tonic-gate * : :
4830Sstevel@tonic-gate * | |
4840Sstevel@tonic-gate * |-----------------------|- econtig32
4850Sstevel@tonic-gate * | vm structures |
4860Sstevel@tonic-gate * 0x00000000.01C00000 |-----------------------|- nalloc_end
4870Sstevel@tonic-gate * | TSBs |
4880Sstevel@tonic-gate * |-----------------------|- end/nalloc_base
4890Sstevel@tonic-gate * | kernel data & bss |
4900Sstevel@tonic-gate * 0x00000000.01800000 -|-----------------------|
4910Sstevel@tonic-gate * : nucleus text hole :
4920Sstevel@tonic-gate * 0x00000000.01400000 -|-----------------------|
4930Sstevel@tonic-gate * : :
4940Sstevel@tonic-gate * |-----------------------|
4950Sstevel@tonic-gate * | module text |
4960Sstevel@tonic-gate * |-----------------------|- e_text/modtext
4970Sstevel@tonic-gate * | kernel text |
4980Sstevel@tonic-gate * |-----------------------|
4990Sstevel@tonic-gate * | trap table (48k) |
5000Sstevel@tonic-gate * 0x00000000.01000000 -|-----------------------|- KERNELBASE
5010Sstevel@tonic-gate * | reserved for trapstat |} TSTAT_TOTAL_SIZE
5020Sstevel@tonic-gate * |-----------------------|
5030Sstevel@tonic-gate * | |
5040Sstevel@tonic-gate * | invalid |
5050Sstevel@tonic-gate * | |
5060Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________|
5070Sstevel@tonic-gate *
5080Sstevel@tonic-gate *
5090Sstevel@tonic-gate *
5100Sstevel@tonic-gate * 32-bit User Virtual Memory Layout.
5110Sstevel@tonic-gate * /-----------------------\
5120Sstevel@tonic-gate * | |
5130Sstevel@tonic-gate * | invalid |
5140Sstevel@tonic-gate * | |
5150Sstevel@tonic-gate * 0xFFC00000 -|-----------------------|- USERLIMIT
5160Sstevel@tonic-gate * | user stack |
5170Sstevel@tonic-gate * : :
5180Sstevel@tonic-gate * : :
5190Sstevel@tonic-gate * : :
5200Sstevel@tonic-gate * | user data |
5210Sstevel@tonic-gate * -|-----------------------|-
5220Sstevel@tonic-gate * | user text |
5230Sstevel@tonic-gate * 0x00002000 -|-----------------------|-
5240Sstevel@tonic-gate * | invalid |
5250Sstevel@tonic-gate * 0x00000000 _|_______________________|
5260Sstevel@tonic-gate *
5270Sstevel@tonic-gate *
5280Sstevel@tonic-gate *
5290Sstevel@tonic-gate * 64-bit User Virtual Memory Layout.
5300Sstevel@tonic-gate * /-----------------------\
5310Sstevel@tonic-gate * | |
5320Sstevel@tonic-gate * | invalid |
5330Sstevel@tonic-gate * | |
5340Sstevel@tonic-gate * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
5350Sstevel@tonic-gate * | user stack |
5360Sstevel@tonic-gate * : :
5370Sstevel@tonic-gate * : :
5380Sstevel@tonic-gate * : :
5390Sstevel@tonic-gate * | user data |
5400Sstevel@tonic-gate * -|-----------------------|-
5410Sstevel@tonic-gate * | user text |
5425648Ssetje * 0x00000000.01000000 -|-----------------------|-
5430Sstevel@tonic-gate * | invalid |
5440Sstevel@tonic-gate * 0x00000000.00000000 _|_______________________|
5450Sstevel@tonic-gate */
5460Sstevel@tonic-gate
5470Sstevel@tonic-gate extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
5480Sstevel@tonic-gate extern uint64_t ecache_flush_address(void);
5490Sstevel@tonic-gate
5500Sstevel@tonic-gate #pragma weak load_platform_modules
5511772Sjl139090 #pragma weak plat_startup_memlist
5520Sstevel@tonic-gate #pragma weak ecache_init_scrub_flush_area
5530Sstevel@tonic-gate #pragma weak ecache_flush_address
5540Sstevel@tonic-gate
5550Sstevel@tonic-gate
5560Sstevel@tonic-gate /*
5570Sstevel@tonic-gate * By default the DR Cage is enabled for maximum OS
5580Sstevel@tonic-gate * MPSS performance. Users needing to disable the cage mechanism
5590Sstevel@tonic-gate * can set this variable to zero via /etc/system.
5600Sstevel@tonic-gate * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
5610Sstevel@tonic-gate * will result in loss of DR functionality.
5620Sstevel@tonic-gate * Platforms wishing to disable kernel Cage by default
5630Sstevel@tonic-gate * should do so in their set_platform_defaults() routine.
5640Sstevel@tonic-gate */
5650Sstevel@tonic-gate int kernel_cage_enable = 1;
5660Sstevel@tonic-gate
5670Sstevel@tonic-gate static void
setup_cage_params(void)5680Sstevel@tonic-gate setup_cage_params(void)
5690Sstevel@tonic-gate {
5700Sstevel@tonic-gate void (*func)(void);
5710Sstevel@tonic-gate
5720Sstevel@tonic-gate func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
5730Sstevel@tonic-gate if (func != NULL) {
5740Sstevel@tonic-gate (*func)();
5750Sstevel@tonic-gate return;
5760Sstevel@tonic-gate }
5770Sstevel@tonic-gate
5780Sstevel@tonic-gate if (kernel_cage_enable == 0) {
5790Sstevel@tonic-gate return;
5800Sstevel@tonic-gate }
5814266Sdp78419 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
5820Sstevel@tonic-gate
5830Sstevel@tonic-gate if (kcage_on) {
5840Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
5850Sstevel@tonic-gate } else {
5860Sstevel@tonic-gate cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
5870Sstevel@tonic-gate }
5880Sstevel@tonic-gate
5890Sstevel@tonic-gate }
5900Sstevel@tonic-gate
5910Sstevel@tonic-gate /*
5920Sstevel@tonic-gate * Machine-dependent startup code
5930Sstevel@tonic-gate */
5940Sstevel@tonic-gate void
startup(void)5950Sstevel@tonic-gate startup(void)
5960Sstevel@tonic-gate {
5970Sstevel@tonic-gate startup_init();
5980Sstevel@tonic-gate if (&startup_platform)
5990Sstevel@tonic-gate startup_platform();
6000Sstevel@tonic-gate startup_memlist();
6010Sstevel@tonic-gate startup_modules();
6020Sstevel@tonic-gate setup_cage_params();
6030Sstevel@tonic-gate startup_bop_gone();
6040Sstevel@tonic-gate startup_vm();
6050Sstevel@tonic-gate startup_end();
6060Sstevel@tonic-gate }
6070Sstevel@tonic-gate
6080Sstevel@tonic-gate struct regs sync_reg_buf;
6090Sstevel@tonic-gate uint64_t sync_tt;
6100Sstevel@tonic-gate
6110Sstevel@tonic-gate void
sync_handler(void)6120Sstevel@tonic-gate sync_handler(void)
6130Sstevel@tonic-gate {
6145084Sjohnlev struct panic_trap_info ti;
6150Sstevel@tonic-gate int i;
6160Sstevel@tonic-gate
6170Sstevel@tonic-gate /*
6180Sstevel@tonic-gate * Prevent trying to talk to the other CPUs since they are
6190Sstevel@tonic-gate * sitting in the prom and won't reply.
6200Sstevel@tonic-gate */
6210Sstevel@tonic-gate for (i = 0; i < NCPU; i++) {
6220Sstevel@tonic-gate if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
6230Sstevel@tonic-gate cpu[i]->cpu_flags &= ~CPU_READY;
6240Sstevel@tonic-gate cpu[i]->cpu_flags |= CPU_QUIESCED;
6250Sstevel@tonic-gate CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
6260Sstevel@tonic-gate }
6270Sstevel@tonic-gate }
6280Sstevel@tonic-gate
6290Sstevel@tonic-gate /*
630*13145SDave.Plauger@Sun.COM * Force a serial dump, since there are no CPUs to help.
631*13145SDave.Plauger@Sun.COM */
632*13145SDave.Plauger@Sun.COM dump_plat_mincpu = 0;
633*13145SDave.Plauger@Sun.COM
634*13145SDave.Plauger@Sun.COM /*
6350Sstevel@tonic-gate * We've managed to get here without going through the
6360Sstevel@tonic-gate * normal panic code path. Try and save some useful
6370Sstevel@tonic-gate * information.
6380Sstevel@tonic-gate */
6390Sstevel@tonic-gate if (!panicstr && (curthread->t_panic_trap == NULL)) {
6400Sstevel@tonic-gate ti.trap_type = sync_tt;
6410Sstevel@tonic-gate ti.trap_regs = &sync_reg_buf;
6420Sstevel@tonic-gate ti.trap_addr = NULL;
6430Sstevel@tonic-gate ti.trap_mmu_fsr = 0x0;
6440Sstevel@tonic-gate
6450Sstevel@tonic-gate curthread->t_panic_trap = &ti;
6460Sstevel@tonic-gate }
6470Sstevel@tonic-gate
6480Sstevel@tonic-gate /*
6490Sstevel@tonic-gate * If we're re-entering the panic path, update the signature
6500Sstevel@tonic-gate * block so that the SC knows we're in the second part of panic.
6510Sstevel@tonic-gate */
6520Sstevel@tonic-gate if (panicstr)
6530Sstevel@tonic-gate CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
6540Sstevel@tonic-gate
6550Sstevel@tonic-gate nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
6560Sstevel@tonic-gate panic("sync initiated");
6570Sstevel@tonic-gate }
6580Sstevel@tonic-gate
6590Sstevel@tonic-gate
6600Sstevel@tonic-gate static void
startup_init(void)6610Sstevel@tonic-gate startup_init(void)
6620Sstevel@tonic-gate {
6630Sstevel@tonic-gate /*
6640Sstevel@tonic-gate * We want to save the registers while we're still in OBP
6650Sstevel@tonic-gate * so that we know they haven't been fiddled with since.
6660Sstevel@tonic-gate * (In principle, OBP can't change them just because it
6670Sstevel@tonic-gate * makes a callback, but we'd rather not depend on that
6680Sstevel@tonic-gate * behavior.)
6690Sstevel@tonic-gate */
6700Sstevel@tonic-gate char sync_str[] =
6714528Spaulsan "warning @ warning off : sync "
6724528Spaulsan "%%tl-c %%tstate h# %p x! "
6734528Spaulsan "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
6744528Spaulsan "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
6754528Spaulsan "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
6764528Spaulsan "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
6774528Spaulsan "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
6784528Spaulsan "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
6794528Spaulsan "%%y h# %p l! %%tl-c %%tt h# %p x! "
6804528Spaulsan "sync ; warning !";
6810Sstevel@tonic-gate
6820Sstevel@tonic-gate /*
6830Sstevel@tonic-gate * 20 == num of %p substrings
6840Sstevel@tonic-gate * 16 == max num of chars %p will expand to.
6850Sstevel@tonic-gate */
6860Sstevel@tonic-gate char bp[sizeof (sync_str) + 16 * 20];
6870Sstevel@tonic-gate
6880Sstevel@tonic-gate /*
6890Sstevel@tonic-gate * Initialize ptl1 stack for the 1st CPU.
6900Sstevel@tonic-gate */
6910Sstevel@tonic-gate ptl1_init_cpu(&cpu0);
6920Sstevel@tonic-gate
6930Sstevel@tonic-gate /*
6940Sstevel@tonic-gate * Initialize the address map for cache consistent mappings
6950Sstevel@tonic-gate * to random pages; must be done after vac_size is set.
6960Sstevel@tonic-gate */
6970Sstevel@tonic-gate ppmapinit();
6980Sstevel@tonic-gate
6990Sstevel@tonic-gate /*
7000Sstevel@tonic-gate * Initialize the PROM callback handler.
7010Sstevel@tonic-gate */
7020Sstevel@tonic-gate init_vx_handler();
7030Sstevel@tonic-gate
7040Sstevel@tonic-gate /*
7050Sstevel@tonic-gate * have prom call sync_callback() to handle the sync and
7060Sstevel@tonic-gate * save some useful information which will be stored in the
7070Sstevel@tonic-gate * core file later.
7080Sstevel@tonic-gate */
7090Sstevel@tonic-gate (void) sprintf((char *)bp, sync_str,
7104528Spaulsan (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
7114528Spaulsan (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
7124528Spaulsan (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
7134528Spaulsan (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
7144528Spaulsan (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
7154528Spaulsan (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
7164528Spaulsan (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
7174528Spaulsan (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
7184528Spaulsan (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
7194528Spaulsan (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
7200Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0);
7210Sstevel@tonic-gate add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
7220Sstevel@tonic-gate }
7230Sstevel@tonic-gate
7245648Ssetje
7255648Ssetje size_t
calc_pp_sz(pgcnt_t npages)7265648Ssetje calc_pp_sz(pgcnt_t npages)
7275648Ssetje {
7285648Ssetje
7295648Ssetje return (npages * sizeof (struct page));
7305648Ssetje }
7315648Ssetje
7325648Ssetje size_t
calc_kpmpp_sz(pgcnt_t npages)7335648Ssetje calc_kpmpp_sz(pgcnt_t npages)
7345648Ssetje {
7355648Ssetje
7365648Ssetje kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
7375648Ssetje kpm_pgsz = 1ull << kpm_pgshft;
7385648Ssetje kpm_pgoff = kpm_pgsz - 1;
7395648Ssetje kpmp2pshft = kpm_pgshft - PAGESHIFT;
7405648Ssetje kpmpnpgs = 1 << kpmp2pshft;
7415648Ssetje
7425648Ssetje if (kpm_smallpages == 0) {
7435648Ssetje /*
7445648Ssetje * Avoid fragmentation problems in kphysm_init()
7455648Ssetje * by allocating for all of physical memory
7465648Ssetje */
7475648Ssetje kpm_npages = ptokpmpr(physinstalled);
7485648Ssetje return (kpm_npages * sizeof (kpm_page_t));
7495648Ssetje } else {
7505648Ssetje kpm_npages = npages;
7515648Ssetje return (kpm_npages * sizeof (kpm_spage_t));
7525648Ssetje }
7535648Ssetje }
7545648Ssetje
7555648Ssetje size_t
calc_pagehash_sz(pgcnt_t npages)7565648Ssetje calc_pagehash_sz(pgcnt_t npages)
7575648Ssetje {
75812230SFrank.Rival@oracle.com /* LINTED */
75912230SFrank.Rival@oracle.com ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
7605648Ssetje /*
7615648Ssetje * The page structure hash table size is a power of 2
7625648Ssetje * such that the average hash chain length is PAGE_HASHAVELEN.
7635648Ssetje */
7645648Ssetje page_hashsz = npages / PAGE_HASHAVELEN;
76512230SFrank.Rival@oracle.com page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
76612230SFrank.Rival@oracle.com highbit(page_hashsz));
76712230SFrank.Rival@oracle.com page_hashsz = 1 << page_hashsz_shift;
7685648Ssetje return (page_hashsz * sizeof (struct page *));
7695648Ssetje }
7705648Ssetje
7717218Ssvemuri int testkmem64_smchunks = 0;
7727218Ssvemuri
7737218Ssvemuri int
alloc_kmem64(caddr_t base,caddr_t end)7745648Ssetje alloc_kmem64(caddr_t base, caddr_t end)
7755648Ssetje {
7765648Ssetje int i;
7775648Ssetje caddr_t aligned_end = NULL;
7785648Ssetje
7797218Ssvemuri if (testkmem64_smchunks)
7807218Ssvemuri return (1);
7817218Ssvemuri
7825648Ssetje /*
7835648Ssetje * Make one large memory alloc after figuring out the 64-bit size. This
7845648Ssetje * will enable use of the largest page size appropriate for the system
7855648Ssetje * architecture.
7865648Ssetje */
7875648Ssetje ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
7885648Ssetje ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
7895648Ssetje for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
7905648Ssetje size_t alloc_size, alignsize;
7915648Ssetje #if !defined(C_OBP)
7925648Ssetje unsigned long long pa;
7935648Ssetje #endif /* !C_OBP */
7945648Ssetje
7955648Ssetje if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
7965648Ssetje continue;
7975648Ssetje alignsize = TTEBYTES(i);
7985648Ssetje kmem64_szc = i;
7995648Ssetje
8005648Ssetje /* limit page size for small memory */
8015648Ssetje if (mmu_btop(alignsize) > (npages >> 2))
8025648Ssetje continue;
8035648Ssetje
8045648Ssetje aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
8055648Ssetje alloc_size = aligned_end - base;
8065648Ssetje #if !defined(C_OBP)
8075648Ssetje if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
8085648Ssetje if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
8095648Ssetje kmem64_pabase = pa;
8105648Ssetje kmem64_aligned_end = aligned_end;
8115648Ssetje install_kmem64_tte();
8125648Ssetje break;
8135648Ssetje } else {
8145648Ssetje prom_free_phys(alloc_size, pa);
8155648Ssetje }
8165648Ssetje }
8175648Ssetje #else /* !C_OBP */
8185648Ssetje if (prom_alloc(base, alloc_size, alignsize) == base) {
8195648Ssetje kmem64_pabase = va_to_pa(kmem64_base);
8205648Ssetje kmem64_aligned_end = aligned_end;
8215648Ssetje break;
8225648Ssetje }
8235648Ssetje #endif /* !C_OBP */
8245648Ssetje if (i == TTE8K) {
8257218Ssvemuri #ifdef sun4v
8267218Ssvemuri /* return failure to try small allocations */
8277218Ssvemuri return (1);
8287218Ssvemuri #else
8295648Ssetje prom_panic("kmem64 allocation failure");
8307218Ssvemuri #endif
8315648Ssetje }
8325648Ssetje }
8335648Ssetje ASSERT(aligned_end != NULL);
8347218Ssvemuri return (0);
8355648Ssetje }
8365648Ssetje
8375648Ssetje static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
8380Sstevel@tonic-gate static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
8390Sstevel@tonic-gate
8403764Sdp78419 #if !defined(C_OBP)
8413764Sdp78419 /*
8423764Sdp78419 * Install a temporary tte handler in OBP for kmem64 area.
8433764Sdp78419 *
8443764Sdp78419 * We map kmem64 area with large pages before the trap table is taken
8453764Sdp78419 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
8463764Sdp78419 * the same area. Duplicate tlb entries with different page sizes
8473764Sdp78419 * cause unpredicatble behavior. To avoid this, we don't create
8483764Sdp78419 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
8493764Sdp78419 * OBP). Instead, we manage translations with a temporary va>tte-data
8503764Sdp78419 * handler (kmem64-tte). This handler is replaced by unix-tte when
8513764Sdp78419 * the trap table is taken over.
8523764Sdp78419 *
8533764Sdp78419 * The temporary handler knows the physical address of the kmem64
8543764Sdp78419 * area. It uses the prom's pgmap@ Forth word for other addresses.
8553764Sdp78419 *
8563764Sdp78419 * We have to use BOP_ALLOC() method for C-OBP platforms because
8573764Sdp78419 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
8583764Sdp78419 * sun4u platforms. On sun4u we flush tlb after trap table is taken
8593764Sdp78419 * over if we use large pages for kernel heap and kmem64. Since sun4u
8603764Sdp78419 * prom (unlike sun4v) calls va>tte-data first for client address
8613764Sdp78419 * translation prom's ttes for kmem64 can't get into TLB even if we
8623764Sdp78419 * later switch to prom's trap table again. C-OBP uses 4M pages for
8633764Sdp78419 * client mappings when possible so on all platforms we get the
8643764Sdp78419 * benefit from large mappings for kmem64 area immediately during
8653764Sdp78419 * boot.
8663764Sdp78419 *
8673764Sdp78419 * pseudo code:
8683764Sdp78419 * if (context != 0) {
8693764Sdp78419 * return false
8703764Sdp78419 * } else if (miss_va in range[kmem64_base, kmem64_end)) {
8713764Sdp78419 * tte = tte_template +
8723764Sdp78419 * (((miss_va & pagemask) - kmem64_base));
8733764Sdp78419 * return tte, true
8743764Sdp78419 * } else {
8753764Sdp78419 * return pgmap@ result
8763764Sdp78419 * }
8773764Sdp78419 */
8783764Sdp78419 char kmem64_obp_str[] =
8795648Ssetje "h# %lx constant kmem64-base "
8805648Ssetje "h# %lx constant kmem64-end "
8815648Ssetje "h# %lx constant kmem64-pagemask "
8825648Ssetje "h# %lx constant kmem64-template "
8833764Sdp78419
8843764Sdp78419 ": kmem64-tte ( addr cnum -- false | tte-data true ) "
8853764Sdp78419 " if ( addr ) "
8863764Sdp78419 " drop false exit then ( false ) "
8875648Ssetje " dup kmem64-base kmem64-end within if ( addr ) "
8885648Ssetje " kmem64-pagemask and ( addr' ) "
8895648Ssetje " kmem64-base - ( addr' ) "
8905648Ssetje " kmem64-template + ( tte ) "
8913764Sdp78419 " true ( tte true ) "
8923764Sdp78419 " else ( addr ) "
8933764Sdp78419 " pgmap@ ( tte ) "
8943764Sdp78419 " dup 0< if true else drop false then ( tte true | false ) "
8953764Sdp78419 " then ( tte true | false ) "
8963764Sdp78419 "; "
8973764Sdp78419
8983764Sdp78419 "' kmem64-tte is va>tte-data "
8993764Sdp78419 ;
9003764Sdp78419
9015648Ssetje static void
install_kmem64_tte()9023764Sdp78419 install_kmem64_tte()
9033764Sdp78419 {
9043764Sdp78419 char b[sizeof (kmem64_obp_str) + (4 * 16)];
9053764Sdp78419 tte_t tte;
9063764Sdp78419
9073764Sdp78419 PRM_DEBUG(kmem64_pabase);
9083764Sdp78419 PRM_DEBUG(kmem64_szc);
9093764Sdp78419 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
91010271SJason.Beloro@Sun.COM PROC_DATA | HAT_NOSYNC, kmem64_szc);
9113764Sdp78419 PRM_DEBUG(tte.ll);
9123764Sdp78419 (void) sprintf(b, kmem64_obp_str,
9133764Sdp78419 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
9143764Sdp78419 ASSERT(strlen(b) < sizeof (b));
9153764Sdp78419 prom_interpret(b, 0, 0, 0, 0, 0);
9163764Sdp78419 }
9173764Sdp78419 #endif /* !C_OBP */
9183764Sdp78419
9190Sstevel@tonic-gate /*
9200Sstevel@tonic-gate * As OBP takes up some RAM when the system boots, pages will already be "lost"
9210Sstevel@tonic-gate * to the system and reflected in npages by the time we see it.
9220Sstevel@tonic-gate *
9230Sstevel@tonic-gate * We only want to allocate kernel structures in the 64-bit virtual address
9240Sstevel@tonic-gate * space on systems with enough RAM to make the overhead of keeping track of
9250Sstevel@tonic-gate * an extra kernel memory segment worthwhile.
9260Sstevel@tonic-gate *
9270Sstevel@tonic-gate * Since OBP has already performed its memory allocations by this point, if we
9280Sstevel@tonic-gate * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
9290Sstevel@tonic-gate * memory in the 64-bit virtual address space; otherwise keep allocations
9300Sstevel@tonic-gate * contiguous with we've mapped so far in the 32-bit virtual address space.
9310Sstevel@tonic-gate */
9320Sstevel@tonic-gate #define MINMOVE_RAM_MB ((size_t)1900)
9330Sstevel@tonic-gate #define MB_TO_BYTES(mb) ((mb) * 1048576ul)
9345872Ssetje #define BYTES_TO_MB(b) ((b) / 1048576ul)
9350Sstevel@tonic-gate
9360Sstevel@tonic-gate pgcnt_t tune_npages = (pgcnt_t)
9370Sstevel@tonic-gate (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
9380Sstevel@tonic-gate
9393733Sdp78419 #pragma weak page_set_colorequiv_arr_cpu
9403733Sdp78419 extern void page_set_colorequiv_arr_cpu(void);
9415648Ssetje extern void page_set_colorequiv_arr(void);
9425648Ssetje
9435872Ssetje static pgcnt_t ramdisk_npages;
9445872Ssetje static struct memlist *old_phys_avail;
9455872Ssetje
9465872Ssetje kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
9473733Sdp78419
9480Sstevel@tonic-gate static void
startup_memlist(void)9490Sstevel@tonic-gate startup_memlist(void)
9500Sstevel@tonic-gate {
9515648Ssetje size_t hmehash_sz, pagelist_sz, tt_sz;
9525648Ssetje size_t psetable_sz;
9530Sstevel@tonic-gate caddr_t alloc_base;
9540Sstevel@tonic-gate caddr_t memspace;
9550Sstevel@tonic-gate struct memlist *cur;
9560Sstevel@tonic-gate size_t syslimit = (size_t)SYSLIMIT;
9570Sstevel@tonic-gate size_t sysbase = (size_t)SYSBASE;
9580Sstevel@tonic-gate
9590Sstevel@tonic-gate /*
9600Sstevel@tonic-gate * Initialize enough of the system to allow kmem_alloc to work by
9610Sstevel@tonic-gate * calling boot to allocate its memory until the time that
9620Sstevel@tonic-gate * kvm_init is completed. The page structs are allocated after
9630Sstevel@tonic-gate * rounding up end to the nearest page boundary; the memsegs are
9640Sstevel@tonic-gate * initialized and the space they use comes from the kernel heap.
9650Sstevel@tonic-gate * With appropriate initialization, they can be reallocated later
9660Sstevel@tonic-gate * to a size appropriate for the machine's configuration.
9670Sstevel@tonic-gate *
9680Sstevel@tonic-gate * At this point, memory is allocated for things that will never
9690Sstevel@tonic-gate * need to be freed, this used to be "valloced". This allows a
9700Sstevel@tonic-gate * savings as the pages don't need page structures to describe
9710Sstevel@tonic-gate * them because them will not be managed by the vm system.
9720Sstevel@tonic-gate */
9730Sstevel@tonic-gate
9740Sstevel@tonic-gate /*
9750Sstevel@tonic-gate * We're loaded by boot with the following configuration (as
9760Sstevel@tonic-gate * specified in the sun4u/conf/Mapfile):
9770Sstevel@tonic-gate *
9780Sstevel@tonic-gate * text: 4 MB chunk aligned on a 4MB boundary
9790Sstevel@tonic-gate * data & bss: 4 MB chunk aligned on a 4MB boundary
9800Sstevel@tonic-gate *
9810Sstevel@tonic-gate * These two chunks will eventually be mapped by 2 locked 4MB
9820Sstevel@tonic-gate * ttes and will represent the nucleus of the kernel. This gives
9830Sstevel@tonic-gate * us some free space that is already allocated, some or all of
9840Sstevel@tonic-gate * which is made available to kernel module text.
9850Sstevel@tonic-gate *
9860Sstevel@tonic-gate * The free space in the data-bss chunk is used for nucleus
9870Sstevel@tonic-gate * allocatable data structures and we reserve it using the
9880Sstevel@tonic-gate * nalloc_base and nalloc_end variables. This space is currently
9890Sstevel@tonic-gate * being used for hat data structures required for tlb miss
9900Sstevel@tonic-gate * handling operations. We align nalloc_base to a l2 cache
9910Sstevel@tonic-gate * linesize because this is the line size the hardware uses to
9920Sstevel@tonic-gate * maintain cache coherency.
9935648Ssetje * 512K is carved out for module data.
9940Sstevel@tonic-gate */
9950Sstevel@tonic-gate
9965648Ssetje moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
9975648Ssetje e_moddata = moddata + MODDATA;
9980Sstevel@tonic-gate nalloc_base = e_moddata;
9990Sstevel@tonic-gate
10000Sstevel@tonic-gate nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
10010Sstevel@tonic-gate valloc_base = nalloc_base;
10020Sstevel@tonic-gate
10030Sstevel@tonic-gate /*
10040Sstevel@tonic-gate * Calculate the start of the data segment.
10050Sstevel@tonic-gate */
10065648Ssetje if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
10075648Ssetje prom_panic("nucleus data overflow");
10080Sstevel@tonic-gate
10090Sstevel@tonic-gate PRM_DEBUG(moddata);
10100Sstevel@tonic-gate PRM_DEBUG(nalloc_base);
10110Sstevel@tonic-gate PRM_DEBUG(nalloc_end);
10120Sstevel@tonic-gate
10130Sstevel@tonic-gate /*
10140Sstevel@tonic-gate * Remember any slop after e_text so we can give it to the modules.
10150Sstevel@tonic-gate */
10160Sstevel@tonic-gate PRM_DEBUG(e_text);
10170Sstevel@tonic-gate modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
10183791Skchow if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
10193764Sdp78419 prom_panic("nucleus text overflow");
10200Sstevel@tonic-gate modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
10210Sstevel@tonic-gate modtext;
10220Sstevel@tonic-gate PRM_DEBUG(modtext);
10230Sstevel@tonic-gate PRM_DEBUG(modtext_sz);
10240Sstevel@tonic-gate
10255648Ssetje init_boot_memlists();
10260Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
10270Sstevel@tonic-gate &boot_physavail, &boot_physavail_len,
10280Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len);
10295648Ssetje
10300Sstevel@tonic-gate /*
10310Sstevel@tonic-gate * Remember what the physically available highest page is
10320Sstevel@tonic-gate * so that dumpsys works properly, and find out how much
10330Sstevel@tonic-gate * memory is installed.
10340Sstevel@tonic-gate */
10350Sstevel@tonic-gate installed_top_size_memlist_array(boot_physinstalled,
10360Sstevel@tonic-gate boot_physinstalled_len, &physmax, &physinstalled);
10370Sstevel@tonic-gate PRM_DEBUG(physinstalled);
10380Sstevel@tonic-gate PRM_DEBUG(physmax);
10390Sstevel@tonic-gate
10400Sstevel@tonic-gate /* Fill out memory nodes config structure */
10410Sstevel@tonic-gate startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
10420Sstevel@tonic-gate
10430Sstevel@tonic-gate /*
10440Sstevel@tonic-gate * npages is the maximum of available physical memory possible.
10450Sstevel@tonic-gate * (ie. it will never be more than this)
10465648Ssetje *
10475648Ssetje * When we boot from a ramdisk, the ramdisk memory isn't free, so
10485648Ssetje * using phys_avail will underestimate what will end up being freed.
10495648Ssetje * A better initial guess is just total memory minus the kernel text
10500Sstevel@tonic-gate */
10515648Ssetje npages = physinstalled - btop(MMU_PAGESIZE4M);
10520Sstevel@tonic-gate
10530Sstevel@tonic-gate /*
10545648Ssetje * First allocate things that can go in the nucleus data page
10555648Ssetje * (fault status, TSBs, dmv, CPUs)
10560Sstevel@tonic-gate */
10575648Ssetje ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
10585648Ssetje
10590Sstevel@tonic-gate if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
10600Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
10610Sstevel@tonic-gate
10620Sstevel@tonic-gate if (ndata_alloc_tsbs(&ndata, npages) != 0)
10630Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
10640Sstevel@tonic-gate
10650Sstevel@tonic-gate if (ndata_alloc_dmv(&ndata) != 0)
10660Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
10670Sstevel@tonic-gate
10685648Ssetje if (ndata_alloc_page_mutexs(&ndata) != 0)
10690Sstevel@tonic-gate cmn_err(CE_PANIC,
10700Sstevel@tonic-gate "no more nucleus memory after page free lists alloc");
10710Sstevel@tonic-gate
107212908SPavel.Tatashin@Sun.COM if (ndata_alloc_hat(&ndata) != 0)
10730Sstevel@tonic-gate cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
10740Sstevel@tonic-gate
10755648Ssetje if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
10765648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
10770Sstevel@tonic-gate
10780Sstevel@tonic-gate /*
10790Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
10800Sstevel@tonic-gate *
10810Sstevel@tonic-gate * There are comments all over the SFMMU code warning of dire
10820Sstevel@tonic-gate * consequences if the TSBs are moved out of 32-bit space. This
10830Sstevel@tonic-gate * is largely because the asm code uses "sethi %hi(addr)"-type
10840Sstevel@tonic-gate * instructions which will not provide the expected result if the
10850Sstevel@tonic-gate * address is a 64-bit one.
10860Sstevel@tonic-gate *
10870Sstevel@tonic-gate * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
10880Sstevel@tonic-gate */
10890Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
10905648Ssetje PRM_DEBUG(alloc_base);
10915648Ssetje
10920Sstevel@tonic-gate alloc_base = sfmmu_ktsb_alloc(alloc_base);
10930Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
10940Sstevel@tonic-gate PRM_DEBUG(alloc_base);
10950Sstevel@tonic-gate
10960Sstevel@tonic-gate /*
10970Sstevel@tonic-gate * Allocate IOMMU TSB array. We do this here so that the physical
10980Sstevel@tonic-gate * memory gets deducted from the PROM's physical memory list.
10990Sstevel@tonic-gate */
11000Sstevel@tonic-gate alloc_base = iommu_tsb_init(alloc_base);
11015648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
11020Sstevel@tonic-gate PRM_DEBUG(alloc_base);
11030Sstevel@tonic-gate
11040Sstevel@tonic-gate /*
11055648Ssetje * Allow for an early allocation of physically contiguous memory.
11065648Ssetje */
11075648Ssetje alloc_base = contig_mem_prealloc(alloc_base, npages);
11085648Ssetje
11095648Ssetje /*
11101772Sjl139090 * Platforms like Starcat and OPL need special structures assigned in
11111772Sjl139090 * 32-bit virtual address space because their probing routines execute
11121772Sjl139090 * FCode, and FCode can't handle 64-bit virtual addresses...
11130Sstevel@tonic-gate */
11141772Sjl139090 if (&plat_startup_memlist) {
11151772Sjl139090 alloc_base = plat_startup_memlist(alloc_base);
11160Sstevel@tonic-gate alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
11170Sstevel@tonic-gate ecache_alignsize);
11180Sstevel@tonic-gate PRM_DEBUG(alloc_base);
11190Sstevel@tonic-gate }
11200Sstevel@tonic-gate
11210Sstevel@tonic-gate /*
11220Sstevel@tonic-gate * Save off where the contiguous allocations to date have ended
11230Sstevel@tonic-gate * in econtig32.
11240Sstevel@tonic-gate */
11250Sstevel@tonic-gate econtig32 = alloc_base;
11260Sstevel@tonic-gate PRM_DEBUG(econtig32);
11270Sstevel@tonic-gate if (econtig32 > (caddr_t)KERNEL_LIMIT32)
11280Sstevel@tonic-gate cmn_err(CE_PANIC, "econtig32 too big");
11290Sstevel@tonic-gate
11305648Ssetje pp_sz = calc_pp_sz(npages);
11315648Ssetje PRM_DEBUG(pp_sz);
11325648Ssetje if (kpm_enable) {
11335648Ssetje kpm_pp_sz = calc_kpmpp_sz(npages);
11345648Ssetje PRM_DEBUG(kpm_pp_sz);
11355648Ssetje }
11365648Ssetje
11375648Ssetje hmehash_sz = calc_hmehash_sz(npages);
11385648Ssetje PRM_DEBUG(hmehash_sz);
11395648Ssetje
11405648Ssetje pagehash_sz = calc_pagehash_sz(npages);
11415648Ssetje PRM_DEBUG(pagehash_sz);
11425648Ssetje
11435648Ssetje pagelist_sz = calc_free_pagelist_sz();
11445648Ssetje PRM_DEBUG(pagelist_sz);
11455648Ssetje
11465648Ssetje #ifdef TRAPTRACE
11475648Ssetje tt_sz = calc_traptrace_sz();
11485648Ssetje PRM_DEBUG(tt_sz);
11495648Ssetje #else
11505648Ssetje tt_sz = 0;
11515648Ssetje #endif /* TRAPTRACE */
11520Sstevel@tonic-gate
11533764Sdp78419 /*
11545648Ssetje * Place the array that protects pp->p_selock in the kmem64 wad.
11553764Sdp78419 */
11566582Ssetje pse_shift = size_pse_array(npages, max_ncpus);
11575648Ssetje PRM_DEBUG(pse_shift);
11585648Ssetje pse_table_size = 1 << pse_shift;
11595648Ssetje PRM_DEBUG(pse_table_size);
11605648Ssetje psetable_sz = roundup(
11615648Ssetje pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
11625648Ssetje PRM_DEBUG(psetable_sz);
11630Sstevel@tonic-gate
11640Sstevel@tonic-gate /*
11655648Ssetje * Now allocate the whole wad
11660Sstevel@tonic-gate */
11675648Ssetje kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
11685648Ssetje pagelist_sz + tt_sz + psetable_sz;
11695648Ssetje kmem64_sz = roundup(kmem64_sz, PAGESIZE);
11705648Ssetje kmem64_base = (caddr_t)syslimit;
11715648Ssetje kmem64_end = kmem64_base + kmem64_sz;
11727218Ssvemuri if (alloc_kmem64(kmem64_base, kmem64_end)) {
11737218Ssvemuri /*
11747218Ssvemuri * Attempt for kmem64 to allocate one big
11757218Ssvemuri * contiguous chunk of memory failed.
11767218Ssvemuri * We get here because we are sun4v.
11777218Ssvemuri * We will proceed by breaking up
11787218Ssvemuri * the allocation into two attempts.
11797218Ssvemuri * First, we allocate kpm_pp_sz, hmehash_sz,
11807218Ssvemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
11817218Ssvemuri * one contiguous chunk. This is a much smaller
11827218Ssvemuri * chunk and we should get it, if not we panic.
11837218Ssvemuri * Note that hmehash and tt need to be physically
11847218Ssvemuri * (in the real address sense) contiguous.
11857218Ssvemuri * Next, we use bop_alloc_chunk() to
11867218Ssvemuri * to allocate the page_t structures.
11877218Ssvemuri * This will allow the page_t to be allocated
11887218Ssvemuri * in multiple smaller chunks.
11897218Ssvemuri * In doing so, the assumption that page_t is
11907218Ssvemuri * physically contiguous no longer hold, this is ok
11917218Ssvemuri * for sun4v but not for sun4u.
11927218Ssvemuri */
11937218Ssvemuri size_t tmp_size;
11947218Ssvemuri caddr_t tmp_base;
11957218Ssvemuri
11967218Ssvemuri pp_sz = roundup(pp_sz, PAGESIZE);
11977218Ssvemuri
11987218Ssvemuri /*
11997218Ssvemuri * Allocate kpm_pp_sz, hmehash_sz,
12007218Ssvemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
12017218Ssvemuri */
12027218Ssvemuri tmp_base = kmem64_base + pp_sz;
12037218Ssvemuri tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
12047218Ssvemuri pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
12057218Ssvemuri if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
12067218Ssvemuri prom_panic("kmem64 prom_alloc contig failed");
12077218Ssvemuri PRM_DEBUG(tmp_base);
12087218Ssvemuri PRM_DEBUG(tmp_size);
12097218Ssvemuri
12107218Ssvemuri /*
12117218Ssvemuri * Allocate the page_ts
12127218Ssvemuri */
12137218Ssvemuri if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
12147218Ssvemuri prom_panic("kmem64 bop_alloc_chunk page_t failed");
12157218Ssvemuri PRM_DEBUG(kmem64_base);
12167218Ssvemuri PRM_DEBUG(pp_sz);
12177218Ssvemuri
12187218Ssvemuri kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
12197218Ssvemuri ASSERT(kmem64_aligned_end >= kmem64_end);
12207218Ssvemuri
12217218Ssvemuri kmem64_smchunks = 1;
12227218Ssvemuri } else {
12237218Ssvemuri
12247218Ssvemuri /*
12257218Ssvemuri * We need to adjust pp_sz for the normal
12267218Ssvemuri * case where kmem64 can allocate one large chunk
12277218Ssvemuri */
12287218Ssvemuri if (kpm_smallpages == 0) {
12297218Ssvemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
12307218Ssvemuri } else {
12317218Ssvemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
12327218Ssvemuri sizeof (kpm_spage_t));
12337218Ssvemuri }
12347218Ssvemuri pp_sz = npages * sizeof (struct page);
12357218Ssvemuri }
12367218Ssvemuri
12375648Ssetje if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
12385648Ssetje cmn_err(CE_PANIC, "not enough kmem64 space");
12395648Ssetje PRM_DEBUG(kmem64_base);
12405648Ssetje PRM_DEBUG(kmem64_end);
12415648Ssetje PRM_DEBUG(kmem64_aligned_end);
12424204Sha137994
12434204Sha137994 /*
12445648Ssetje * ... and divy it up
12450Sstevel@tonic-gate */
12465648Ssetje alloc_base = kmem64_base;
12475872Ssetje
12485648Ssetje pp_base = (page_t *)alloc_base;
12495648Ssetje alloc_base += pp_sz;
12505648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12515648Ssetje PRM_DEBUG(pp_base);
12525648Ssetje PRM_DEBUG(npages);
12535648Ssetje
12545648Ssetje if (kpm_enable) {
12555648Ssetje kpm_pp_base = alloc_base;
12565648Ssetje if (kpm_smallpages == 0) {
12575648Ssetje /* kpm_npages based on physinstalled, don't reset */
12585648Ssetje kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
12595648Ssetje } else {
12605648Ssetje kpm_npages = ptokpmpr(npages);
12615648Ssetje kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
12620Sstevel@tonic-gate }
12635648Ssetje alloc_base += kpm_pp_sz;
12645648Ssetje alloc_base =
12655648Ssetje (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12665648Ssetje PRM_DEBUG(kpm_pp_base);
12670Sstevel@tonic-gate }
12680Sstevel@tonic-gate
12695648Ssetje alloc_base = alloc_hmehash(alloc_base);
12705648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12715648Ssetje PRM_DEBUG(alloc_base);
12725648Ssetje
12735648Ssetje page_hash = (page_t **)alloc_base;
12745648Ssetje alloc_base += pagehash_sz;
12755648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12765648Ssetje PRM_DEBUG(page_hash);
12775648Ssetje
12785648Ssetje alloc_base = alloc_page_freelists(alloc_base);
12795648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12805648Ssetje PRM_DEBUG(alloc_base);
12815648Ssetje
12825648Ssetje #ifdef TRAPTRACE
12835648Ssetje ttrace_buf = alloc_base;
12845648Ssetje alloc_base += tt_sz;
12855648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12865648Ssetje PRM_DEBUG(alloc_base);
12875648Ssetje #endif /* TRAPTRACE */
12885648Ssetje
12895648Ssetje pse_mutex = (pad_mutex_t *)alloc_base;
12905648Ssetje alloc_base += psetable_sz;
12915648Ssetje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
12925648Ssetje PRM_DEBUG(alloc_base);
12935648Ssetje
12947218Ssvemuri /*
12957218Ssvemuri * Note that if we use small chunk allocations for
12967218Ssvemuri * kmem64, we need to ensure kmem64_end is the same as
12977218Ssvemuri * kmem64_aligned_end to prevent subsequent logic from
12987218Ssvemuri * trying to reuse the overmapping.
12997218Ssvemuri * Otherwise we adjust kmem64_end to what we really allocated.
13007218Ssvemuri */
13017218Ssvemuri if (kmem64_smchunks) {
13027218Ssvemuri kmem64_end = kmem64_aligned_end;
13037218Ssvemuri } else {
13047218Ssvemuri kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
13057218Ssvemuri }
13065648Ssetje kmem64_sz = kmem64_end - kmem64_base;
13070Sstevel@tonic-gate
13080Sstevel@tonic-gate if (&ecache_init_scrub_flush_area) {
13095648Ssetje alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
13105648Ssetje ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
13110Sstevel@tonic-gate }
13120Sstevel@tonic-gate
13130Sstevel@tonic-gate /*
13140Sstevel@tonic-gate * If physmem is patched to be non-zero, use it instead of
13150Sstevel@tonic-gate * the monitor value unless physmem is larger than the total
13160Sstevel@tonic-gate * amount of memory on hand.
13170Sstevel@tonic-gate */
13180Sstevel@tonic-gate if (physmem == 0 || physmem > npages)
13190Sstevel@tonic-gate physmem = npages;
13200Sstevel@tonic-gate
13210Sstevel@tonic-gate /*
13225648Ssetje * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
13235648Ssetje * is mounted as root. This memory is held down by OBP and unlike
13245648Ssetje * the stub boot_archive is never released.
13250Sstevel@tonic-gate *
13265648Ssetje * In order to get things sized correctly on lower memory
13275648Ssetje * machines (where the memory used by the ramdisk represents
13285648Ssetje * a significant portion of memory), physmem is adjusted.
13295648Ssetje *
13305648Ssetje * This is done by subtracting the ramdisk_size which is set
13315648Ssetje * to the size of the ramdisk (in Kb) in /etc/system at the
13325648Ssetje * time the miniroot archive is constructed.
13330Sstevel@tonic-gate */
13345872Ssetje if (root_is_ramdisk == B_TRUE) {
13355872Ssetje ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
13365872Ssetje physmem -= ramdisk_npages;
13375872Ssetje }
13385648Ssetje
13395648Ssetje if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
13405648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
13410Sstevel@tonic-gate
13424878Sblakej /*
13435648Ssetje * Allocate space for the interrupt vector table.
13443764Sdp78419 */
13455648Ssetje memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
13462973Sgovinda if (memspace != (caddr_t)intr_vec_table)
13473764Sdp78419 prom_panic("interrupt vector table allocation failure");
13480Sstevel@tonic-gate
13490Sstevel@tonic-gate /*
13500Sstevel@tonic-gate * Between now and when we finish copying in the memory lists,
13510Sstevel@tonic-gate * allocations happen so the space gets fragmented and the
13525648Ssetje * lists longer. Leave enough space for lists twice as
13535648Ssetje * long as we have now; then roundup to a pagesize.
13540Sstevel@tonic-gate */
13555648Ssetje memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
13565648Ssetje prom_phys_avail_len() + prom_virt_avail_len());
13575648Ssetje memlist_sz *= 2;
13585648Ssetje memlist_sz = roundup(memlist_sz, PAGESIZE);
13595648Ssetje memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
13600Sstevel@tonic-gate if (memspace == NULL)
13615648Ssetje cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
13620Sstevel@tonic-gate
13630Sstevel@tonic-gate memlist = (struct memlist *)memspace;
13640Sstevel@tonic-gate memlist_end = (char *)memspace + memlist_sz;
13650Sstevel@tonic-gate PRM_DEBUG(memlist);
13660Sstevel@tonic-gate PRM_DEBUG(memlist_end);
13675648Ssetje
13680Sstevel@tonic-gate PRM_DEBUG(sysbase);
13690Sstevel@tonic-gate PRM_DEBUG(syslimit);
13700Sstevel@tonic-gate kernelheap_init((void *)sysbase, (void *)syslimit,
13710Sstevel@tonic-gate (caddr_t)sysbase + PAGESIZE, NULL, NULL);
13720Sstevel@tonic-gate
13730Sstevel@tonic-gate /*
13740Sstevel@tonic-gate * Take the most current snapshot we can by calling mem-update.
13750Sstevel@tonic-gate */
13760Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
13770Sstevel@tonic-gate &boot_physavail, &boot_physavail_len,
13780Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len);
13790Sstevel@tonic-gate
13800Sstevel@tonic-gate /*
13815648Ssetje * Remove the space used by prom_alloc from the kernel heap
13820Sstevel@tonic-gate * plus the area actually used by the OBP (if any)
13830Sstevel@tonic-gate * ignoring virtual addresses in virt_avail, above syslimit.
13840Sstevel@tonic-gate */
13850Sstevel@tonic-gate virt_avail = memlist;
13860Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
13870Sstevel@tonic-gate
138811474SJonathan.Adams@Sun.COM for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
13890Sstevel@tonic-gate uint64_t range_base, range_size;
13900Sstevel@tonic-gate
139111474SJonathan.Adams@Sun.COM if ((range_base = cur->ml_address + cur->ml_size) <
139211474SJonathan.Adams@Sun.COM (uint64_t)sysbase)
13930Sstevel@tonic-gate continue;
13940Sstevel@tonic-gate if (range_base >= (uint64_t)syslimit)
13950Sstevel@tonic-gate break;
13960Sstevel@tonic-gate /*
13970Sstevel@tonic-gate * Limit the range to end at syslimit.
13980Sstevel@tonic-gate */
139911474SJonathan.Adams@Sun.COM range_size = MIN(cur->ml_next->ml_address,
14000Sstevel@tonic-gate (uint64_t)syslimit) - range_base;
14010Sstevel@tonic-gate (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
14020Sstevel@tonic-gate 0, 0, (void *)range_base, (void *)(range_base + range_size),
14030Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
14040Sstevel@tonic-gate }
14050Sstevel@tonic-gate
14060Sstevel@tonic-gate phys_avail = memlist;
14075648Ssetje copy_memlist(boot_physavail, boot_physavail_len, &memlist);
14080Sstevel@tonic-gate
14090Sstevel@tonic-gate /*
14104104Sblakej * Add any extra memory at the end of the ndata region if there's at
14114104Sblakej * least a page to add. There might be a few more pages available in
14124104Sblakej * the middle of the ndata region, but for now they are ignored.
14130Sstevel@tonic-gate */
14144104Sblakej nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
14154104Sblakej if (nalloc_base == NULL)
14160Sstevel@tonic-gate nalloc_base = nalloc_end;
14170Sstevel@tonic-gate ndata_remain_sz = nalloc_end - nalloc_base;
14180Sstevel@tonic-gate
14195648Ssetje /*
14205648Ssetje * Copy physinstalled list into kernel space.
14215648Ssetje */
14225648Ssetje phys_install = memlist;
14235648Ssetje copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
14245648Ssetje
14255648Ssetje /*
14265648Ssetje * Create list of physical addrs we don't need pp's for:
14275648Ssetje * kernel text 4M page
14285648Ssetje * kernel data 4M page - ndata_remain_sz
14295648Ssetje * kmem64 pages
14305648Ssetje *
14315648Ssetje * NB if adding any pages here, make sure no kpm page
14325648Ssetje * overlaps can occur (see ASSERTs in kphysm_memsegs)
14335648Ssetje */
14345648Ssetje nopp_list = memlist;
14355648Ssetje memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
14365648Ssetje memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
14375648Ssetje &memlist, &nopp_list);
14387218Ssvemuri
14397218Ssvemuri /* Don't add to nopp_list if kmem64 was allocated in smchunks */
14407218Ssvemuri if (!kmem64_smchunks)
14417218Ssvemuri memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
14420Sstevel@tonic-gate
14430Sstevel@tonic-gate if ((caddr_t)memlist > (memspace + memlist_sz))
14443764Sdp78419 prom_panic("memlist overflow");
14450Sstevel@tonic-gate
14460Sstevel@tonic-gate /*
14476880Sdv142724 * Size the pcf array based on the number of cpus in the box at
14486880Sdv142724 * boot time.
14496880Sdv142724 */
14506880Sdv142724 pcf_init();
14516880Sdv142724
14526880Sdv142724 /*
14530Sstevel@tonic-gate * Initialize the page structures from the memory lists.
14540Sstevel@tonic-gate */
14555648Ssetje kphysm_init();
14560Sstevel@tonic-gate
14570Sstevel@tonic-gate availrmem_initial = availrmem = freemem;
14580Sstevel@tonic-gate PRM_DEBUG(availrmem);
14590Sstevel@tonic-gate
14600Sstevel@tonic-gate /*
14610Sstevel@tonic-gate * Some of the locks depend on page_hashsz being set!
14620Sstevel@tonic-gate * kmem_init() depends on this; so, keep it here.
14630Sstevel@tonic-gate */
14640Sstevel@tonic-gate page_lock_init();
14650Sstevel@tonic-gate
14660Sstevel@tonic-gate /*
14670Sstevel@tonic-gate * Initialize kernel memory allocator.
14680Sstevel@tonic-gate */
14690Sstevel@tonic-gate kmem_init();
14700Sstevel@tonic-gate
14710Sstevel@tonic-gate /*
14723717Sdp78419 * Factor in colorequiv to check additional 'equivalent' bins
14733717Sdp78419 */
14743733Sdp78419 if (&page_set_colorequiv_arr_cpu != NULL)
14753733Sdp78419 page_set_colorequiv_arr_cpu();
14763733Sdp78419 else
14773733Sdp78419 page_set_colorequiv_arr();
14783717Sdp78419
14793717Sdp78419 /*
14800Sstevel@tonic-gate * Initialize bp_mapin().
14810Sstevel@tonic-gate */
14820Sstevel@tonic-gate bp_init(shm_alignment, HAT_STRICTORDER);
14830Sstevel@tonic-gate
14840Sstevel@tonic-gate /*
148511444SGangadhar.M@Sun.COM * Reserve space for MPO mblock structs from the 32-bit heap.
14860Sstevel@tonic-gate */
14870Sstevel@tonic-gate
14885358Sjc25722 if (mpo_heap32_bufsz > (size_t)0) {
14895358Sjc25722 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
14905358Sjc25722 PAGESIZE, 0, 0, mpo_heap32_buf,
14915358Sjc25722 mpo_heap32_buf + mpo_heap32_bufsz,
14925358Sjc25722 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
14935358Sjc25722 }
14940Sstevel@tonic-gate mem_config_init();
14950Sstevel@tonic-gate }
14960Sstevel@tonic-gate
14970Sstevel@tonic-gate static void
startup_modules(void)14980Sstevel@tonic-gate startup_modules(void)
14990Sstevel@tonic-gate {
15005648Ssetje int nhblk1, nhblk8;
15010Sstevel@tonic-gate size_t nhblksz;
15023764Sdp78419 pgcnt_t pages_per_hblk;
15030Sstevel@tonic-gate size_t hme8blk_sz, hme1blk_sz;
15040Sstevel@tonic-gate
15050Sstevel@tonic-gate /*
150612931SDave.Plauger@Sun.COM * The system file /etc/system was read already under startup_memlist.
15070Sstevel@tonic-gate */
15080Sstevel@tonic-gate if (&set_platform_defaults)
15090Sstevel@tonic-gate set_platform_defaults();
15100Sstevel@tonic-gate
15110Sstevel@tonic-gate /*
15120Sstevel@tonic-gate * Calculate default settings of system parameters based upon
15130Sstevel@tonic-gate * maxusers, yet allow to be overridden via the /etc/system file.
15140Sstevel@tonic-gate */
15150Sstevel@tonic-gate param_calc(0);
15160Sstevel@tonic-gate
15170Sstevel@tonic-gate mod_setup();
15180Sstevel@tonic-gate
15190Sstevel@tonic-gate /*
15200Sstevel@tonic-gate * If this is a positron, complain and halt.
15210Sstevel@tonic-gate */
15220Sstevel@tonic-gate if (&iam_positron && iam_positron()) {
15230Sstevel@tonic-gate cmn_err(CE_WARN, "This hardware platform is not supported"
15240Sstevel@tonic-gate " by this release of Solaris.\n");
15250Sstevel@tonic-gate #ifdef DEBUG
15260Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to resume */
15270Sstevel@tonic-gate cmn_err(CE_WARN, "Booting an unsupported platform.\n");
15280Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
15290Sstevel@tonic-gate
15300Sstevel@tonic-gate #else /* DEBUG */
15310Sstevel@tonic-gate halt(0);
15320Sstevel@tonic-gate #endif /* DEBUG */
15330Sstevel@tonic-gate }
15340Sstevel@tonic-gate
15350Sstevel@tonic-gate /*
15360Sstevel@tonic-gate * If we are running firmware that isn't 64-bit ready
15370Sstevel@tonic-gate * then complain and halt.
15380Sstevel@tonic-gate */
15390Sstevel@tonic-gate do_prom_version_check();
15400Sstevel@tonic-gate
15410Sstevel@tonic-gate /*
15420Sstevel@tonic-gate * Initialize system parameters
15430Sstevel@tonic-gate */
15440Sstevel@tonic-gate param_init();
15450Sstevel@tonic-gate
15460Sstevel@tonic-gate /*
15470Sstevel@tonic-gate * maxmem is the amount of physical memory we're playing with.
15480Sstevel@tonic-gate */
15490Sstevel@tonic-gate maxmem = physmem;
15500Sstevel@tonic-gate
15510Sstevel@tonic-gate /* Set segkp limits. */
15523446Smrj ncbase = kdi_segdebugbase;
15533446Smrj ncend = kdi_segdebugbase;
15540Sstevel@tonic-gate
15550Sstevel@tonic-gate /*
15560Sstevel@tonic-gate * Initialize the hat layer.
15570Sstevel@tonic-gate */
15580Sstevel@tonic-gate hat_init();
15590Sstevel@tonic-gate
15600Sstevel@tonic-gate /*
15610Sstevel@tonic-gate * Initialize segment management stuff.
15620Sstevel@tonic-gate */
15630Sstevel@tonic-gate seg_init();
15640Sstevel@tonic-gate
15650Sstevel@tonic-gate /*
15660Sstevel@tonic-gate * Create the va>tte handler, so the prom can understand
15670Sstevel@tonic-gate * kernel translations. The handler is installed later, just
15680Sstevel@tonic-gate * as we are about to take over the trap table from the prom.
15690Sstevel@tonic-gate */
15700Sstevel@tonic-gate create_va_to_tte();
15710Sstevel@tonic-gate
15720Sstevel@tonic-gate /*
15730Sstevel@tonic-gate * Load the forthdebugger (optional)
15740Sstevel@tonic-gate */
15750Sstevel@tonic-gate forthdebug_init();
15760Sstevel@tonic-gate
15770Sstevel@tonic-gate /*
15780Sstevel@tonic-gate * Create OBP node for console input callbacks
15790Sstevel@tonic-gate * if it is needed.
15800Sstevel@tonic-gate */
15811253Slq150181 startup_create_io_node();
15820Sstevel@tonic-gate
15830Sstevel@tonic-gate if (modloadonly("fs", "specfs") == -1)
15840Sstevel@tonic-gate halt("Can't load specfs");
15850Sstevel@tonic-gate
15860Sstevel@tonic-gate if (modloadonly("fs", "devfs") == -1)
15870Sstevel@tonic-gate halt("Can't load devfs");
15880Sstevel@tonic-gate
158911173SJonathan.Adams@Sun.COM if (modloadonly("fs", "procfs") == -1)
159011173SJonathan.Adams@Sun.COM halt("Can't load procfs");
159111173SJonathan.Adams@Sun.COM
15920Sstevel@tonic-gate if (modloadonly("misc", "swapgeneric") == -1)
15930Sstevel@tonic-gate halt("Can't load swapgeneric");
15940Sstevel@tonic-gate
15951676Sjpk (void) modloadonly("sys", "lbl_edition");
15961676Sjpk
15970Sstevel@tonic-gate dispinit();
15980Sstevel@tonic-gate
15990Sstevel@tonic-gate /*
16000Sstevel@tonic-gate * Infer meanings to the members of the idprom buffer.
16010Sstevel@tonic-gate */
16020Sstevel@tonic-gate parse_idprom();
16030Sstevel@tonic-gate
16040Sstevel@tonic-gate /* Read cluster configuration data. */
16050Sstevel@tonic-gate clconf_init();
16060Sstevel@tonic-gate
16070Sstevel@tonic-gate setup_ddi();
16080Sstevel@tonic-gate
16090Sstevel@tonic-gate /*
16100Sstevel@tonic-gate * Lets take this opportunity to load the root device.
16110Sstevel@tonic-gate */
16120Sstevel@tonic-gate if (loadrootmodules() != 0)
16130Sstevel@tonic-gate debug_enter("Can't load the root filesystem");
16140Sstevel@tonic-gate
16150Sstevel@tonic-gate /*
16160Sstevel@tonic-gate * Load tod driver module for the tod part found on this system.
16170Sstevel@tonic-gate * Recompute the cpu frequency/delays based on tod as tod part
16180Sstevel@tonic-gate * tends to keep time more accurately.
16190Sstevel@tonic-gate */
16200Sstevel@tonic-gate if (&load_tod_module)
16210Sstevel@tonic-gate load_tod_module();
16220Sstevel@tonic-gate
16230Sstevel@tonic-gate /*
16240Sstevel@tonic-gate * Allow platforms to load modules which might
16250Sstevel@tonic-gate * be needed after bootops are gone.
16260Sstevel@tonic-gate */
16270Sstevel@tonic-gate if (&load_platform_modules)
16280Sstevel@tonic-gate load_platform_modules();
16290Sstevel@tonic-gate
16300Sstevel@tonic-gate setcpudelay();
16310Sstevel@tonic-gate
16320Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
16330Sstevel@tonic-gate &boot_physavail, &boot_physavail_len,
16340Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len);
16350Sstevel@tonic-gate
16360Sstevel@tonic-gate /*
16370Sstevel@tonic-gate * Calculation and allocation of hmeblks needed to remap
16383764Sdp78419 * the memory allocated by PROM till now.
16393764Sdp78419 * Overestimate the number of hblk1 elements by assuming
16403764Sdp78419 * worst case of TTE64K mappings.
16413764Sdp78419 * sfmmu_hblk_alloc will panic if this calculation is wrong.
16420Sstevel@tonic-gate */
16433764Sdp78419 bop_alloc_pages = btopr(kmem64_end - kmem64_base);
16443764Sdp78419 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
16453764Sdp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
16463764Sdp78419 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
16473764Sdp78419
16483764Sdp78419 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
16493764Sdp78419
16503764Sdp78419 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */
16510Sstevel@tonic-gate hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
16520Sstevel@tonic-gate hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
16530Sstevel@tonic-gate
16543764Sdp78419 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
16553764Sdp78419
16560Sstevel@tonic-gate pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
16573764Sdp78419 nhblk8 = 0;
16583764Sdp78419 while (bop_alloc_pages > 1) {
16593764Sdp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
16603764Sdp78419 nhblk8 += bop_alloc_pages /= pages_per_hblk;
16613764Sdp78419 bop_alloc_pages *= hme8blk_sz;
16623764Sdp78419 bop_alloc_pages = btopr(bop_alloc_pages);
16633764Sdp78419 }
16643764Sdp78419 nhblk8 += 2;
16650Sstevel@tonic-gate
16660Sstevel@tonic-gate /*
16670Sstevel@tonic-gate * Since hblk8's can hold up to 64k of mappings aligned on a 64k
16680Sstevel@tonic-gate * boundary, the number of hblk8's needed to map the entries in the
16690Sstevel@tonic-gate * boot_virtavail list needs to be adjusted to take this into
16700Sstevel@tonic-gate * consideration. Thus, we need to add additional hblk8's since it
16710Sstevel@tonic-gate * is possible that an hblk8 will not have all 8 slots used due to
16720Sstevel@tonic-gate * alignment constraints. Since there were boot_virtavail_len entries
16730Sstevel@tonic-gate * in that list, we need to add that many hblk8's to the number
16740Sstevel@tonic-gate * already calculated to make sure we don't underestimate.
16750Sstevel@tonic-gate */
16760Sstevel@tonic-gate nhblk8 += boot_virtavail_len;
16770Sstevel@tonic-gate nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
16780Sstevel@tonic-gate
16790Sstevel@tonic-gate /* Allocate in pagesize chunks */
16800Sstevel@tonic-gate nhblksz = roundup(nhblksz, MMU_PAGESIZE);
16810Sstevel@tonic-gate hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
16820Sstevel@tonic-gate sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
16830Sstevel@tonic-gate }
16840Sstevel@tonic-gate
16850Sstevel@tonic-gate static void
startup_bop_gone(void)16860Sstevel@tonic-gate startup_bop_gone(void)
16870Sstevel@tonic-gate {
16880Sstevel@tonic-gate
16890Sstevel@tonic-gate /*
16901991Sheppo * Destroy the MD initialized at startup
16911991Sheppo * The startup initializes the MD framework
16921991Sheppo * using prom and BOP alloc free it now.
16931991Sheppo */
16941991Sheppo mach_descrip_startup_fini();
16951991Sheppo
16961991Sheppo /*
16975648Ssetje * We're done with prom allocations.
16980Sstevel@tonic-gate */
16995648Ssetje bop_fini();
17000Sstevel@tonic-gate
17010Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
17020Sstevel@tonic-gate &boot_physavail, &boot_physavail_len,
17030Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len);
17040Sstevel@tonic-gate
17050Sstevel@tonic-gate /*
17060Sstevel@tonic-gate * setup physically contiguous area twice as large as the ecache.
17070Sstevel@tonic-gate * this is used while doing displacement flush of ecaches
17080Sstevel@tonic-gate */
17090Sstevel@tonic-gate if (&ecache_flush_address) {
17100Sstevel@tonic-gate ecache_flushaddr = ecache_flush_address();
17110Sstevel@tonic-gate if (ecache_flushaddr == (uint64_t)-1) {
17120Sstevel@tonic-gate cmn_err(CE_PANIC,
17130Sstevel@tonic-gate "startup: no memory to set ecache_flushaddr");
17140Sstevel@tonic-gate }
17150Sstevel@tonic-gate }
17160Sstevel@tonic-gate
17170Sstevel@tonic-gate /*
17180Sstevel@tonic-gate * Virtual available next.
17190Sstevel@tonic-gate */
17200Sstevel@tonic-gate ASSERT(virt_avail != NULL);
17210Sstevel@tonic-gate memlist_free_list(virt_avail);
17220Sstevel@tonic-gate virt_avail = memlist;
17230Sstevel@tonic-gate copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
17240Sstevel@tonic-gate
17250Sstevel@tonic-gate }
17260Sstevel@tonic-gate
17270Sstevel@tonic-gate
17280Sstevel@tonic-gate /*
17290Sstevel@tonic-gate * startup_fixup_physavail - called from mach_sfmmu.c after the final
17300Sstevel@tonic-gate * allocations have been performed. We can't call it in startup_bop_gone
17310Sstevel@tonic-gate * since later operations can cause obp to allocate more memory.
17320Sstevel@tonic-gate */
17330Sstevel@tonic-gate void
startup_fixup_physavail(void)17340Sstevel@tonic-gate startup_fixup_physavail(void)
17350Sstevel@tonic-gate {
17360Sstevel@tonic-gate struct memlist *cur;
17373764Sdp78419 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
17383764Sdp78419
17393764Sdp78419 PRM_DEBUG(kmem64_overmap_size);
17400Sstevel@tonic-gate
17410Sstevel@tonic-gate /*
17420Sstevel@tonic-gate * take the most current snapshot we can by calling mem-update
17430Sstevel@tonic-gate */
17440Sstevel@tonic-gate copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
17450Sstevel@tonic-gate &boot_physavail, &boot_physavail_len,
17460Sstevel@tonic-gate &boot_virtavail, &boot_virtavail_len);
17470Sstevel@tonic-gate
17480Sstevel@tonic-gate /*
17490Sstevel@tonic-gate * Copy phys_avail list, again.
17500Sstevel@tonic-gate * Both the kernel/boot and the prom have been allocating
17510Sstevel@tonic-gate * from the original list we copied earlier.
17520Sstevel@tonic-gate */
17530Sstevel@tonic-gate cur = memlist;
17545648Ssetje copy_memlist(boot_physavail, boot_physavail_len, &memlist);
17550Sstevel@tonic-gate
17560Sstevel@tonic-gate /*
17573764Sdp78419 * Add any unused kmem64 memory from overmapped page
17583764Sdp78419 * (Note: va_to_pa does not work for kmem64_end)
17593764Sdp78419 */
17603764Sdp78419 if (kmem64_overmap_size) {
17613764Sdp78419 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
17625648Ssetje kmem64_overmap_size, &memlist, &cur);
17633764Sdp78419 }
17643764Sdp78419
17653764Sdp78419 /*
17665648Ssetje * Add any extra memory after e_data we added to the phys_avail list
17670Sstevel@tonic-gate * back to the old list.
17680Sstevel@tonic-gate */
17690Sstevel@tonic-gate if (ndata_remain_sz >= MMU_PAGESIZE)
17700Sstevel@tonic-gate memlist_add(va_to_pa(nalloc_base),
17710Sstevel@tonic-gate (uint64_t)ndata_remain_sz, &memlist, &cur);
17720Sstevel@tonic-gate
17730Sstevel@tonic-gate /*
17740Sstevel@tonic-gate * There isn't any bounds checking on the memlist area
17750Sstevel@tonic-gate * so ensure it hasn't overgrown.
17760Sstevel@tonic-gate */
17770Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end)
17780Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: memlist size exceeded");
17790Sstevel@tonic-gate
17800Sstevel@tonic-gate /*
17810Sstevel@tonic-gate * The kernel removes the pages that were allocated for it from
17820Sstevel@tonic-gate * the freelist, but we now have to find any -extra- pages that
17830Sstevel@tonic-gate * the prom has allocated for it's own book-keeping, and remove
17840Sstevel@tonic-gate * them from the freelist too. sigh.
17850Sstevel@tonic-gate */
17865648Ssetje sync_memlists(phys_avail, cur);
17870Sstevel@tonic-gate
17880Sstevel@tonic-gate ASSERT(phys_avail != NULL);
17895872Ssetje
17905872Ssetje old_phys_avail = phys_avail;
17910Sstevel@tonic-gate phys_avail = cur;
17925872Ssetje }
17935872Ssetje
17945872Ssetje void
update_kcage_ranges(uint64_t addr,uint64_t len)17955872Ssetje update_kcage_ranges(uint64_t addr, uint64_t len)
17965872Ssetje {
17975872Ssetje pfn_t base = btop(addr);
17985872Ssetje pgcnt_t num = btop(len);
17995872Ssetje int rv;
18005872Ssetje
18015872Ssetje rv = kcage_range_add(base, num, kcage_startup_dir);
18025872Ssetje
18035872Ssetje if (rv == ENOMEM) {
18045872Ssetje cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
18055872Ssetje (len == 0 ? 0 : BYTES_TO_MB(len)));
18065872Ssetje } else if (rv != 0) {
18075872Ssetje /* catch this in debug kernels */
18085872Ssetje ASSERT(0);
18095872Ssetje
18105872Ssetje cmn_err(CE_WARN, "unexpected kcage_range_add"
18115872Ssetje " return value %d", rv);
18125872Ssetje }
18130Sstevel@tonic-gate }
18140Sstevel@tonic-gate
18150Sstevel@tonic-gate static void
startup_vm(void)18160Sstevel@tonic-gate startup_vm(void)
18170Sstevel@tonic-gate {
18180Sstevel@tonic-gate size_t i;
18190Sstevel@tonic-gate struct segmap_crargs a;
18200Sstevel@tonic-gate struct segkpm_crargs b;
18210Sstevel@tonic-gate
18220Sstevel@tonic-gate uint64_t avmem;
18230Sstevel@tonic-gate caddr_t va;
18240Sstevel@tonic-gate pgcnt_t max_phys_segkp;
18250Sstevel@tonic-gate int mnode;
18260Sstevel@tonic-gate
18272991Ssusans extern int use_brk_lpg, use_stk_lpg;
1828423Sdavemq
18290Sstevel@tonic-gate /*
18300Sstevel@tonic-gate * get prom's mappings, create hments for them and switch
18310Sstevel@tonic-gate * to the kernel context.
18320Sstevel@tonic-gate */
18330Sstevel@tonic-gate hat_kern_setup();
18340Sstevel@tonic-gate
18350Sstevel@tonic-gate /*
18360Sstevel@tonic-gate * Take over trap table
18370Sstevel@tonic-gate */
18380Sstevel@tonic-gate setup_trap_table();
18390Sstevel@tonic-gate
18400Sstevel@tonic-gate /*
18410Sstevel@tonic-gate * Install the va>tte handler, so that the prom can handle
18420Sstevel@tonic-gate * misses and understand the kernel table layout in case
18430Sstevel@tonic-gate * we need call into the prom.
18440Sstevel@tonic-gate */
18450Sstevel@tonic-gate install_va_to_tte();
18460Sstevel@tonic-gate
18470Sstevel@tonic-gate /*
18480Sstevel@tonic-gate * Set a flag to indicate that the tba has been taken over.
18490Sstevel@tonic-gate */
18500Sstevel@tonic-gate tba_taken_over = 1;
18510Sstevel@tonic-gate
18520Sstevel@tonic-gate /* initialize MMU primary context register */
18530Sstevel@tonic-gate mmu_init_kcontext();
18540Sstevel@tonic-gate
18550Sstevel@tonic-gate /*
18560Sstevel@tonic-gate * The boot cpu can now take interrupts, x-calls, x-traps
18570Sstevel@tonic-gate */
18580Sstevel@tonic-gate CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
18590Sstevel@tonic-gate CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
18600Sstevel@tonic-gate
18610Sstevel@tonic-gate /*
18620Sstevel@tonic-gate * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
18630Sstevel@tonic-gate */
18640Sstevel@tonic-gate tbr_wr_addr_inited = 1;
18650Sstevel@tonic-gate
18660Sstevel@tonic-gate /*
18670Sstevel@tonic-gate * Initialize VM system, and map kernel address space.
18680Sstevel@tonic-gate */
18690Sstevel@tonic-gate kvm_init();
18700Sstevel@tonic-gate
18715872Ssetje ASSERT(old_phys_avail != NULL && phys_avail != NULL);
18725872Ssetje if (kernel_cage_enable) {
18735872Ssetje diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
18745872Ssetje }
18755872Ssetje memlist_free_list(old_phys_avail);
18765872Ssetje
18770Sstevel@tonic-gate /*
18780Sstevel@tonic-gate * If the following is true, someone has patched
18790Sstevel@tonic-gate * phsymem to be less than the number of pages that
18800Sstevel@tonic-gate * the system actually has. Remove pages until system
18810Sstevel@tonic-gate * memory is limited to the requested amount. Since we
18820Sstevel@tonic-gate * have allocated page structures for all pages, we
18830Sstevel@tonic-gate * correct the amount of memory we want to remove
18840Sstevel@tonic-gate * by the size of the memory used to hold page structures
18850Sstevel@tonic-gate * for the non-used pages.
18860Sstevel@tonic-gate */
18875872Ssetje if (physmem + ramdisk_npages < npages) {
18880Sstevel@tonic-gate pgcnt_t diff, off;
18890Sstevel@tonic-gate struct page *pp;
18900Sstevel@tonic-gate struct seg kseg;
18910Sstevel@tonic-gate
18920Sstevel@tonic-gate cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
18930Sstevel@tonic-gate
18940Sstevel@tonic-gate off = 0;
18955872Ssetje diff = npages - (physmem + ramdisk_npages);
18960Sstevel@tonic-gate diff -= mmu_btopr(diff * sizeof (struct page));
18970Sstevel@tonic-gate kseg.s_as = &kas;
18980Sstevel@tonic-gate while (diff--) {
18990Sstevel@tonic-gate pp = page_create_va(&unused_pages_vp, (offset_t)off,
19000Sstevel@tonic-gate MMU_PAGESIZE, PG_WAIT | PG_EXCL,
19010Sstevel@tonic-gate &kseg, (caddr_t)off);
19020Sstevel@tonic-gate if (pp == NULL)
19030Sstevel@tonic-gate cmn_err(CE_PANIC, "limited physmem too much!");
19040Sstevel@tonic-gate page_io_unlock(pp);
19050Sstevel@tonic-gate page_downgrade(pp);
19060Sstevel@tonic-gate availrmem--;
19070Sstevel@tonic-gate off += MMU_PAGESIZE;
19080Sstevel@tonic-gate }
19090Sstevel@tonic-gate }
19100Sstevel@tonic-gate
19110Sstevel@tonic-gate /*
19120Sstevel@tonic-gate * When printing memory, show the total as physmem less
19130Sstevel@tonic-gate * that stolen by a debugger.
19140Sstevel@tonic-gate */
19150Sstevel@tonic-gate cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
19160Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 10),
19170Sstevel@tonic-gate (ulong_t)(physinstalled) << (PAGESHIFT - 12));
19180Sstevel@tonic-gate
19190Sstevel@tonic-gate avmem = (uint64_t)freemem << PAGESHIFT;
19200Sstevel@tonic-gate cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
19210Sstevel@tonic-gate
19222991Ssusans /*
19232991Ssusans * For small memory systems disable automatic large pages.
19242991Ssusans */
19252991Ssusans if (physmem < privm_lpg_min_physmem) {
1926423Sdavemq use_brk_lpg = 0;
1927423Sdavemq use_stk_lpg = 0;
1928423Sdavemq }
1929423Sdavemq
19300Sstevel@tonic-gate /*
19310Sstevel@tonic-gate * Perform platform specific freelist processing
19320Sstevel@tonic-gate */
19330Sstevel@tonic-gate if (&plat_freelist_process) {
19340Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++)
19350Sstevel@tonic-gate if (mem_node_config[mnode].exists)
19360Sstevel@tonic-gate plat_freelist_process(mnode);
19370Sstevel@tonic-gate }
19380Sstevel@tonic-gate
19390Sstevel@tonic-gate /*
19400Sstevel@tonic-gate * Initialize the segkp segment type. We position it
19410Sstevel@tonic-gate * after the configured tables and buffers (whose end
19420Sstevel@tonic-gate * is given by econtig) and before V_WKBASE_ADDR.
19430Sstevel@tonic-gate * Also in this area is segkmap (size SEGMAPSIZE).
19440Sstevel@tonic-gate */
19450Sstevel@tonic-gate
19460Sstevel@tonic-gate /* XXX - cache alignment? */
19470Sstevel@tonic-gate va = (caddr_t)SEGKPBASE;
19480Sstevel@tonic-gate ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
19490Sstevel@tonic-gate
19500Sstevel@tonic-gate max_phys_segkp = (physmem * 2);
19510Sstevel@tonic-gate
19520Sstevel@tonic-gate if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
19530Sstevel@tonic-gate segkpsize = btop(SEGKPDEFSIZE);
19540Sstevel@tonic-gate cmn_err(CE_WARN, "Illegal value for segkpsize. "
19550Sstevel@tonic-gate "segkpsize has been reset to %ld pages", segkpsize);
19560Sstevel@tonic-gate }
19570Sstevel@tonic-gate
19580Sstevel@tonic-gate i = ptob(MIN(segkpsize, max_phys_segkp));
19590Sstevel@tonic-gate
19600Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER);
19610Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkp) < 0)
19620Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: cannot attach segkp");
19630Sstevel@tonic-gate if (segkp_create(segkp) != 0)
19640Sstevel@tonic-gate cmn_err(CE_PANIC, "startup: segkp_create failed");
19650Sstevel@tonic-gate rw_exit(&kas.a_lock);
19660Sstevel@tonic-gate
19670Sstevel@tonic-gate /*
19680Sstevel@tonic-gate * kpm segment
19690Sstevel@tonic-gate */
19700Sstevel@tonic-gate segmap_kpm = kpm_enable &&
19714528Spaulsan segmap_kpm && PAGESIZE == MAXBSIZE;
19720Sstevel@tonic-gate
19730Sstevel@tonic-gate if (kpm_enable) {
19740Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER);
19750Sstevel@tonic-gate
19760Sstevel@tonic-gate /*
19770Sstevel@tonic-gate * The segkpm virtual range range is larger than the
19780Sstevel@tonic-gate * actual physical memory size and also covers gaps in
19790Sstevel@tonic-gate * the physical address range for the following reasons:
19800Sstevel@tonic-gate * . keep conversion between segkpm and physical addresses
19810Sstevel@tonic-gate * simple, cheap and unambiguous.
19820Sstevel@tonic-gate * . avoid extension/shrink of the the segkpm in case of DR.
19830Sstevel@tonic-gate * . avoid complexity for handling of virtual addressed
19840Sstevel@tonic-gate * caches, segkpm and the regular mapping scheme must be
19850Sstevel@tonic-gate * kept in sync wrt. the virtual color of mapped pages.
19860Sstevel@tonic-gate * Any accesses to virtual segkpm ranges not backed by
19870Sstevel@tonic-gate * physical memory will fall through the memseg pfn hash
19880Sstevel@tonic-gate * and will be handled in segkpm_fault.
19890Sstevel@tonic-gate * Additional kpm_size spaces needed for vac alias prevention.
19900Sstevel@tonic-gate */
19910Sstevel@tonic-gate if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
19920Sstevel@tonic-gate segkpm) < 0)
19930Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkpm");
19940Sstevel@tonic-gate
19950Sstevel@tonic-gate b.prot = PROT_READ | PROT_WRITE;
19960Sstevel@tonic-gate b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
19970Sstevel@tonic-gate
19980Sstevel@tonic-gate if (segkpm_create(segkpm, (caddr_t)&b) != 0)
19990Sstevel@tonic-gate panic("segkpm_create segkpm");
20000Sstevel@tonic-gate
20010Sstevel@tonic-gate rw_exit(&kas.a_lock);
20022296Sae112802
20032296Sae112802 mach_kpm_init();
20040Sstevel@tonic-gate }
20050Sstevel@tonic-gate
200610106SJason.Beloro@Sun.COM va = kpm_vbase + (kpm_size * vac_colors);
200710106SJason.Beloro@Sun.COM
20083290Sjohansen if (!segzio_fromheap) {
20093290Sjohansen size_t size;
20103552Sjohansen size_t physmem_b = mmu_ptob(physmem);
20113290Sjohansen
20123290Sjohansen /* size is in bytes, segziosize is in pages */
20133290Sjohansen if (segziosize == 0) {
20143552Sjohansen size = physmem_b;
20153290Sjohansen } else {
20163290Sjohansen size = mmu_ptob(segziosize);
20173290Sjohansen }
20183290Sjohansen
20193290Sjohansen if (size < SEGZIOMINSIZE) {
20203290Sjohansen size = SEGZIOMINSIZE;
20213552Sjohansen } else if (size > SEGZIOMAXSIZE) {
20223552Sjohansen size = SEGZIOMAXSIZE;
20233552Sjohansen /*
20243552Sjohansen * On 64-bit x86, we only have 2TB of KVA. This exists
20253552Sjohansen * for parity with x86.
20263552Sjohansen *
20273552Sjohansen * SEGZIOMAXSIZE is capped at 512gb so that segzio
20283552Sjohansen * doesn't consume all of KVA. However, if we have a
20293552Sjohansen * system that has more thant 512gb of physical memory,
20303552Sjohansen * we can actually consume about half of the difference
20313552Sjohansen * between 512gb and the rest of the available physical
20323552Sjohansen * memory.
20333552Sjohansen */
20343552Sjohansen if (physmem_b > SEGZIOMAXSIZE) {
20353552Sjohansen size += (physmem_b - SEGZIOMAXSIZE) / 2;
20363552Sjohansen }
20373290Sjohansen }
20383290Sjohansen segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
20393290Sjohansen /* put the base of the ZIO segment after the kpm segment */
204010106SJason.Beloro@Sun.COM segzio_base = va;
204110106SJason.Beloro@Sun.COM va += mmu_ptob(segziosize);
20423290Sjohansen PRM_DEBUG(segziosize);
20433290Sjohansen PRM_DEBUG(segzio_base);
20443290Sjohansen
20453290Sjohansen /*
20463290Sjohansen * On some platforms, kvm_init is called after the kpm
20473290Sjohansen * sizes have been determined. On SPARC, kvm_init is called
20483290Sjohansen * before, so we have to attach the kzioseg after kvm is
20493290Sjohansen * initialized, otherwise we'll try to allocate from the boot
20503290Sjohansen * area since the kernel heap hasn't yet been configured.
20513290Sjohansen */
20523290Sjohansen rw_enter(&kas.a_lock, RW_WRITER);
20533290Sjohansen
20543290Sjohansen (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
20553290Sjohansen &kzioseg);
20563290Sjohansen (void) segkmem_zio_create(&kzioseg);
20573290Sjohansen
20583290Sjohansen /* create zio area covering new segment */
20593290Sjohansen segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
20603290Sjohansen
20613290Sjohansen rw_exit(&kas.a_lock);
20623290Sjohansen }
20633290Sjohansen
206410106SJason.Beloro@Sun.COM if (ppvm_enable) {
206510106SJason.Beloro@Sun.COM caddr_t ppvm_max;
206610106SJason.Beloro@Sun.COM
206710106SJason.Beloro@Sun.COM /*
206810106SJason.Beloro@Sun.COM * ppvm refers to the static VA space used to map
206910106SJason.Beloro@Sun.COM * the page_t's for dynamically added memory.
207010106SJason.Beloro@Sun.COM *
207110106SJason.Beloro@Sun.COM * ppvm_base should not cross a potential VA hole.
207210106SJason.Beloro@Sun.COM *
207310106SJason.Beloro@Sun.COM * ppvm_size should be large enough to map the
207410106SJason.Beloro@Sun.COM * page_t's needed to manage all of KPM range.
207510106SJason.Beloro@Sun.COM */
207610106SJason.Beloro@Sun.COM ppvm_size =
207710106SJason.Beloro@Sun.COM roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
207810106SJason.Beloro@Sun.COM MMU_PAGESIZE);
207910106SJason.Beloro@Sun.COM ppvm_max = (caddr_t)(0ull - ppvm_size);
208010106SJason.Beloro@Sun.COM ppvm_base = (page_t *)va;
208110106SJason.Beloro@Sun.COM
208210106SJason.Beloro@Sun.COM if ((caddr_t)ppvm_base <= hole_end) {
208310106SJason.Beloro@Sun.COM cmn_err(CE_WARN,
208410106SJason.Beloro@Sun.COM "Memory DR disabled: invalid DR map base: 0x%p\n",
208510106SJason.Beloro@Sun.COM (void *)ppvm_base);
208610106SJason.Beloro@Sun.COM ppvm_enable = 0;
208710106SJason.Beloro@Sun.COM } else if ((caddr_t)ppvm_base > ppvm_max) {
208810106SJason.Beloro@Sun.COM uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
208910106SJason.Beloro@Sun.COM
209010106SJason.Beloro@Sun.COM cmn_err(CE_WARN,
209110106SJason.Beloro@Sun.COM "Memory DR disabled: insufficient DR map size:"
209210106SJason.Beloro@Sun.COM " 0x%lx (needed 0x%lx)\n",
209310106SJason.Beloro@Sun.COM ppvm_size - diff, ppvm_size);
209410106SJason.Beloro@Sun.COM ppvm_enable = 0;
209510106SJason.Beloro@Sun.COM }
209610106SJason.Beloro@Sun.COM PRM_DEBUG(ppvm_size);
209710106SJason.Beloro@Sun.COM PRM_DEBUG(ppvm_base);
209810106SJason.Beloro@Sun.COM }
20993290Sjohansen
21000Sstevel@tonic-gate /*
21010Sstevel@tonic-gate * Now create generic mapping segment. This mapping
21020Sstevel@tonic-gate * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total
21030Sstevel@tonic-gate * virtual address is greater than the amount of free
21040Sstevel@tonic-gate * memory that is available, then we trim back the
21050Sstevel@tonic-gate * segment size to that amount
21060Sstevel@tonic-gate */
21070Sstevel@tonic-gate va = (caddr_t)SEGMAPBASE;
21080Sstevel@tonic-gate
21090Sstevel@tonic-gate /*
21100Sstevel@tonic-gate * 1201049: segkmap base address must be MAXBSIZE aligned
21110Sstevel@tonic-gate */
21120Sstevel@tonic-gate ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
21130Sstevel@tonic-gate
21140Sstevel@tonic-gate /*
21150Sstevel@tonic-gate * Set size of segmap to percentage of freemem at boot,
21160Sstevel@tonic-gate * but stay within the allowable range
21170Sstevel@tonic-gate * Note we take percentage before converting from pages
21180Sstevel@tonic-gate * to bytes to avoid an overflow on 32-bit kernels.
21190Sstevel@tonic-gate */
21200Sstevel@tonic-gate i = mmu_ptob((freemem * segmap_percent) / 100);
21210Sstevel@tonic-gate
21220Sstevel@tonic-gate if (i < MINMAPSIZE)
21230Sstevel@tonic-gate i = MINMAPSIZE;
21240Sstevel@tonic-gate
21250Sstevel@tonic-gate if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
21260Sstevel@tonic-gate i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
21270Sstevel@tonic-gate
21280Sstevel@tonic-gate i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */
21290Sstevel@tonic-gate
21300Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER);
21310Sstevel@tonic-gate if (seg_attach(&kas, va, i, segkmap) < 0)
21320Sstevel@tonic-gate cmn_err(CE_PANIC, "cannot attach segkmap");
21330Sstevel@tonic-gate
21340Sstevel@tonic-gate a.prot = PROT_READ | PROT_WRITE;
21350Sstevel@tonic-gate a.shmsize = shm_alignment;
21360Sstevel@tonic-gate a.nfreelist = 0; /* use segmap driver defaults */
21370Sstevel@tonic-gate
21380Sstevel@tonic-gate if (segmap_create(segkmap, (caddr_t)&a) != 0)
21390Sstevel@tonic-gate panic("segmap_create segkmap");
21400Sstevel@tonic-gate rw_exit(&kas.a_lock);
21410Sstevel@tonic-gate
21420Sstevel@tonic-gate segdev_init();
21430Sstevel@tonic-gate }
21440Sstevel@tonic-gate
21450Sstevel@tonic-gate static void
startup_end(void)21460Sstevel@tonic-gate startup_end(void)
21470Sstevel@tonic-gate {
21480Sstevel@tonic-gate if ((caddr_t)memlist > (caddr_t)memlist_end)
21490Sstevel@tonic-gate panic("memlist overflow 2");
21500Sstevel@tonic-gate memlist_free_block((caddr_t)memlist,
21510Sstevel@tonic-gate ((caddr_t)memlist_end - (caddr_t)memlist));
21520Sstevel@tonic-gate memlist = NULL;
21530Sstevel@tonic-gate
21540Sstevel@tonic-gate /* enable page_relocation since OBP is now done */
21550Sstevel@tonic-gate page_relocate_ready = 1;
21560Sstevel@tonic-gate
21570Sstevel@tonic-gate /*
21580Sstevel@tonic-gate * Perform tasks that get done after most of the VM
21590Sstevel@tonic-gate * initialization has been done but before the clock
21600Sstevel@tonic-gate * and other devices get started.
21610Sstevel@tonic-gate */
21620Sstevel@tonic-gate kern_setup1();
21630Sstevel@tonic-gate
21640Sstevel@tonic-gate /*
21658803SJonathan.Haslam@Sun.COM * Perform CPC initialization for this CPU.
21668803SJonathan.Haslam@Sun.COM */
21678803SJonathan.Haslam@Sun.COM kcpc_hw_init();
21688803SJonathan.Haslam@Sun.COM
21698803SJonathan.Haslam@Sun.COM /*
21700Sstevel@tonic-gate * Intialize the VM arenas for allocating physically
21710Sstevel@tonic-gate * contiguus memory chunk for interrupt queues snd
21720Sstevel@tonic-gate * allocate/register boot cpu's queues, if any and
21730Sstevel@tonic-gate * allocate dump buffer for sun4v systems to store
21740Sstevel@tonic-gate * extra crash information during crash dump
21750Sstevel@tonic-gate */
21760Sstevel@tonic-gate contig_mem_init();
21770Sstevel@tonic-gate mach_descrip_init();
21784050Sjb145095
21794050Sjb145095 if (cpu_intrq_setup(CPU)) {
21804050Sjb145095 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
21814050Sjb145095 }
21820Sstevel@tonic-gate cpu_intrq_register(CPU);
21831077Ssvemuri mach_htraptrace_setup(CPU->cpu_id);
21841077Ssvemuri mach_htraptrace_configure(CPU->cpu_id);
21850Sstevel@tonic-gate mach_dump_buffer_init();
21860Sstevel@tonic-gate
21870Sstevel@tonic-gate /*
21880Sstevel@tonic-gate * Initialize interrupt related stuff
21890Sstevel@tonic-gate */
21901455Sandrei cpu_intr_alloc(CPU, NINTR_THREADS);
21910Sstevel@tonic-gate
21920Sstevel@tonic-gate (void) splzs(); /* allow hi clock ints but not zs */
21930Sstevel@tonic-gate
21940Sstevel@tonic-gate /*
21950Sstevel@tonic-gate * Initialize errors.
21960Sstevel@tonic-gate */
21970Sstevel@tonic-gate error_init();
21980Sstevel@tonic-gate
21990Sstevel@tonic-gate /*
22000Sstevel@tonic-gate * Note that we may have already used kernel bcopy before this
22010Sstevel@tonic-gate * point - but if you really care about this, adb the use_hw_*
22020Sstevel@tonic-gate * variables to 0 before rebooting.
22030Sstevel@tonic-gate */
22040Sstevel@tonic-gate mach_hw_copy_limit();
22050Sstevel@tonic-gate
22060Sstevel@tonic-gate /*
22070Sstevel@tonic-gate * Install the "real" preemption guards before DDI services
22080Sstevel@tonic-gate * are available.
22090Sstevel@tonic-gate */
22100Sstevel@tonic-gate (void) prom_set_preprom(kern_preprom);
22110Sstevel@tonic-gate (void) prom_set_postprom(kern_postprom);
22120Sstevel@tonic-gate CPU->cpu_m.mutex_ready = 1;
22130Sstevel@tonic-gate
22140Sstevel@tonic-gate /*
22150Sstevel@tonic-gate * Initialize segnf (kernel support for non-faulting loads).
22160Sstevel@tonic-gate */
22170Sstevel@tonic-gate segnf_init();
22180Sstevel@tonic-gate
22190Sstevel@tonic-gate /*
22200Sstevel@tonic-gate * Configure the root devinfo node.
22210Sstevel@tonic-gate */
22220Sstevel@tonic-gate configure(); /* set up devices */
22230Sstevel@tonic-gate mach_cpu_halt_idle();
22240Sstevel@tonic-gate }
22250Sstevel@tonic-gate
22260Sstevel@tonic-gate
22270Sstevel@tonic-gate void
post_startup(void)22280Sstevel@tonic-gate post_startup(void)
22290Sstevel@tonic-gate {
22300Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG
22310Sstevel@tonic-gate extern void init_ptl1_thread(void);
22320Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */
22330Sstevel@tonic-gate extern void abort_sequence_init(void);
22340Sstevel@tonic-gate
22350Sstevel@tonic-gate /*
22360Sstevel@tonic-gate * Set the system wide, processor-specific flags to be passed
22370Sstevel@tonic-gate * to userland via the aux vector for performance hints and
22380Sstevel@tonic-gate * instruction set extensions.
22390Sstevel@tonic-gate */
22400Sstevel@tonic-gate bind_hwcap();
22410Sstevel@tonic-gate
22420Sstevel@tonic-gate /*
22430Sstevel@tonic-gate * Startup memory scrubber (if any)
22440Sstevel@tonic-gate */
22450Sstevel@tonic-gate mach_memscrub();
22460Sstevel@tonic-gate
22470Sstevel@tonic-gate /*
22480Sstevel@tonic-gate * Allocate soft interrupt to handle abort sequence.
22490Sstevel@tonic-gate */
22500Sstevel@tonic-gate abort_sequence_init();
22510Sstevel@tonic-gate
22520Sstevel@tonic-gate /*
22530Sstevel@tonic-gate * Configure the rest of the system.
22540Sstevel@tonic-gate * Perform forceloading tasks for /etc/system.
22550Sstevel@tonic-gate */
22560Sstevel@tonic-gate (void) mod_sysctl(SYS_FORCELOAD, NULL);
22570Sstevel@tonic-gate /*
22580Sstevel@tonic-gate * ON4.0: Force /proc module in until clock interrupt handle fixed
22590Sstevel@tonic-gate * ON4.0: This must be fixed or restated in /etc/systems.
22600Sstevel@tonic-gate */
22610Sstevel@tonic-gate (void) modload("fs", "procfs");
22620Sstevel@tonic-gate
22631991Sheppo /* load machine class specific drivers */
22641991Sheppo load_mach_drivers();
22651991Sheppo
22661991Sheppo /* load platform specific drivers */
22670Sstevel@tonic-gate if (&load_platform_drivers)
22680Sstevel@tonic-gate load_platform_drivers();
22690Sstevel@tonic-gate
22700Sstevel@tonic-gate /* load vis simulation module, if we are running w/fpu off */
22710Sstevel@tonic-gate if (!fpu_exists) {
22720Sstevel@tonic-gate if (modload("misc", "vis") == -1)
22730Sstevel@tonic-gate halt("Can't load vis");
22740Sstevel@tonic-gate }
22750Sstevel@tonic-gate
22760Sstevel@tonic-gate mach_fpras();
22770Sstevel@tonic-gate
22780Sstevel@tonic-gate maxmem = freemem;
22790Sstevel@tonic-gate
22808906SEric.Saxe@Sun.COM pg_init();
22818906SEric.Saxe@Sun.COM
22820Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG
22830Sstevel@tonic-gate init_ptl1_thread();
22840Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */
22850Sstevel@tonic-gate }
22860Sstevel@tonic-gate
22870Sstevel@tonic-gate #ifdef PTL1_PANIC_DEBUG
22880Sstevel@tonic-gate int ptl1_panic_test = 0;
22890Sstevel@tonic-gate int ptl1_panic_xc_one_test = 0;
22900Sstevel@tonic-gate int ptl1_panic_xc_all_test = 0;
22910Sstevel@tonic-gate int ptl1_panic_xt_one_test = 0;
22920Sstevel@tonic-gate int ptl1_panic_xt_all_test = 0;
22930Sstevel@tonic-gate kthread_id_t ptl1_thread_p = NULL;
22940Sstevel@tonic-gate kcondvar_t ptl1_cv;
22950Sstevel@tonic-gate kmutex_t ptl1_mutex;
22960Sstevel@tonic-gate int ptl1_recurse_count_threshold = 0x40;
22970Sstevel@tonic-gate int ptl1_recurse_trap_threshold = 0x3d;
22980Sstevel@tonic-gate extern void ptl1_recurse(int, int);
22990Sstevel@tonic-gate extern void ptl1_panic_xt(int, int);
23000Sstevel@tonic-gate
23010Sstevel@tonic-gate /*
23020Sstevel@tonic-gate * Called once per second by timeout() to wake up
23030Sstevel@tonic-gate * the ptl1_panic thread to see if it should cause
23040Sstevel@tonic-gate * a trap to the ptl1_panic() code.
23050Sstevel@tonic-gate */
23060Sstevel@tonic-gate /* ARGSUSED */
23070Sstevel@tonic-gate static void
ptl1_wakeup(void * arg)23080Sstevel@tonic-gate ptl1_wakeup(void *arg)
23090Sstevel@tonic-gate {
23100Sstevel@tonic-gate mutex_enter(&ptl1_mutex);
23110Sstevel@tonic-gate cv_signal(&ptl1_cv);
23120Sstevel@tonic-gate mutex_exit(&ptl1_mutex);
23130Sstevel@tonic-gate }
23140Sstevel@tonic-gate
23150Sstevel@tonic-gate /*
23160Sstevel@tonic-gate * ptl1_panic cross call function:
23170Sstevel@tonic-gate * Needed because xc_one() and xc_some() can pass
23180Sstevel@tonic-gate * 64 bit args but ptl1_recurse() expects ints.
23190Sstevel@tonic-gate */
23200Sstevel@tonic-gate static void
ptl1_panic_xc(void)23210Sstevel@tonic-gate ptl1_panic_xc(void)
23220Sstevel@tonic-gate {
23230Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold,
23240Sstevel@tonic-gate ptl1_recurse_trap_threshold);
23250Sstevel@tonic-gate }
23260Sstevel@tonic-gate
23270Sstevel@tonic-gate /*
23280Sstevel@tonic-gate * The ptl1 thread waits for a global flag to be set
23290Sstevel@tonic-gate * and uses the recurse thresholds to set the stack depth
23300Sstevel@tonic-gate * to cause a ptl1_panic() directly via a call to ptl1_recurse
23310Sstevel@tonic-gate * or indirectly via the cross call and cross trap functions.
23320Sstevel@tonic-gate *
23330Sstevel@tonic-gate * This is useful testing stack overflows and normal
23340Sstevel@tonic-gate * ptl1_panic() states with a know stack frame.
23350Sstevel@tonic-gate *
23360Sstevel@tonic-gate * ptl1_recurse() is an asm function in ptl1_panic.s that
23370Sstevel@tonic-gate * sets the {In, Local, Out, and Global} registers to a
23380Sstevel@tonic-gate * know state on the stack and just prior to causing a
23390Sstevel@tonic-gate * test ptl1_panic trap.
23400Sstevel@tonic-gate */
23410Sstevel@tonic-gate static void
ptl1_thread(void)23420Sstevel@tonic-gate ptl1_thread(void)
23430Sstevel@tonic-gate {
23440Sstevel@tonic-gate mutex_enter(&ptl1_mutex);
23450Sstevel@tonic-gate while (ptl1_thread_p) {
23460Sstevel@tonic-gate cpuset_t other_cpus;
23470Sstevel@tonic-gate int cpu_id;
23480Sstevel@tonic-gate int my_cpu_id;
23490Sstevel@tonic-gate int target_cpu_id;
23500Sstevel@tonic-gate int target_found;
23510Sstevel@tonic-gate
23520Sstevel@tonic-gate if (ptl1_panic_test) {
23530Sstevel@tonic-gate ptl1_recurse(ptl1_recurse_count_threshold,
23540Sstevel@tonic-gate ptl1_recurse_trap_threshold);
23550Sstevel@tonic-gate }
23560Sstevel@tonic-gate
23570Sstevel@tonic-gate /*
23580Sstevel@tonic-gate * Find potential targets for x-call and x-trap,
23590Sstevel@tonic-gate * if any exist while preempt is disabled we
23600Sstevel@tonic-gate * start a ptl1_panic if requested via a
23610Sstevel@tonic-gate * globals.
23620Sstevel@tonic-gate */
23630Sstevel@tonic-gate kpreempt_disable();
23640Sstevel@tonic-gate my_cpu_id = CPU->cpu_id;
23650Sstevel@tonic-gate other_cpus = cpu_ready_set;
23660Sstevel@tonic-gate CPUSET_DEL(other_cpus, CPU->cpu_id);
23670Sstevel@tonic-gate target_found = 0;
23680Sstevel@tonic-gate if (!CPUSET_ISNULL(other_cpus)) {
23690Sstevel@tonic-gate /*
23700Sstevel@tonic-gate * Pick the first one
23710Sstevel@tonic-gate */
23720Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
23730Sstevel@tonic-gate if (cpu_id == my_cpu_id)
23740Sstevel@tonic-gate continue;
23750Sstevel@tonic-gate
23760Sstevel@tonic-gate if (CPU_XCALL_READY(cpu_id)) {
23770Sstevel@tonic-gate target_cpu_id = cpu_id;
23780Sstevel@tonic-gate target_found = 1;
23790Sstevel@tonic-gate break;
23800Sstevel@tonic-gate }
23810Sstevel@tonic-gate }
23820Sstevel@tonic-gate ASSERT(target_found);
23830Sstevel@tonic-gate
23840Sstevel@tonic-gate if (ptl1_panic_xc_one_test) {
23850Sstevel@tonic-gate xc_one(target_cpu_id,
23860Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0);
23870Sstevel@tonic-gate }
23880Sstevel@tonic-gate if (ptl1_panic_xc_all_test) {
23890Sstevel@tonic-gate xc_some(other_cpus,
23900Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xc, 0, 0);
23910Sstevel@tonic-gate }
23920Sstevel@tonic-gate if (ptl1_panic_xt_one_test) {
23930Sstevel@tonic-gate xt_one(target_cpu_id,
23940Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0);
23950Sstevel@tonic-gate }
23960Sstevel@tonic-gate if (ptl1_panic_xt_all_test) {
23970Sstevel@tonic-gate xt_some(other_cpus,
23980Sstevel@tonic-gate (xcfunc_t *)ptl1_panic_xt, 0, 0);
23990Sstevel@tonic-gate }
24000Sstevel@tonic-gate }
24010Sstevel@tonic-gate kpreempt_enable();
24020Sstevel@tonic-gate (void) timeout(ptl1_wakeup, NULL, hz);
24030Sstevel@tonic-gate (void) cv_wait(&ptl1_cv, &ptl1_mutex);
24040Sstevel@tonic-gate }
24050Sstevel@tonic-gate mutex_exit(&ptl1_mutex);
24060Sstevel@tonic-gate }
24070Sstevel@tonic-gate
24080Sstevel@tonic-gate /*
24090Sstevel@tonic-gate * Called during early startup to create the ptl1_thread
24100Sstevel@tonic-gate */
24110Sstevel@tonic-gate void
init_ptl1_thread(void)24120Sstevel@tonic-gate init_ptl1_thread(void)
24130Sstevel@tonic-gate {
24140Sstevel@tonic-gate ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
24150Sstevel@tonic-gate &p0, TS_RUN, 0);
24160Sstevel@tonic-gate }
24170Sstevel@tonic-gate #endif /* PTL1_PANIC_DEBUG */
24180Sstevel@tonic-gate
24190Sstevel@tonic-gate
24205648Ssetje static void
memlist_new(uint64_t start,uint64_t len,struct memlist ** memlistp)24215648Ssetje memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
24225648Ssetje {
24235648Ssetje struct memlist *new;
24245648Ssetje
24255648Ssetje new = *memlistp;
242611474SJonathan.Adams@Sun.COM new->ml_address = start;
242711474SJonathan.Adams@Sun.COM new->ml_size = len;
24285648Ssetje *memlistp = new + 1;
24295648Ssetje }
24305648Ssetje
24310Sstevel@tonic-gate /*
24320Sstevel@tonic-gate * Add to a memory list.
24330Sstevel@tonic-gate * start = start of new memory segment
24340Sstevel@tonic-gate * len = length of new memory segment in bytes
24350Sstevel@tonic-gate * memlistp = pointer to array of available memory segment structures
24360Sstevel@tonic-gate * curmemlistp = memory list to which to add segment.
24370Sstevel@tonic-gate */
24380Sstevel@tonic-gate static void
memlist_add(uint64_t start,uint64_t len,struct memlist ** memlistp,struct memlist ** curmemlistp)24390Sstevel@tonic-gate memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
24400Sstevel@tonic-gate struct memlist **curmemlistp)
24410Sstevel@tonic-gate {
24425648Ssetje struct memlist *new = *memlistp;
24435648Ssetje
24445648Ssetje memlist_new(start, len, memlistp);
24450Sstevel@tonic-gate memlist_insert(new, curmemlistp);
24460Sstevel@tonic-gate }
24470Sstevel@tonic-gate
24485648Ssetje static int
ndata_alloc_memseg(struct memlist * ndata,size_t avail)24495648Ssetje ndata_alloc_memseg(struct memlist *ndata, size_t avail)
24505648Ssetje {
24515648Ssetje int nseg;
24525648Ssetje size_t memseg_sz;
24535648Ssetje struct memseg *msp;
24545648Ssetje
24555648Ssetje /*
24565648Ssetje * The memseg list is for the chunks of physical memory that
24575648Ssetje * will be managed by the vm system. The number calculated is
24585648Ssetje * a guess as boot may fragment it more when memory allocations
24595648Ssetje * are made before kphysm_init().
24605648Ssetje */
24615648Ssetje memseg_sz = (avail + 10) * sizeof (struct memseg);
24625648Ssetje memseg_sz = roundup(memseg_sz, PAGESIZE);
24635648Ssetje nseg = memseg_sz / sizeof (struct memseg);
24645648Ssetje msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
24655648Ssetje if (msp == NULL)
24665648Ssetje return (1);
24675648Ssetje PRM_DEBUG(memseg_free);
24685648Ssetje
24695648Ssetje while (nseg--) {
24705648Ssetje msp->next = memseg_free;
24715648Ssetje memseg_free = msp;
24725648Ssetje msp++;
24735648Ssetje }
24745648Ssetje return (0);
24755648Ssetje }
24765648Ssetje
24770Sstevel@tonic-gate /*
24780Sstevel@tonic-gate * In the case of architectures that support dynamic addition of
24790Sstevel@tonic-gate * memory at run-time there are two cases where memsegs need to
24800Sstevel@tonic-gate * be initialized and added to the memseg list.
24810Sstevel@tonic-gate * 1) memsegs that are constructed at startup.
24820Sstevel@tonic-gate * 2) memsegs that are constructed at run-time on
24830Sstevel@tonic-gate * hot-plug capable architectures.
24840Sstevel@tonic-gate * This code was originally part of the function kphysm_init().
24850Sstevel@tonic-gate */
24860Sstevel@tonic-gate
24870Sstevel@tonic-gate static void
memseg_list_add(struct memseg * memsegp)24880Sstevel@tonic-gate memseg_list_add(struct memseg *memsegp)
24890Sstevel@tonic-gate {
24900Sstevel@tonic-gate struct memseg **prev_memsegp;
24910Sstevel@tonic-gate pgcnt_t num;
24920Sstevel@tonic-gate
24930Sstevel@tonic-gate /* insert in memseg list, decreasing number of pages order */
24940Sstevel@tonic-gate
24950Sstevel@tonic-gate num = MSEG_NPAGES(memsegp);
24960Sstevel@tonic-gate
24970Sstevel@tonic-gate for (prev_memsegp = &memsegs; *prev_memsegp;
24980Sstevel@tonic-gate prev_memsegp = &((*prev_memsegp)->next)) {
24990Sstevel@tonic-gate if (num > MSEG_NPAGES(*prev_memsegp))
25000Sstevel@tonic-gate break;
25010Sstevel@tonic-gate }
25020Sstevel@tonic-gate
25030Sstevel@tonic-gate memsegp->next = *prev_memsegp;
25040Sstevel@tonic-gate *prev_memsegp = memsegp;
25050Sstevel@tonic-gate
25060Sstevel@tonic-gate if (kpm_enable) {
25070Sstevel@tonic-gate memsegp->nextpa = (memsegp->next) ?
25084528Spaulsan va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
25090Sstevel@tonic-gate
25100Sstevel@tonic-gate if (prev_memsegp != &memsegs) {
25110Sstevel@tonic-gate struct memseg *msp;
25120Sstevel@tonic-gate msp = (struct memseg *)((caddr_t)prev_memsegp -
25134528Spaulsan offsetof(struct memseg, next));
25140Sstevel@tonic-gate msp->nextpa = va_to_pa(memsegp);
25150Sstevel@tonic-gate } else {
25160Sstevel@tonic-gate memsegspa = va_to_pa(memsegs);
25170Sstevel@tonic-gate }
25180Sstevel@tonic-gate }
25190Sstevel@tonic-gate }
25200Sstevel@tonic-gate
25210Sstevel@tonic-gate /*
25220Sstevel@tonic-gate * PSM add_physmem_cb(). US-II and newer processors have some
25230Sstevel@tonic-gate * flavor of the prefetch capability implemented. We exploit
25240Sstevel@tonic-gate * this capability for optimum performance.
25250Sstevel@tonic-gate */
25260Sstevel@tonic-gate #define PREFETCH_BYTES 64
25270Sstevel@tonic-gate
25280Sstevel@tonic-gate void
add_physmem_cb(page_t * pp,pfn_t pnum)25290Sstevel@tonic-gate add_physmem_cb(page_t *pp, pfn_t pnum)
25300Sstevel@tonic-gate {
25310Sstevel@tonic-gate extern void prefetch_page_w(void *);
25320Sstevel@tonic-gate
25330Sstevel@tonic-gate pp->p_pagenum = pnum;
25340Sstevel@tonic-gate
25350Sstevel@tonic-gate /*
25360Sstevel@tonic-gate * Prefetch one more page_t into E$. To prevent future
25370Sstevel@tonic-gate * mishaps with the sizeof(page_t) changing on us, we
25380Sstevel@tonic-gate * catch this on debug kernels if we can't bring in the
25390Sstevel@tonic-gate * entire hpage with 2 PREFETCH_BYTES reads. See
25400Sstevel@tonic-gate * also, sun4u/cpu/cpu_module.c
25410Sstevel@tonic-gate */
25420Sstevel@tonic-gate /*LINTED*/
25430Sstevel@tonic-gate ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
25440Sstevel@tonic-gate prefetch_page_w((char *)pp);
25450Sstevel@tonic-gate }
25460Sstevel@tonic-gate
25470Sstevel@tonic-gate /*
25485648Ssetje * Find memseg with given pfn
25495648Ssetje */
25505648Ssetje static struct memseg *
memseg_find(pfn_t base,pfn_t * next)25515648Ssetje memseg_find(pfn_t base, pfn_t *next)
25525648Ssetje {
25535648Ssetje struct memseg *seg;
25545648Ssetje
25555648Ssetje if (next != NULL)
25565648Ssetje *next = LONG_MAX;
25575648Ssetje for (seg = memsegs; seg != NULL; seg = seg->next) {
25585648Ssetje if (base >= seg->pages_base && base < seg->pages_end)
25595648Ssetje return (seg);
25605648Ssetje if (next != NULL && seg->pages_base > base &&
25615648Ssetje seg->pages_base < *next)
25625648Ssetje *next = seg->pages_base;
25635648Ssetje }
25645648Ssetje return (NULL);
25655648Ssetje }
25665648Ssetje
25675648Ssetje /*
25685648Ssetje * Put page allocated by OBP on prom_ppages
25690Sstevel@tonic-gate */
25700Sstevel@tonic-gate static void
kphysm_erase(uint64_t addr,uint64_t len)25715648Ssetje kphysm_erase(uint64_t addr, uint64_t len)
25725648Ssetje {
25735648Ssetje struct page *pp;
25745648Ssetje struct memseg *seg;
25755648Ssetje pfn_t base = btop(addr), next;
25765648Ssetje pgcnt_t num = btop(len);
25775648Ssetje
25785648Ssetje while (num != 0) {
25795648Ssetje pgcnt_t off, left;
25805648Ssetje
25815648Ssetje seg = memseg_find(base, &next);
25825648Ssetje if (seg == NULL) {
25835648Ssetje if (next == LONG_MAX)
25845648Ssetje break;
25855648Ssetje left = MIN(next - base, num);
25865648Ssetje base += left, num -= left;
25875648Ssetje continue;
25885648Ssetje }
25895648Ssetje off = base - seg->pages_base;
25905648Ssetje pp = seg->pages + off;
25915648Ssetje left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
25925648Ssetje while (num != left) {
25935648Ssetje /*
25945648Ssetje * init it, lock it, and hashin on prom_pages vp.
25955648Ssetje *
259610106SJason.Beloro@Sun.COM * Mark it as NONRELOC to let DR know the page
259710106SJason.Beloro@Sun.COM * is locked long term, otherwise DR hangs when
259810106SJason.Beloro@Sun.COM * trying to remove those pages.
259910106SJason.Beloro@Sun.COM *
26005648Ssetje * XXX vnode offsets on the prom_ppages vnode
26015648Ssetje * are page numbers (gack) for >32 bit
26025648Ssetje * physical memory machines.
26035648Ssetje */
260410106SJason.Beloro@Sun.COM PP_SETNORELOC(pp);
26055648Ssetje add_physmem_cb(pp, base);
26065648Ssetje if (page_trylock(pp, SE_EXCL) == 0)
26075648Ssetje cmn_err(CE_PANIC, "prom page locked");
260811185SSean.McEnroe@Sun.COM (void) page_hashin(pp, &promvp,
26095648Ssetje (offset_t)base, NULL);
26105648Ssetje (void) page_pp_lock(pp, 0, 1);
26115648Ssetje pp++, base++, num--;
26125648Ssetje }
26135648Ssetje }
26145648Ssetje }
26155648Ssetje
26165648Ssetje static page_t *ppnext;
26175648Ssetje static pgcnt_t ppleft;
26185648Ssetje
26195648Ssetje static void *kpm_ppnext;
26205648Ssetje static pgcnt_t kpm_ppleft;
26215648Ssetje
26225648Ssetje /*
26235648Ssetje * Create a memseg
26245648Ssetje */
26255648Ssetje static void
kphysm_memseg(uint64_t addr,uint64_t len)26265648Ssetje kphysm_memseg(uint64_t addr, uint64_t len)
26270Sstevel@tonic-gate {
26285648Ssetje pfn_t base = btop(addr);
26295648Ssetje pgcnt_t num = btop(len);
26305648Ssetje struct memseg *seg;
26315648Ssetje
26325648Ssetje seg = memseg_free;
26335648Ssetje memseg_free = seg->next;
26345648Ssetje ASSERT(seg != NULL);
26355648Ssetje
26365648Ssetje seg->pages = ppnext;
26375648Ssetje seg->epages = ppnext + num;
26385648Ssetje seg->pages_base = base;
26395648Ssetje seg->pages_end = base + num;
26405648Ssetje ppnext += num;
26415648Ssetje ppleft -= num;
26425648Ssetje
26435648Ssetje if (kpm_enable) {
26445648Ssetje pgcnt_t kpnum = ptokpmpr(num);
26455648Ssetje
26465648Ssetje if (kpnum > kpm_ppleft)
26475648Ssetje panic("kphysm_memseg: kpm_pp overflow");
26485648Ssetje seg->pagespa = va_to_pa(seg->pages);
26495648Ssetje seg->epagespa = va_to_pa(seg->epages);
26505648Ssetje seg->kpm_pbase = kpmptop(ptokpmp(base));
26515648Ssetje seg->kpm_nkpmpgs = kpnum;
26525648Ssetje /*
26535648Ssetje * In the kpm_smallpage case, the kpm array
26545648Ssetje * is 1-1 wrt the page array
26555648Ssetje */
26565648Ssetje if (kpm_smallpages) {
26575648Ssetje kpm_spage_t *kpm_pp = kpm_ppnext;
26585648Ssetje
26595648Ssetje kpm_ppnext = kpm_pp + kpnum;
26605648Ssetje seg->kpm_spages = kpm_pp;
26615648Ssetje seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
26625648Ssetje } else {
26635648Ssetje kpm_page_t *kpm_pp = kpm_ppnext;
26645648Ssetje
26655648Ssetje kpm_ppnext = kpm_pp + kpnum;
26665648Ssetje seg->kpm_pages = kpm_pp;
26675648Ssetje seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
26685648Ssetje /* ASSERT no kpm overlaps */
26695648Ssetje ASSERT(
26705648Ssetje memseg_find(base - pmodkpmp(base), NULL) == NULL);
26715648Ssetje ASSERT(memseg_find(
26725648Ssetje roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
26735648Ssetje }
267410106SJason.Beloro@Sun.COM kpm_ppleft -= kpnum;
26755648Ssetje }
26765648Ssetje
26775648Ssetje memseg_list_add(seg);
26785648Ssetje }
26795648Ssetje
26805648Ssetje /*
26815648Ssetje * Add range to free list
26825648Ssetje */
26835648Ssetje void
kphysm_add(uint64_t addr,uint64_t len,int reclaim)26845648Ssetje kphysm_add(uint64_t addr, uint64_t len, int reclaim)
26855648Ssetje {
26865648Ssetje struct page *pp;
26875648Ssetje struct memseg *seg;
26885648Ssetje pfn_t base = btop(addr);
26895648Ssetje pgcnt_t num = btop(len);
26905648Ssetje
26915648Ssetje seg = memseg_find(base, NULL);
26925648Ssetje ASSERT(seg != NULL);
26935648Ssetje pp = seg->pages + (base - seg->pages_base);
26945648Ssetje
26955648Ssetje if (reclaim) {
26965648Ssetje struct page *rpp = pp;
26975648Ssetje struct page *lpp = pp + num;
26980Sstevel@tonic-gate
26990Sstevel@tonic-gate /*
27005648Ssetje * page should be locked on prom_ppages
27015648Ssetje * unhash and unlock it
27020Sstevel@tonic-gate */
27035648Ssetje while (rpp < lpp) {
270411185SSean.McEnroe@Sun.COM ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
270510106SJason.Beloro@Sun.COM ASSERT(PP_ISNORELOC(rpp));
270610106SJason.Beloro@Sun.COM PP_CLRNORELOC(rpp);
27075648Ssetje page_pp_unlock(rpp, 0, 1);
27085648Ssetje page_hashout(rpp, NULL);
27095648Ssetje page_unlock(rpp);
27105648Ssetje rpp++;
27110Sstevel@tonic-gate }
27120Sstevel@tonic-gate }
27130Sstevel@tonic-gate
27145648Ssetje /*
27155648Ssetje * add_physmem() initializes the PSM part of the page
27165648Ssetje * struct by calling the PSM back with add_physmem_cb().
27175648Ssetje * In addition it coalesces pages into larger pages as
27185648Ssetje * it initializes them.
27195648Ssetje */
27205648Ssetje add_physmem(pp, num, base);
27215648Ssetje }
27225648Ssetje
27235648Ssetje /*
27245648Ssetje * kphysm_init() tackles the problem of initializing physical memory.
27255648Ssetje */
27265648Ssetje static void
kphysm_init(void)27275648Ssetje kphysm_init(void)
27285648Ssetje {
27295648Ssetje struct memlist *pmem;
27305648Ssetje
27315648Ssetje ASSERT(page_hash != NULL && page_hashsz != 0);
27325648Ssetje
27335648Ssetje ppnext = pp_base;
27345648Ssetje ppleft = npages;
27355648Ssetje kpm_ppnext = kpm_pp_base;
27365648Ssetje kpm_ppleft = kpm_npages;
27375648Ssetje
27385648Ssetje /*
27395648Ssetje * installed pages not on nopp_memlist go in memseg list
27405648Ssetje */
27415648Ssetje diff_memlists(phys_install, nopp_list, kphysm_memseg);
27425648Ssetje
27435648Ssetje /*
27445648Ssetje * Free the avail list
27455648Ssetje */
274611474SJonathan.Adams@Sun.COM for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
274711474SJonathan.Adams@Sun.COM kphysm_add(pmem->ml_address, pmem->ml_size, 0);
27485648Ssetje
27495648Ssetje /*
27505648Ssetje * Erase pages that aren't available
27515648Ssetje */
27525648Ssetje diff_memlists(phys_install, phys_avail, kphysm_erase);
27535648Ssetje
27540Sstevel@tonic-gate build_pfn_hash();
27550Sstevel@tonic-gate }
27560Sstevel@tonic-gate
27570Sstevel@tonic-gate /*
27580Sstevel@tonic-gate * Kernel VM initialization.
27590Sstevel@tonic-gate * Assumptions about kernel address space ordering:
27600Sstevel@tonic-gate * (1) gap (user space)
27610Sstevel@tonic-gate * (2) kernel text
27620Sstevel@tonic-gate * (3) kernel data/bss
27630Sstevel@tonic-gate * (4) gap
27640Sstevel@tonic-gate * (5) kernel data structures
27650Sstevel@tonic-gate * (6) gap
27660Sstevel@tonic-gate * (7) debugger (optional)
27670Sstevel@tonic-gate * (8) monitor
27680Sstevel@tonic-gate * (9) gap (possibly null)
27690Sstevel@tonic-gate * (10) dvma
27700Sstevel@tonic-gate * (11) devices
27710Sstevel@tonic-gate */
27720Sstevel@tonic-gate static void
kvm_init(void)27730Sstevel@tonic-gate kvm_init(void)
27740Sstevel@tonic-gate {
27750Sstevel@tonic-gate /*
27760Sstevel@tonic-gate * Put the kernel segments in kernel address space.
27770Sstevel@tonic-gate */
27780Sstevel@tonic-gate rw_enter(&kas.a_lock, RW_WRITER);
27790Sstevel@tonic-gate as_avlinit(&kas);
27800Sstevel@tonic-gate
27810Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)KERNELBASE,
27820Sstevel@tonic-gate (size_t)(e_moddata - KERNELBASE), &ktextseg);
27830Sstevel@tonic-gate (void) segkmem_create(&ktextseg);
27840Sstevel@tonic-gate
27850Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
27860Sstevel@tonic-gate (size_t)(MMU_PAGESIZE4M), &ktexthole);
27870Sstevel@tonic-gate (void) segkmem_create(&ktexthole);
27880Sstevel@tonic-gate
27890Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)valloc_base,
27900Sstevel@tonic-gate (size_t)(econtig32 - valloc_base), &kvalloc);
27910Sstevel@tonic-gate (void) segkmem_create(&kvalloc);
27920Sstevel@tonic-gate
27930Sstevel@tonic-gate if (kmem64_base) {
27944528Spaulsan (void) seg_attach(&kas, (caddr_t)kmem64_base,
27954528Spaulsan (size_t)(kmem64_end - kmem64_base), &kmem64);
27964528Spaulsan (void) segkmem_create(&kmem64);
27970Sstevel@tonic-gate }
27980Sstevel@tonic-gate
27990Sstevel@tonic-gate /*
28000Sstevel@tonic-gate * We're about to map out /boot. This is the beginning of the
28010Sstevel@tonic-gate * system resource management transition. We can no longer
28020Sstevel@tonic-gate * call into /boot for I/O or memory allocations.
28030Sstevel@tonic-gate */
28040Sstevel@tonic-gate (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
28050Sstevel@tonic-gate (void) segkmem_create(&kvseg);
28060Sstevel@tonic-gate hblk_alloc_dynamic = 1;
28070Sstevel@tonic-gate
28080Sstevel@tonic-gate /*
28090Sstevel@tonic-gate * we need to preallocate pages for DR operations before enabling large
28100Sstevel@tonic-gate * page kernel heap because of memseg_remap_init() hat_unload() hack.
28110Sstevel@tonic-gate */
28120Sstevel@tonic-gate memseg_remap_init();
28130Sstevel@tonic-gate
28140Sstevel@tonic-gate /* at this point we are ready to use large page heap */
28150Sstevel@tonic-gate segkmem_heap_lp_init();
28160Sstevel@tonic-gate
28170Sstevel@tonic-gate (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
28180Sstevel@tonic-gate &kvseg32);
28190Sstevel@tonic-gate (void) segkmem_create(&kvseg32);
28200Sstevel@tonic-gate
28210Sstevel@tonic-gate /*
28220Sstevel@tonic-gate * Create a segment for the debugger.
28230Sstevel@tonic-gate */
28243446Smrj (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
28250Sstevel@tonic-gate (void) segkmem_create(&kdebugseg);
28260Sstevel@tonic-gate
28270Sstevel@tonic-gate rw_exit(&kas.a_lock);
28280Sstevel@tonic-gate }
28290Sstevel@tonic-gate
28300Sstevel@tonic-gate char obp_tte_str[] =
28310Sstevel@tonic-gate "h# %x constant MMU_PAGESHIFT "
28320Sstevel@tonic-gate "h# %x constant TTE8K "
28330Sstevel@tonic-gate "h# %x constant SFHME_SIZE "
28340Sstevel@tonic-gate "h# %x constant SFHME_TTE "
28350Sstevel@tonic-gate "h# %x constant HMEBLK_TAG "
28360Sstevel@tonic-gate "h# %x constant HMEBLK_NEXT "
28370Sstevel@tonic-gate "h# %x constant HMEBLK_MISC "
28380Sstevel@tonic-gate "h# %x constant HMEBLK_HME1 "
28390Sstevel@tonic-gate "h# %x constant NHMENTS "
28400Sstevel@tonic-gate "h# %x constant HBLK_SZMASK "
28410Sstevel@tonic-gate "h# %x constant HBLK_RANGE_SHIFT "
28420Sstevel@tonic-gate "h# %x constant HMEBP_HBLK "
28438187SPaul.Sandhu@Sun.COM "h# %x constant HMEBLK_ENDPA "
28440Sstevel@tonic-gate "h# %x constant HMEBUCKET_SIZE "
28450Sstevel@tonic-gate "h# %x constant HTAG_SFMMUPSZ "
28464528Spaulsan "h# %x constant HTAG_BSPAGE_SHIFT "
28474528Spaulsan "h# %x constant HTAG_REHASH_SHIFT "
28484528Spaulsan "h# %x constant SFMMU_INVALID_SHMERID "
28490Sstevel@tonic-gate "h# %x constant mmu_hashcnt "
28500Sstevel@tonic-gate "h# %p constant uhme_hash "
28510Sstevel@tonic-gate "h# %p constant khme_hash "
28520Sstevel@tonic-gate "h# %x constant UHMEHASH_SZ "
28530Sstevel@tonic-gate "h# %x constant KHMEHASH_SZ "
28542241Shuah "h# %p constant KCONTEXT "
28550Sstevel@tonic-gate "h# %p constant KHATID "
28560Sstevel@tonic-gate "h# %x constant ASI_MEM "
28570Sstevel@tonic-gate
28580Sstevel@tonic-gate ": PHYS-X@ ( phys -- data ) "
28590Sstevel@tonic-gate " ASI_MEM spacex@ "
28600Sstevel@tonic-gate "; "
28610Sstevel@tonic-gate
28620Sstevel@tonic-gate ": PHYS-W@ ( phys -- data ) "
28630Sstevel@tonic-gate " ASI_MEM spacew@ "
28640Sstevel@tonic-gate "; "
28650Sstevel@tonic-gate
28660Sstevel@tonic-gate ": PHYS-L@ ( phys -- data ) "
28670Sstevel@tonic-gate " ASI_MEM spaceL@ "
28680Sstevel@tonic-gate "; "
28690Sstevel@tonic-gate
28700Sstevel@tonic-gate ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
28710Sstevel@tonic-gate " 3 * MMU_PAGESHIFT + "
28720Sstevel@tonic-gate "; "
28730Sstevel@tonic-gate
28740Sstevel@tonic-gate ": TTE_IS_VALID ( ttep -- flag ) "
28750Sstevel@tonic-gate " PHYS-X@ 0< "
28760Sstevel@tonic-gate "; "
28770Sstevel@tonic-gate
28780Sstevel@tonic-gate ": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
28790Sstevel@tonic-gate " dup TTE8K = if "
28800Sstevel@tonic-gate " drop HBLK_RANGE_SHIFT "
28810Sstevel@tonic-gate " else "
28820Sstevel@tonic-gate " TTE_PAGE_SHIFT "
28830Sstevel@tonic-gate " then "
28840Sstevel@tonic-gate "; "
28850Sstevel@tonic-gate
28860Sstevel@tonic-gate ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
28870Sstevel@tonic-gate " tuck >> swap MMU_PAGESHIFT - << "
28880Sstevel@tonic-gate "; "
28890Sstevel@tonic-gate
28900Sstevel@tonic-gate ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
28910Sstevel@tonic-gate " >> over xor swap ( hash sfmmup ) "
28920Sstevel@tonic-gate " KHATID <> if ( hash ) "
28930Sstevel@tonic-gate " UHMEHASH_SZ and ( bucket ) "
28940Sstevel@tonic-gate " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "
28950Sstevel@tonic-gate " else ( hash ) "
28960Sstevel@tonic-gate " KHMEHASH_SZ and ( bucket ) "
28970Sstevel@tonic-gate " HMEBUCKET_SIZE * khme_hash + ( hmebp ) "
28980Sstevel@tonic-gate " then ( hmebp ) "
28990Sstevel@tonic-gate "; "
29000Sstevel@tonic-gate
29010Sstevel@tonic-gate ": HME_HASH_TABLE_SEARCH "
29020Sstevel@tonic-gate " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "
29030Sstevel@tonic-gate " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
29048187SPaul.Sandhu@Sun.COM " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) "
29050Sstevel@tonic-gate " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "
29060Sstevel@tonic-gate " dup hmeblk_tag + 8 + phys-x@ 2 pick = if "
29070Sstevel@tonic-gate " true ( sfmmup hmeblkp true ) ( r: hblktag ) "
29080Sstevel@tonic-gate " else "
29090Sstevel@tonic-gate " hmeblk_next + phys-x@ false "
29100Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) "
29110Sstevel@tonic-gate " then "
29120Sstevel@tonic-gate " else "
29130Sstevel@tonic-gate " hmeblk_next + phys-x@ false "
29140Sstevel@tonic-gate " ( sfmmup hmeblkp false ) ( r: hblktag ) "
29150Sstevel@tonic-gate " then "
29160Sstevel@tonic-gate " else "
29178187SPaul.Sandhu@Sun.COM " drop 0 true "
29180Sstevel@tonic-gate " then "
29190Sstevel@tonic-gate " until r> drop "
29200Sstevel@tonic-gate "; "
29210Sstevel@tonic-gate
29220Sstevel@tonic-gate ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
29234528Spaulsan " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "
29244528Spaulsan " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )"
29254528Spaulsan " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )"
29264528Spaulsan " SFMMU_INVALID_SHMERID or nip ( hblktag ) "
29270Sstevel@tonic-gate "; "
29280Sstevel@tonic-gate
29290Sstevel@tonic-gate ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
29300Sstevel@tonic-gate " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "
29310Sstevel@tonic-gate " TTE8K = if ( hmeblkp addr ) "
29320Sstevel@tonic-gate " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "
29330Sstevel@tonic-gate " else ( hmeblkp addr ) "
29340Sstevel@tonic-gate " drop 0 ( hmeblkp 0 ) "
29350Sstevel@tonic-gate " then ( hmeblkp hme-index ) "
29360Sstevel@tonic-gate " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "
29370Sstevel@tonic-gate " SFHME_TTE + ( ttep ) "
29380Sstevel@tonic-gate "; "
29390Sstevel@tonic-gate
29400Sstevel@tonic-gate ": unix-tte ( addr cnum -- false | tte-data true ) "
29412241Shuah " KCONTEXT = if ( addr ) "
29422241Shuah " KHATID ( addr khatid ) "
29432241Shuah " else ( addr ) "
29442241Shuah " drop false exit ( false ) "
29452241Shuah " then "
29462241Shuah " ( addr khatid ) "
29470Sstevel@tonic-gate " mmu_hashcnt 1+ 1 do ( addr sfmmup ) "
29480Sstevel@tonic-gate " 2dup swap i HME_HASH_SHIFT "
29490Sstevel@tonic-gate "( addr sfmmup sfmmup addr hmeshift ) "
29500Sstevel@tonic-gate " HME_HASH_FUNCTION ( addr sfmmup hmebp ) "
29510Sstevel@tonic-gate " over i 4 pick "
29520Sstevel@tonic-gate "( addr sfmmup hmebp sfmmup rehash addr ) "
29530Sstevel@tonic-gate " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "
29540Sstevel@tonic-gate " HME_HASH_TABLE_SEARCH "
29550Sstevel@tonic-gate "( addr sfmmup { null | hmeblkp } ) "
29560Sstevel@tonic-gate " ?dup if ( addr sfmmup hmeblkp ) "
29570Sstevel@tonic-gate " nip swap HBLK_TO_TTEP ( ttep ) "
29580Sstevel@tonic-gate " dup TTE_IS_VALID if ( valid-ttep ) "
295910271SJason.Beloro@Sun.COM " PHYS-X@ true ( tte-data true ) "
29600Sstevel@tonic-gate " else ( invalid-tte ) "
29610Sstevel@tonic-gate " drop false ( false ) "
29620Sstevel@tonic-gate " then ( false | tte-data true ) "
29630Sstevel@tonic-gate " unloop exit ( false | tte-data true ) "
29640Sstevel@tonic-gate " then ( addr sfmmup ) "
29650Sstevel@tonic-gate " loop ( addr sfmmup ) "
29660Sstevel@tonic-gate " 2drop false ( false ) "
29670Sstevel@tonic-gate "; "
29680Sstevel@tonic-gate ;
29690Sstevel@tonic-gate
29700Sstevel@tonic-gate void
create_va_to_tte(void)29710Sstevel@tonic-gate create_va_to_tte(void)
29720Sstevel@tonic-gate {
29730Sstevel@tonic-gate char *bp;
29740Sstevel@tonic-gate extern int khmehash_num, uhmehash_num;
29750Sstevel@tonic-gate extern struct hmehash_bucket *khme_hash, *uhme_hash;
29760Sstevel@tonic-gate
29770Sstevel@tonic-gate #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))
29780Sstevel@tonic-gate
29790Sstevel@tonic-gate bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
29800Sstevel@tonic-gate
29810Sstevel@tonic-gate /*
29820Sstevel@tonic-gate * Teach obp how to parse our sw ttes.
29830Sstevel@tonic-gate */
29840Sstevel@tonic-gate (void) sprintf(bp, obp_tte_str,
29850Sstevel@tonic-gate MMU_PAGESHIFT,
29860Sstevel@tonic-gate TTE8K,
29870Sstevel@tonic-gate sizeof (struct sf_hment),
29880Sstevel@tonic-gate OFFSET(struct sf_hment, hme_tte),
29890Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_tag),
29900Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_nextpa),
29910Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_misc),
29920Sstevel@tonic-gate OFFSET(struct hme_blk, hblk_hme),
29930Sstevel@tonic-gate NHMENTS,
29940Sstevel@tonic-gate HBLK_SZMASK,
29950Sstevel@tonic-gate HBLK_RANGE_SHIFT,
29960Sstevel@tonic-gate OFFSET(struct hmehash_bucket, hmeh_nextpa),
29978187SPaul.Sandhu@Sun.COM HMEBLK_ENDPA,
29980Sstevel@tonic-gate sizeof (struct hmehash_bucket),
29990Sstevel@tonic-gate HTAG_SFMMUPSZ,
30004528Spaulsan HTAG_BSPAGE_SHIFT,
30014528Spaulsan HTAG_REHASH_SHIFT,
30024528Spaulsan SFMMU_INVALID_SHMERID,
30030Sstevel@tonic-gate mmu_hashcnt,
30040Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)uhme_hash),
30050Sstevel@tonic-gate (caddr_t)va_to_pa((caddr_t)khme_hash),
30060Sstevel@tonic-gate UHMEHASH_SZ,
30070Sstevel@tonic-gate KHMEHASH_SZ,
30082241Shuah KCONTEXT,
30090Sstevel@tonic-gate KHATID,
301010271SJason.Beloro@Sun.COM ASI_MEM);
30110Sstevel@tonic-gate prom_interpret(bp, 0, 0, 0, 0, 0);
30120Sstevel@tonic-gate
30130Sstevel@tonic-gate kobj_free(bp, MMU_PAGESIZE);
30140Sstevel@tonic-gate }
30150Sstevel@tonic-gate
30160Sstevel@tonic-gate void
install_va_to_tte(void)30170Sstevel@tonic-gate install_va_to_tte(void)
30180Sstevel@tonic-gate {
30190Sstevel@tonic-gate /*
30200Sstevel@tonic-gate * advise prom that he can use unix-tte
30210Sstevel@tonic-gate */
30220Sstevel@tonic-gate prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
30230Sstevel@tonic-gate }
30240Sstevel@tonic-gate
30251253Slq150181 /*
30263928Slq150181 * Here we add "device-type=console" for /os-io node, for currently
30273928Slq150181 * our kernel console output only supports displaying text and
30283928Slq150181 * performing cursor-positioning operations (through kernel framebuffer
30293928Slq150181 * driver) and it doesn't support other functionalities required for a
30303928Slq150181 * standard "display" device as specified in 1275 spec. The main missing
30313928Slq150181 * interface defined by the 1275 spec is "draw-logo".
30323928Slq150181 * also see the comments above prom_stdout_is_framebuffer().
30331253Slq150181 */
30340Sstevel@tonic-gate static char *create_node =
30351492Slq150181 "\" /\" find-device "
30360Sstevel@tonic-gate "new-device "
30370Sstevel@tonic-gate "\" os-io\" device-name "
30383928Slq150181 "\" "OBP_DISPLAY_CONSOLE"\" device-type "
30390Sstevel@tonic-gate ": cb-r/w ( adr,len method$ -- #read/#written ) "
30400Sstevel@tonic-gate " 2>r swap 2 2r> ['] $callback catch if "
30410Sstevel@tonic-gate " 2drop 3drop 0 "
30420Sstevel@tonic-gate " then "
30430Sstevel@tonic-gate "; "
30440Sstevel@tonic-gate ": read ( adr,len -- #read ) "
30450Sstevel@tonic-gate " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "
30460Sstevel@tonic-gate " ( retN ... ret1 N ) "
30470Sstevel@tonic-gate " ?dup if "
30480Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> "
30490Sstevel@tonic-gate " else "
30500Sstevel@tonic-gate " -2 "
30511492Slq150181 " then "
30520Sstevel@tonic-gate "; "
30530Sstevel@tonic-gate ": write ( adr,len -- #written ) "
30540Sstevel@tonic-gate " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "
30550Sstevel@tonic-gate " ( retN ... ret1 N ) "
30560Sstevel@tonic-gate " ?dup if "
30570Sstevel@tonic-gate " swap >r 1- 0 ?do drop loop r> "
30580Sstevel@tonic-gate " else "
30590Sstevel@tonic-gate " 0 "
30600Sstevel@tonic-gate " then "
30610Sstevel@tonic-gate "; "
30620Sstevel@tonic-gate ": poll-tty ( -- ) ; "
30630Sstevel@tonic-gate ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "
30640Sstevel@tonic-gate ": remove-abort ( -- ) ['] poll-tty 0 alarm ; "
30650Sstevel@tonic-gate ": cb-give/take ( $method -- ) "
30660Sstevel@tonic-gate " 0 -rot ['] $callback catch ?dup if "
30670Sstevel@tonic-gate " >r 2drop 2drop r> throw "
30680Sstevel@tonic-gate " else "
30690Sstevel@tonic-gate " 0 ?do drop loop "
30700Sstevel@tonic-gate " then "
30710Sstevel@tonic-gate "; "
30720Sstevel@tonic-gate ": give ( -- ) \" exit-input\" cb-give/take ; "
30730Sstevel@tonic-gate ": take ( -- ) \" enter-input\" cb-give/take ; "
30740Sstevel@tonic-gate ": open ( -- ok? ) true ; "
30750Sstevel@tonic-gate ": close ( -- ) ; "
30760Sstevel@tonic-gate "finish-device "
30770Sstevel@tonic-gate "device-end ";
30780Sstevel@tonic-gate
30790Sstevel@tonic-gate /*
30801253Slq150181 * Create the OBP input/output node (FCode serial driver).
30811253Slq150181 * It is needed for both USB console keyboard and for
30821253Slq150181 * the kernel terminal emulator. It is too early to check for a
30831253Slq150181 * kernel console compatible framebuffer now, so we create this
30841253Slq150181 * so that we're ready if we need to enable kernel terminal emulation.
30850Sstevel@tonic-gate *
30861253Slq150181 * When the USB software takes over the input device at the time
30871253Slq150181 * consconfig runs, OBP's stdin is redirected to this node.
30881253Slq150181 * Whenever the FORTH user interface is used after this switch,
30891253Slq150181 * the node will call back into the kernel for console input.
30901253Slq150181 * If a serial device such as ttya or a UART with a Type 5 keyboard
30911253Slq150181 * attached is used, OBP takes over the serial device when the system
30921253Slq150181 * goes to the debugger after the system is booted. This sharing
30931253Slq150181 * of the relatively simple serial device is difficult but possible.
30941253Slq150181 * Sharing the USB host controller is impossible due its complexity.
30951253Slq150181 *
30961253Slq150181 * Similarly to USB keyboard input redirection, after consconfig_dacf
30971253Slq150181 * configures a kernel console framebuffer as the standard output
30981253Slq150181 * device, OBP's stdout is switched to to vector through the
30991253Slq150181 * /os-io node into the kernel terminal emulator.
31000Sstevel@tonic-gate */
31010Sstevel@tonic-gate static void
startup_create_io_node(void)31021253Slq150181 startup_create_io_node(void)
31030Sstevel@tonic-gate {
31041253Slq150181 prom_interpret(create_node, 0, 0, 0, 0, 0);
31050Sstevel@tonic-gate }
31060Sstevel@tonic-gate
31070Sstevel@tonic-gate
31080Sstevel@tonic-gate static void
do_prom_version_check(void)31090Sstevel@tonic-gate do_prom_version_check(void)
31100Sstevel@tonic-gate {
31110Sstevel@tonic-gate int i;
3112789Sahrens pnode_t node;
31130Sstevel@tonic-gate char buf[64];
31140Sstevel@tonic-gate static char drev[] = "Down-rev firmware detected%s\n"
31154528Spaulsan "\tPlease upgrade to the following minimum version:\n"
31164528Spaulsan "\t\t%s\n";
31170Sstevel@tonic-gate
31180Sstevel@tonic-gate i = prom_version_check(buf, sizeof (buf), &node);
31190Sstevel@tonic-gate
31200Sstevel@tonic-gate if (i == PROM_VER64_OK)
31210Sstevel@tonic-gate return;
31220Sstevel@tonic-gate
31230Sstevel@tonic-gate if (i == PROM_VER64_UPGRADE) {
31240Sstevel@tonic-gate cmn_err(CE_WARN, drev, "", buf);
31250Sstevel@tonic-gate
31260Sstevel@tonic-gate #ifdef DEBUG
31270Sstevel@tonic-gate prom_enter_mon(); /* Type 'go' to continue */
31280Sstevel@tonic-gate cmn_err(CE_WARN, "Booting with down-rev firmware\n");
31290Sstevel@tonic-gate return;
31300Sstevel@tonic-gate #else
31310Sstevel@tonic-gate halt(0);
31320Sstevel@tonic-gate #endif
31330Sstevel@tonic-gate }
31340Sstevel@tonic-gate
31350Sstevel@tonic-gate /*
31360Sstevel@tonic-gate * The other possibility is that this is a server running
31370Sstevel@tonic-gate * good firmware, but down-rev firmware was detected on at
31380Sstevel@tonic-gate * least one other cpu board. We just complain if we see
31390Sstevel@tonic-gate * that.
31400Sstevel@tonic-gate */
31410Sstevel@tonic-gate cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
31420Sstevel@tonic-gate }
31430Sstevel@tonic-gate
31440Sstevel@tonic-gate
31450Sstevel@tonic-gate /*
31460Sstevel@tonic-gate * Must be defined in platform dependent code.
31470Sstevel@tonic-gate */
31480Sstevel@tonic-gate extern caddr_t modtext;
31490Sstevel@tonic-gate extern size_t modtext_sz;
31500Sstevel@tonic-gate extern caddr_t moddata;
31510Sstevel@tonic-gate
31520Sstevel@tonic-gate #define HEAPTEXT_ARENA(addr) \
31530Sstevel@tonic-gate ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
31540Sstevel@tonic-gate (((uintptr_t)(addr) - HEAPTEXT_BASE) / \
31550Sstevel@tonic-gate (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
31560Sstevel@tonic-gate
31570Sstevel@tonic-gate #define HEAPTEXT_OVERSIZED(addr) \
31580Sstevel@tonic-gate ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
31590Sstevel@tonic-gate
31607428SJonathan.Haslam@Sun.COM #define HEAPTEXT_IN_NUCLEUSDATA(addr) \
31617428SJonathan.Haslam@Sun.COM (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
31627428SJonathan.Haslam@Sun.COM ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
31637428SJonathan.Haslam@Sun.COM
31640Sstevel@tonic-gate vmem_t *texthole_source[HEAPTEXT_NARENAS];
31650Sstevel@tonic-gate vmem_t *texthole_arena[HEAPTEXT_NARENAS];
31660Sstevel@tonic-gate kmutex_t texthole_lock;
31670Sstevel@tonic-gate
31680Sstevel@tonic-gate char kern_bootargs[OBP_MAXPATHLEN];
31697656SSherry.Moore@Sun.COM char kern_bootfile[OBP_MAXPATHLEN];
31700Sstevel@tonic-gate
31710Sstevel@tonic-gate void
kobj_vmem_init(vmem_t ** text_arena,vmem_t ** data_arena)31720Sstevel@tonic-gate kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
31730Sstevel@tonic-gate {
31740Sstevel@tonic-gate uintptr_t addr, limit;
31750Sstevel@tonic-gate
31760Sstevel@tonic-gate addr = HEAPTEXT_BASE;
31770Sstevel@tonic-gate limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
31780Sstevel@tonic-gate
31790Sstevel@tonic-gate /*
31800Sstevel@tonic-gate * Before we initialize the text_arena, we want to punch holes in the
31810Sstevel@tonic-gate * underlying heaptext_arena. This guarantees that for any text
31820Sstevel@tonic-gate * address we can find a text hole less than HEAPTEXT_MAPPED away.
31830Sstevel@tonic-gate */
31840Sstevel@tonic-gate for (; addr + HEAPTEXT_UNMAPPED <= limit;
31850Sstevel@tonic-gate addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
31860Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
31870Sstevel@tonic-gate 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
31880Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
31890Sstevel@tonic-gate }
31900Sstevel@tonic-gate
31910Sstevel@tonic-gate /*
31920Sstevel@tonic-gate * Allocate one page at the oversize to break up the text region
31930Sstevel@tonic-gate * from the oversized region.
31940Sstevel@tonic-gate */
31950Sstevel@tonic-gate (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
31960Sstevel@tonic-gate (void *)limit, (void *)(limit + PAGESIZE),
31970Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
31980Sstevel@tonic-gate
31993791Skchow *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
32003791Skchow modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
32010Sstevel@tonic-gate heaptext_arena, 0, VM_SLEEP);
32020Sstevel@tonic-gate *data_arena = vmem_create("module_data", moddata, MODDATA, 1,
32030Sstevel@tonic-gate segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
32040Sstevel@tonic-gate }
32050Sstevel@tonic-gate
32060Sstevel@tonic-gate caddr_t
kobj_text_alloc(vmem_t * arena,size_t size)32070Sstevel@tonic-gate kobj_text_alloc(vmem_t *arena, size_t size)
32080Sstevel@tonic-gate {
32090Sstevel@tonic-gate caddr_t rval, better;
32100Sstevel@tonic-gate
32110Sstevel@tonic-gate /*
32120Sstevel@tonic-gate * First, try a sleeping allocation.
32130Sstevel@tonic-gate */
32140Sstevel@tonic-gate rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
32150Sstevel@tonic-gate
32160Sstevel@tonic-gate if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
32170Sstevel@tonic-gate return (rval);
32180Sstevel@tonic-gate
32190Sstevel@tonic-gate /*
32200Sstevel@tonic-gate * We didn't get the area that we wanted. We're going to try to do an
32210Sstevel@tonic-gate * allocation with explicit constraints.
32220Sstevel@tonic-gate */
32230Sstevel@tonic-gate better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
32240Sstevel@tonic-gate (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
32250Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT);
32260Sstevel@tonic-gate
32270Sstevel@tonic-gate if (better != NULL) {
32280Sstevel@tonic-gate /*
32290Sstevel@tonic-gate * That worked. Free our first attempt and return.
32300Sstevel@tonic-gate */
32310Sstevel@tonic-gate vmem_free(arena, rval, size);
32320Sstevel@tonic-gate return (better);
32330Sstevel@tonic-gate }
32340Sstevel@tonic-gate
32350Sstevel@tonic-gate /*
32360Sstevel@tonic-gate * That didn't work; we'll have to return our first attempt.
32370Sstevel@tonic-gate */
32380Sstevel@tonic-gate return (rval);
32390Sstevel@tonic-gate }
32400Sstevel@tonic-gate
32410Sstevel@tonic-gate caddr_t
kobj_texthole_alloc(caddr_t addr,size_t size)32420Sstevel@tonic-gate kobj_texthole_alloc(caddr_t addr, size_t size)
32430Sstevel@tonic-gate {
32440Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr);
32450Sstevel@tonic-gate char c[30];
32460Sstevel@tonic-gate uintptr_t base;
32470Sstevel@tonic-gate
32487428SJonathan.Haslam@Sun.COM if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
32490Sstevel@tonic-gate /*
32507428SJonathan.Haslam@Sun.COM * If this is an oversized allocation or it is allocated in
32517428SJonathan.Haslam@Sun.COM * the nucleus data page, there is no text hole available for
32527428SJonathan.Haslam@Sun.COM * it; return NULL.
32530Sstevel@tonic-gate */
32540Sstevel@tonic-gate return (NULL);
32550Sstevel@tonic-gate }
32560Sstevel@tonic-gate
32570Sstevel@tonic-gate mutex_enter(&texthole_lock);
32580Sstevel@tonic-gate
32590Sstevel@tonic-gate if (texthole_arena[arena] == NULL) {
32600Sstevel@tonic-gate ASSERT(texthole_source[arena] == NULL);
32610Sstevel@tonic-gate
32620Sstevel@tonic-gate if (arena == 0) {
32630Sstevel@tonic-gate texthole_source[0] = vmem_create("module_text_holesrc",
32640Sstevel@tonic-gate (void *)(KERNELBASE + MMU_PAGESIZE4M),
32650Sstevel@tonic-gate MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
32660Sstevel@tonic-gate 0, VM_SLEEP);
32670Sstevel@tonic-gate } else {
32680Sstevel@tonic-gate base = HEAPTEXT_BASE +
32690Sstevel@tonic-gate (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
32700Sstevel@tonic-gate
32710Sstevel@tonic-gate (void) snprintf(c, sizeof (c),
32720Sstevel@tonic-gate "heaptext_holesrc_%d", arena);
32730Sstevel@tonic-gate
32740Sstevel@tonic-gate texthole_source[arena] = vmem_create(c, (void *)base,
32750Sstevel@tonic-gate HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
32760Sstevel@tonic-gate 0, VM_SLEEP);
32770Sstevel@tonic-gate }
32780Sstevel@tonic-gate
32790Sstevel@tonic-gate (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
32800Sstevel@tonic-gate
32810Sstevel@tonic-gate texthole_arena[arena] = vmem_create(c, NULL, 0,
32820Sstevel@tonic-gate sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
32830Sstevel@tonic-gate texthole_source[arena], 0, VM_SLEEP);
32840Sstevel@tonic-gate }
32850Sstevel@tonic-gate
32860Sstevel@tonic-gate mutex_exit(&texthole_lock);
32870Sstevel@tonic-gate
32880Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL);
32890Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
32900Sstevel@tonic-gate return (vmem_alloc(texthole_arena[arena], size,
32910Sstevel@tonic-gate VM_BESTFIT | VM_NOSLEEP));
32920Sstevel@tonic-gate }
32930Sstevel@tonic-gate
32940Sstevel@tonic-gate void
kobj_texthole_free(caddr_t addr,size_t size)32950Sstevel@tonic-gate kobj_texthole_free(caddr_t addr, size_t size)
32960Sstevel@tonic-gate {
32970Sstevel@tonic-gate int arena = HEAPTEXT_ARENA(addr);
32980Sstevel@tonic-gate
32990Sstevel@tonic-gate ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
33000Sstevel@tonic-gate ASSERT(texthole_arena[arena] != NULL);
33010Sstevel@tonic-gate vmem_free(texthole_arena[arena], addr, size);
33020Sstevel@tonic-gate }
33035974Sjm22469
33045974Sjm22469 void
release_bootstrap(void)33055974Sjm22469 release_bootstrap(void)
33065974Sjm22469 {
33075974Sjm22469 if (&cif_init)
33085974Sjm22469 cif_init();
33095974Sjm22469 }
3310