xref: /netbsd-src/sys/arch/mips/mips/mips_machdep.c (revision 35353221eb5e05b4516841f2cc32e1f9d9e7da44)

/*	$NetBSD: mips_machdep.c,v 1.306 2024/01/06 07:27:35 simonb Exp $	*/

/*
 * Copyright 2002 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Simon Burge for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed for the NetBSD Project by
 *	Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright 2000, 2001
 * Broadcom Corporation. All rights reserved.
 *
 * This software is furnished under license and may be used and copied only
 * in accordance with the following terms and conditions.  Subject to these
 * conditions, you may download, copy, install, use, modify and distribute
 * modified or unmodified copies of this software in source and/or binary
 * form. No title or ownership is transferred hereby.
 *
 * 1) Any source code used, modified or distributed must reproduce and
 *    retain this copyright notice and list of conditions as they appear in
 *    the source file.
 *
 * 2) No right is granted to use any trade name, trademark, or logo of
 *    Broadcom Corporation.  The "Broadcom Corporation" name may not be
 *    used to endorse or promote products derived from this software
 *    without the prior written permission of Broadcom Corporation.
 *
 * 3) THIS SOFTWARE IS PROVIDED "AS-IS" AND ANY EXPRESS OR IMPLIED
 *    WARRANTIES, INCLUDING BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES OF
 *    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
 *    NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM BE LIABLE
 *    FOR ANY DAMAGES WHATSOEVER, AND IN PARTICULAR, BROADCOM SHALL NOT BE
 *    LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 *    BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 *    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 *    OR OTHERWISE), EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*-
 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center and by Chris Demetriou.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright 1996 The Board of Trustees of The Leland Stanford
 * Junior University. All Rights Reserved.
 *
 * Permission to use, copy, modify, and distribute this
 * software and its documentation for any purpose and without
 * fee is hereby granted, provided that the above copyright
 * notice appear in all copies.  Stanford University
 * makes no representations about the suitability of this
 * software for any purpose.  It is provided "as is" without
 * express or implied warranty.
 */

#include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
__KERNEL_RCSID(0, "$NetBSD: mips_machdep.c,v 1.306 2024/01/06 07:27:35 simonb Exp $");

#define __INTR_PRIVATE
#include "opt_cputype.h"
#include "opt_compat_netbsd32.h"
#include "opt_multiprocessor.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/intr.h>
#include <sys/exec.h>
#include <sys/reboot.h>
#include <sys/module.h>
#include <sys/mount.h>			/* fsid_t for syscallargs */
#include <sys/lwp.h>
#include <sys/sysctl.h>
#include <sys/msgbuf.h>
#include <sys/conf.h>
#include <sys/core.h>
#include <sys/device.h>
#include <sys/kcore.h>
#include <sys/kmem.h>
#include <sys/ras.h>
#include <sys/cpu.h>
#include <sys/atomic.h>
#include <sys/ucontext.h>
#include <sys/bitops.h>

#include <mips/kcore.h>

#ifdef COMPAT_NETBSD32
#include <compat/netbsd32/netbsd32.h>
#endif

#include <uvm/uvm.h>
#include <uvm/uvm_physseg.h>

#include <dev/cons.h>
#include <dev/mm.h>

#include <mips/pcb.h>
#include <mips/cache.h>
#include <mips/frame.h>
#include <mips/regnum.h>
#include <mips/mips_opcode.h>

#include <mips/cpu.h>
#include <mips/locore.h>
#include <mips/psl.h>
#include <mips/pte.h>
#include <mips/userret.h>

#ifdef __HAVE_BOOTINFO_H
#include <machine/bootinfo.h>
#endif

#ifdef MIPS64_OCTEON
#include <mips/cavium/octeonvar.h>
#endif

#if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
#include <mips/mipsNN.h>		/* MIPS32/MIPS64 registers */

#define	_MKINSN(a,b,c,d,e) ((uint32_t)(((a) << 26)|((b) << 21)|((c) << 16)|((d) << 11)|(e)))

#ifdef _LP64
#define	_LOAD_V0_L_PRIVATE_A0	_MKINSN(OP_LD, _R_A0, _R_V0, 0, offsetof(lwp_t, l_private))
#define	_MTC0_V0_USERLOCAL	_MKINSN(OP_COP0, OP_DMT, _R_V0, MIPS_COP_0_TLB_CONTEXT, 2)
#else
#define	_LOAD_V0_L_PRIVATE_A0	_MKINSN(OP_LW, _R_A0, _R_V0, 0, offsetof(lwp_t, l_private))
#define	_MTC0_V0_USERLOCAL	_MKINSN(OP_COP0, OP_MT, _R_V0, MIPS_COP_0_TLB_CONTEXT, 2)
#endif
#define	JR_RA			_MKINSN(OP_SPECIAL, _R_RA, 0, 0, OP_JR)

#endif

/* Internal routines. */
int	cpu_dumpsize(void);
u_long	cpu_dump_mempagecnt(void);
int	cpu_dump(void);

#if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
static void mips_watchpoint_init(void);
#endif

#if defined(_LP64) && defined(ENABLE_MIPS_16KB_PAGE)
vaddr_t mips_vm_maxuser_address = MIPS_VM_MAXUSER_ADDRESS;
#endif

#if defined(MIPS3_PLUS)
uint32_t mips3_cp0_tlb_page_mask_probe(void);
uint64_t mips3_cp0_tlb_entry_hi_probe(void);
uint64_t mips3_cp0_tlb_entry_lo_probe(void);

static void mips3_tlb_probe(void);
#endif

#if defined(MIPS1)
static void	mips1_vector_init(const struct splsw *);
extern const struct locoresw mips1_locoresw;
extern const mips_locore_jumpvec_t mips1_locore_vec;
#endif

#if defined(MIPS3)
static void	mips3_vector_init(const struct splsw *);
extern const struct locoresw mips3_locoresw;
extern const mips_locore_jumpvec_t mips3_locore_vec;
#endif

#if defined(MIPS3_LOONGSON2)
static void	loongson2_vector_init(const struct splsw *);
extern const struct locoresw loongson2_locoresw;
extern const mips_locore_jumpvec_t loongson2_locore_vec;
#endif

#if defined(MIPS32)
static void	mips32_vector_init(const struct splsw *);
extern const struct locoresw mips32_locoresw;
extern const mips_locore_jumpvec_t mips32_locore_vec;
#endif

#if defined(MIPS32R2)
static void	mips32r2_vector_init(const struct splsw *);
extern const struct locoresw mips32r2_locoresw;
extern const mips_locore_jumpvec_t mips32r2_locore_vec;
#endif

#if defined(MIPS64)
static void	mips64_vector_init(const struct splsw *);
extern const struct locoresw mips64_locoresw;
extern const mips_locore_jumpvec_t mips64_locore_vec;
#endif

#if defined(MIPS64R2)
extern const struct locoresw mips64r2_locoresw;
extern const mips_locore_jumpvec_t mips64r2_locore_vec;
#endif

#if defined(PARANOIA)
void std_splsw_test(void);
#endif

mips_locore_jumpvec_t mips_locore_jumpvec;

struct locoresw mips_locoresw;

extern const struct splsw std_splsw;
struct splsw mips_splsw;

struct mips_options mips_options = {
	.mips_cpu_id = 0xffffffff,
	.mips_fpu_id = 0xffffffff,
};

void *	msgbufaddr;

/* the following is used by DDB to reset the system */
void	(*cpu_reset_address)(void);

/* the following is used externally (sysctl_hw) */
char	machine[] = MACHINE;		/* from <machine/param.h> */
char	machine_arch[] = MACHINE_ARCH;	/* from <machine/param.h> */

/*
 * Assumptions:
 *  - All MIPS3+ have an r4k-style MMU.  _Many_ assumptions throughout
 *    much of the mips code about this.  Includes overloaded usage of
 *    MIPS3_PLUS.
 *  - All MIPS3+ use the same exception model (cp0 status, cause bits,
 *    etc).  _Many_ assumptions throughout much of the mips code about
 *    this.  Includes overloaded usage of MIPS3_PLUS.
 *  - All MIPS3+ have a count register.  MIPS_HAS_CLOCK in <mips/cpu.h>
 *    will need to be revised if this is false.
 */
#define	MIPS32_FLAGS	CPU_MIPS_R4K_MMU | CPU_MIPS_CAUSE_IV | CPU_MIPS_USE_WAIT
#define	MIPS64_FLAGS	MIPS32_FLAGS	/* same as MIPS32 flags (for now) */

static const struct pridtab cputab[] = {
	{ 0, MIPS_R2000, -1, -1,		CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"MIPS R2000 CPU"	},
	{ 0, MIPS_R3000, MIPS_REV_R2000A, -1,	CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"MIPS R2000A CPU"	},
	{ 0, MIPS_R3000, MIPS_REV_R3000, -1,	CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"MIPS R3000 CPU"	},
	{ 0, MIPS_R3000, MIPS_REV_R3000A, -1,	CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"MIPS R3000A CPU"	},
	{ 0, MIPS_R6000, -1, -1,		CPU_ARCH_MIPS2, 32,
	  MIPS_NOT_SUPP, 0, 0,			"MIPS R6000 CPU"	},

	/*
	 * rev 0x00, 0x22 and 0x30 are R4000, 0x40, 0x50 and 0x60 are R4400.
	 * should we allow ranges and use 0x00 - 0x3f for R4000 and
	 * 0x40 - 0xff for R4400?
	 */
	{ 0, MIPS_R4000, MIPS_REV_R4000_A, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4000 CPU"	},
	{ 0, MIPS_R4000, MIPS_REV_R4000_B, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4000 CPU"	},
	{ 0, MIPS_R4000, MIPS_REV_R4000_C, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4000 CPU"	},
	{ 0, MIPS_R4000, MIPS_REV_R4400_A, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4400 CPU"	},
	{ 0, MIPS_R4000, MIPS_REV_R4400_B, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4400 CPU"	},
	{ 0, MIPS_R4000, MIPS_REV_R4400_C, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R4400 CPU"	},

	{ 0, MIPS_R3LSI, -1, -1,		CPU_ARCH_MIPS1, -1,
	  MIPS_NOT_SUPP, 0, 0,			"LSI Logic R3000 derivative" },
	{ 0, MIPS_R6000A, -1, -1,		CPU_ARCH_MIPS2, 32,
	  MIPS_NOT_SUPP, 0, 0,			"MIPS R6000A CPU"	},
	{ 0, MIPS_R3IDT, -1, -1,		CPU_ARCH_MIPS1, -1,
	  MIPS_NOT_SUPP, 0, 0,			"IDT R3041 or RC36100 CPU" },
	{ 0, MIPS_R4100, -1, -1,		CPU_ARCH_MIPS3, 32,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_NO_LLSC, 0, 0,
						"NEC VR4100 CPU"	},
	{ 0, MIPS_R4200, -1, -1,		CPU_ARCH_MIPS3, -1,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
						"NEC VR4200 CPU"	},
	{ 0, MIPS_R4300, -1, -1,		CPU_ARCH_MIPS3, 32,
	  CPU_MIPS_R4K_MMU, 0, 0,		"NEC VR4300 CPU"	},
	{ 0, MIPS_R4600, -1, -1,		CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"QED R4600 Orion CPU"	},
	{ 0, MIPS_R4700, -1, -1,		CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU, 0, 0,		"QED R4700 Orion CPU"	},

	{ 0, MIPS_R8000, -1, -1,		CPU_ARCH_MIPS4, 384,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
					 "MIPS R8000 Blackbird/TFP CPU" },
	{ 0, MIPS_R10000, -1, -1,		CPU_ARCH_MIPS4, 64,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R10000 CPU"	},
	{ 0, MIPS_R12000, -1, -1,		CPU_ARCH_MIPS4, 64,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R12000 CPU"	},
	{ 0, MIPS_R14000, -1, -1,		CPU_ARCH_MIPS4, 64,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R14000 CPU"	},

	/* XXX
	 * If the Processor Revision ID of the 4650 isn't 0, the following
	 * entry needs to be adjusted.  Can't use a wildcard match because
	 * the TX39 series processors share the same Processor ID value.
	 * Or maybe put TX39 CPUs first if the revid doesn't overlap with
	 * the 4650...
	 */
	{ 0, MIPS_R4650, 0, -1,			CPU_ARCH_MIPS3, -1,
	  MIPS_NOT_SUPP /* no MMU! */, 0, 0,	"QED R4650 CPU"	},
	{ 0, MIPS_TX3900, MIPS_REV_TX3912, -1,	CPU_ARCH_MIPS1, 32,
	  CPU_MIPS_NO_LLSC, 0, 0,		"Toshiba TX3912 CPU"	},
	{ 0, MIPS_TX3900, MIPS_REV_TX3922, -1,	CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"Toshiba TX3922 CPU"	},
	{ 0, MIPS_TX3900, MIPS_REV_TX3927, -1,	CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_LLSC, 0, 0,		"Toshiba TX3927 CPU"	},
	{ 0, MIPS_R5000, -1, -1,		CPU_ARCH_MIPS4, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"MIPS R5000 CPU"	},
	{ 0, MIPS_RM5200, -1, -1,		CPU_ARCH_MIPS4, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_CAUSE_IV | CPU_MIPS_DOUBLE_COUNT |
	  CPU_MIPS_USE_WAIT, 0, 0,		"QED RM5200 CPU"	},

	/* XXX
	 * The rm7000 rev 2.0 can have 64 tlbs, and has 6 extra interrupts.  See
	 *    "Migrating to the RM7000 from other MIPS Microprocessors"
	 * for more details.
	 */
	{ 0, MIPS_RM7000, -1, -1,		CPU_ARCH_MIPS4, 48,
	  MIPS_NOT_SUPP | CPU_MIPS_CAUSE_IV | CPU_MIPS_DOUBLE_COUNT |
	  CPU_MIPS_USE_WAIT, 0, 0,		"QED RM7000 CPU"	},

	/*
	 * IDT RC32300 core is a 32 bit MIPS2 processor with
	 * MIPS3/MIPS4 extensions. It has an R4000-style TLB,
	 * while all registers are 32 bits and any 64 bit
	 * instructions like ld/sd/dmfc0/dmtc0 are not allowed.
	 *
	 * note that the Config register has a non-standard base
	 * for IC and DC (2^9 instead of 2^12).
	 *
	 */
	{ 0, MIPS_RC32300, -1, -1,		CPU_ARCH_MIPS3, 16,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
						"IDT RC32300 CPU"	},
	{ 0, MIPS_RC32364, -1, -1,		CPU_ARCH_MIPS3, 16,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
						"IDT RC32364 CPU"	},
	{ 0, MIPS_RC64470, -1, -1,		CPU_ARCH_MIPSx, -1,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
						"IDT RC64474/RC64475 CPU" },

	{ 0, MIPS_R5400, -1, -1,		CPU_ARCH_MIPSx, -1,
	  MIPS_NOT_SUPP | CPU_MIPS_R4K_MMU, 0, 0,
						"NEC VR5400 CPU"	},
	{ 0, MIPS_R5900, -1, -1,		CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_NO_LLSC | CPU_MIPS_R4K_MMU, 0, 0,
						"Toshiba R5900 CPU"	},

	{ 0, MIPS_TX4900, MIPS_REV_TX4927, -1,	CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"Toshiba TX4927 CPU"	},

	{ 0, MIPS_TX4900, -1, -1,		CPU_ARCH_MIPS3, 48,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT, 0, 0,
						"Toshiba TX4900 CPU"	},

	/*
	 * ICT Loongson2 is a MIPS64 CPU with a few quirks.  For some reason
	 * the virtual aliases present with 4KB pages make the caches misbehave
	 * so we make all accesses uncached.  With 16KB pages, no virtual
	 * aliases are possible so we can use caching.
	 */
#ifdef ENABLE_MIPS_16KB_PAGE
#define	MIPS_LOONGSON2_CCA	0
#else
#define	MIPS_LOONGSON2_CCA	(CPU_MIPS_HAVE_SPECIAL_CCA | \
				(2 << CPU_MIPS_CACHED_CCA_SHIFT))
#endif
	{ 0, MIPS_LOONGSON2, MIPS_REV_LOONGSON2E, -1, CPU_ARCH_MIPS3, 64,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT | CPU_MIPS_LOONGSON2
	  | MIPS_LOONGSON2_CCA, 0, 0, "ICT Loongson 2E CPU"	},
	{ 0, MIPS_LOONGSON2, MIPS_REV_LOONGSON2F, -1, CPU_ARCH_MIPS3, 64,
	  CPU_MIPS_R4K_MMU | CPU_MIPS_DOUBLE_COUNT | CPU_MIPS_LOONGSON2
	  | MIPS_LOONGSON2_CCA, 0, 0, "ICT Loongson 2F CPU"	},

#if 0 /* ID collisions : can we use a CU1 test or similar? */
	{ 0, MIPS_R3SONY, -1, -1,		CPU_ARCH_MIPS1, -1,
	  MIPS_NOT_SUPP, 0, 0,			"SONY R3000 derivative"	},	/* 0x21; crash R4700? */
	{ 0, MIPS_R3NKK, -1, -1,		CPU_ARCH_MIPS1, -1,
	  MIPS_NOT_SUPP, 0, 0,			"NKK R3000 derivative"	},	/* 0x23; crash R5000? */
#endif

	{ MIPS_PRID_CID_MTI, MIPS_4Kc, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "4Kc"		},
	{ MIPS_PRID_CID_MTI, MIPS_4KEc, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "4KEc"		},
	{ MIPS_PRID_CID_MTI, MIPS_4KEc_R2, -1, -1, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "4KEc (Rev 2)"	},
	{ MIPS_PRID_CID_MTI, MIPS_4KSc, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "4KSc"		},
	{ MIPS_PRID_CID_MTI, MIPS_5Kc, -1, -1,	-1, 0,
	  MIPS64_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "5Kc"		},
	{ MIPS_PRID_CID_MTI, MIPS_20Kc, -1, -1,	-1, 0,
	  MIPS64_FLAGS,				0, 0, "20Kc"		},
	{ MIPS_PRID_CID_MTI, MIPS_25Kf, -1, -1,	-1, 0,
	  MIPS64_FLAGS,				0, 0, "25Kf"		},
	{ MIPS_PRID_CID_MTI, MIPS_24K, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG7,
	  0, "24K" },
	{ MIPS_PRID_CID_MTI, MIPS_24KE, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG7,
	  0, "24KE" },
	{ MIPS_PRID_CID_MTI, MIPS_34K, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG7,
	  0, "34K" },
	{ MIPS_PRID_CID_MTI, MIPS_74K, -1, -1,	-1, 0,
	  CPU_MIPS_HAVE_SPECIAL_CCA | (0 << CPU_MIPS_CACHED_CCA_SHIFT) |
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG6 | MIPS_CP0FL_CONFIG7,
	  0, "74K" },
	{ MIPS_PRID_CID_MTI, MIPS_1004K, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG6 | MIPS_CP0FL_CONFIG7,
	  0, "1004K" },
	{ MIPS_PRID_CID_MTI, MIPS_1074K, -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG6 | MIPS_CP0FL_CONFIG7,
	  0, "1074K" },

	{ MIPS_PRID_CID_BROADCOM, MIPS_BCM3302, -1, -1, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_DOUBLE_COUNT, 0, 0, "BCM3302"	},

	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV1, -1, MIPS_AU1000, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1000 (Rev 1 core)"	},
	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV2, -1, MIPS_AU1000, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1000 (Rev 2 core)" 	},

	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV1, -1, MIPS_AU1100, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1100 (Rev 1 core)"	},
	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV2, -1, MIPS_AU1100, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1100 (Rev 2 core)" 	},

	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV1, -1, MIPS_AU1500, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1500 (Rev 1 core)"	},
	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV2, -1, MIPS_AU1500, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1500 (Rev 2 core)" 	},

	{ MIPS_PRID_CID_ALCHEMY, MIPS_AU_REV2, -1, MIPS_AU1550, -1, 0,
	  MIPS32_FLAGS | CPU_MIPS_NO_WAIT | CPU_MIPS_I_D_CACHE_COHERENT, 0, 0,
						"Au1550 (Rev 2 core)" 	},

	/* The SB-1 CPU uses a CCA of 5 - "Cacheable Coherent Shareable" */
	{ MIPS_PRID_CID_SIBYTE, MIPS_SB1, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT |
	  CPU_MIPS_HAVE_SPECIAL_CCA |
	  (CCA_SB_CACHEABLE_COHERENT << CPU_MIPS_CACHED_CCA_SHIFT), 0, 0,
						"SB-1"			},
	{ MIPS_PRID_CID_SIBYTE, MIPS_SB1_11, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT |
	  CPU_MIPS_HAVE_SPECIAL_CCA |
	  (CCA_SB_CACHEABLE_COHERENT << CPU_MIPS_CACHED_CCA_SHIFT), 0, 0,
						"SB-1 (0x11)"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLR732B, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLR|MIPS_CIDFL_RMI_CPUS(8,4)|MIPS_CIDFL_RMI_L2(2MB),
	  "XLR732B"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLR732C, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLR|MIPS_CIDFL_RMI_CPUS(8,4)|MIPS_CIDFL_RMI_L2(2MB),
	  "XLR732C"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS616, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(4,4)|MIPS_CIDFL_RMI_L2(1MB),
	  "XLS616"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS416, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(4,4)|MIPS_CIDFL_RMI_L2(1MB),
	  "XLS416"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS408, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(2,4)|MIPS_CIDFL_RMI_L2(1MB),
	  "XLS408"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS408LITE, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(2,4)|MIPS_CIDFL_RMI_L2(1MB),
	  "XLS408lite"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS404LITE, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(1,4)|MIPS_CIDFL_RMI_L2(512KB),
	  "XLS404lite"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS208, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(2,4)|MIPS_CIDFL_RMI_L2(512KB),
	  "XLS208"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS204, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(1,4)|MIPS_CIDFL_RMI_L2(256KB),
	  "XLS204"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS108, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(2,4)|MIPS_CIDFL_RMI_L2(512KB),
	  "XLS108"		},

	{ MIPS_PRID_CID_RMI, MIPS_XLS104, -1,	-1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR |
	  CPU_MIPS_I_D_CACHE_COHERENT | CPU_MIPS_HAVE_MxCR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EIRR | MIPS_CP0FL_EIMR | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG7,
	  CIDFL_RMI_TYPE_XLS|MIPS_CIDFL_RMI_CPUS(1,4)|MIPS_CIDFL_RMI_L2(256KB),
	  "XLS104"		},

	{ MIPS_PRID_CID_CAVIUM, MIPS_CN31XX, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE | MIPS_CP0FL_CONFIG |
	  MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 | MIPS_CP0FL_CONFIG3,
	  0,
	  "CN31xx"		},

	{ MIPS_PRID_CID_CAVIUM, MIPS_CN30XX, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE | MIPS_CP0FL_CONFIG |
	  MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 | MIPS_CP0FL_CONFIG3,
	  0,
	  "CN30xx"		},

	{ MIPS_PRID_CID_CAVIUM, MIPS_CN50XX, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_EBASE | MIPS_CP0FL_CONFIG |
	  MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 | MIPS_CP0FL_CONFIG3,
	  0,
	  "CN50xx"		},

	{ MIPS_PRID_CID_CAVIUM, MIPS_CN68XX, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR,
	  MIPS_CP0FL_USE |
	  MIPS_CP0FL_CONFIG  | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG4,
	  0,
	  "CN68xx"		},

	{ MIPS_PRID_CID_CAVIUM, MIPS_CN70XX, -1, -1, -1, 0,
	  MIPS64_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_NO_LLADDR,
	  MIPS_CP0FL_USE | MIPS_CP0FL_EBASE |
	  MIPS_CP0FL_CONFIG  | MIPS_CP0FL_CONFIG1 | MIPS_CP0FL_CONFIG2 |
	  MIPS_CP0FL_CONFIG3 | MIPS_CP0FL_CONFIG4 | MIPS_CP0FL_CONFIG5 |
	  MIPS_CP0FL_CONFIG6 | MIPS_CP0FL_CONFIG7,
	  0,
	  "CN70xx/CN71xx"	},

	/* Microsoft Research' extensible MIPS */
	{ MIPS_PRID_CID_MICROSOFT, MIPS_eMIPS, 1, -1, CPU_ARCH_MIPS1, 64,
	  CPU_MIPS_NO_WAIT, 0, 0,		"eMIPS CPU"		},

	/* Ingenic XBurst */
	{ MIPS_PRID_CID_INGENIC, MIPS_XBURST,  -1, -1,	-1, 0,
	  MIPS32_FLAGS | CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_DOUBLE_COUNT,
	  0, 0, "XBurst"		},

	{ 0, 0, 0,				0, 0, 0,
	  0, 0, 0,				NULL			}
};

static const struct pridtab fputab[] = {
    { 0, MIPS_SOFT,  -1, 0, 0, 0, 0, 0, 0, "software emulated floating point" },
    { 0, MIPS_R2360, -1, 0, 0, 0, 0, 0, 0, "MIPS R2360 Floating Point Board" },
    { 0, MIPS_R2010, -1, 0, 0, 0, 0, 0, 0, "MIPS R2010 FPC" },
    { 0, MIPS_R3010, -1, 0, 0, 0, 0, 0, 0, "MIPS R3010 FPC" },
    { 0, MIPS_R6010, -1, 0, 0, 0, 0, 0, 0, "MIPS R6010 FPC" },
    { 0, MIPS_R4010, -1, 0, 0, 0, 0, 0, 0, "MIPS R4010 FPC" },
};

/*
 * Company ID's are not sparse (yet), this array is indexed directly
 * by pridtab->cpu_cid.
 */
static const char * const cidnames[] = {
	"Prehistoric",
	"MIPS",		/* or "MIPS Technologies, Inc.	*/
	"Broadcom",	/* or "Broadcom Corp."		*/
	"Alchemy",	/* or "Alchemy Semiconductor"	*/
	"SiByte",	/* or "Broadcom Corp. (SiByte)"	*/
	"SandCraft",
	"Phillips",
	"Toshiba or Microsoft",
	"LSI",
	"(unannounced)",
	"(unannounced)",
	"Lexra",
	"RMI",
	"Cavium",
};
#define	ncidnames __arraycount(cidnames)

#if defined(MIPS1)
/*
 * MIPS-I locore function vector
 */

static void
mips1_vector_init(const struct splsw *splsw)
{
	extern char mips1_utlb_miss[], mips1_utlb_miss_end[];
	extern char mips1_exception[], mips1_exception_end[];

	/*
	 * Copy down exception vector code.
	 */
	if (mips1_utlb_miss_end - mips1_utlb_miss > 0x80)
		panic("startup: UTLB vector code too large");
	if (mips1_exception_end - mips1_exception > 0x80)
		panic("startup: general exception vector code too large");
	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, mips1_utlb_miss,
		mips1_exception_end - mips1_utlb_miss);

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = mips1_locore_vec;

	/*
	 * Clear out the I and D caches.
	 */
	mips_icache_sync_all();
	mips_dcache_wbinv_all();
}
#endif /* MIPS1 */

#if defined(MIPS3)
static void
mips3_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address and end */
	extern char mips3_exception[], mips3_exception_end[];

	/* TLB miss handler address and end */
	extern char mips3_tlb_miss[];
	extern char mips3_xtlb_miss[];

	/* Cache error handler */
	extern char mips3_cache[];
	/*
	 * Copy down exception vector code.
	 */

	if (mips3_xtlb_miss - mips3_tlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (mips3_cache - mips3_xtlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "XTLB");
	if (mips3_exception - mips3_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (mips3_exception_end - mips3_exception > 0x80)
		panic("startup: %s vector code too large",
		    "General exception");

	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, mips3_tlb_miss,
	      mips3_exception_end - mips3_tlb_miss);

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = mips3_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);
}
#endif /* MIPS3 */

#if defined(MIPS3_LOONGSON2)
static void
loongson2_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address and end */
	extern char loongson2_exception[], loongson2_exception_end[];

	/* TLB miss handler address and end */
	extern char loongson2_tlb_miss[];
	extern char loongson2_xtlb_miss[];

	/* Cache error handler */
	extern char loongson2_cache[];

	/*
	 * Copy down exception vector code.
	 */

	if (loongson2_xtlb_miss - loongson2_tlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (loongson2_cache - loongson2_xtlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "XTLB");
	if (loongson2_exception - loongson2_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (loongson2_exception_end - loongson2_exception > 0x80)
		panic("startup: %s vector code too large",
		    "General exception");

	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, loongson2_tlb_miss,
	      loongson2_exception_end - loongson2_tlb_miss);

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = loongson2_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);
}
#endif /* MIPS3_LOONGSON2 */

#if defined(MIPS32)
static void
mips32_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address */
	extern char mips32_exception[];

	/* TLB miss handler addresses */
	extern char mips32_tlb_miss[];

	/* Cache error handler */
	extern char mips32_cache[];

	/* MIPS32 interrupt exception handler */
	extern char mips32_intr[], mips32_intr_end[];

	/*
	 * Copy down exception vector code.
	 */

	if (mips32_cache - mips32_tlb_miss != 0x100)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (mips32_exception - mips32_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (mips32_intr - mips32_exception != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "General exception");
	if (mips32_intr_end - mips32_intr > 0x80)
		panic("startup: %s vector code too large",
		    "interrupt exception");

	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, mips32_tlb_miss,
	      mips32_intr_end - mips32_tlb_miss);

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = mips32_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);

	mips_watchpoint_init();
}
#endif /* MIPS32 */

#if defined(MIPS32R2)
static void
mips32r2_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address */
	extern char mips32r2_exception[];

	/* TLB miss handler addresses */
	extern char mips32r2_tlb_miss[];

	/* Cache error handler */
	extern char mips32r2_cache[];

	/* MIPS32 interrupt exception handler */
	extern char mips32r2_intr[], mips32r2_intr_end[];

	/*
	 * Copy down exception vector code.
	 */
	if (mips32r2_cache - mips32r2_tlb_miss != 0x100)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (mips32r2_exception - mips32r2_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (mips32r2_intr - mips32r2_exception != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "General exception");
	if (mips32r2_intr_end - mips32r2_intr > 0x80)
		panic("startup: %s vector code too large",
		    "interrupt exception");

	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, mips32r2_tlb_miss,
	      mips32r2_intr_end - mips32r2_tlb_miss);

	/*
	 * Let's see if this cpu has USERLOCAL or DSP V2 ASE...
	 */
	if (mipsNN_cp0_config2_read() & MIPSNN_CFG2_M) {
		const uint32_t cfg3 = mipsNN_cp0_config3_read();
		if (cfg3 & MIPSNN_CFG3_ULRI) {
			mips_options.mips_cpu_flags |= CPU_MIPS_HAVE_USERLOCAL;
		}
		if (cfg3 & MIPSNN_CFG3_DSP2P) {
			mips_options.mips_cpu_flags |= CPU_MIPS_HAVE_DSP;
		}
	}

	/*
	 * If this CPU doesn't have a COP0 USERLOCAL register, at the end
	 * of cpu_switch resume overwrite the instructions which update it.
	 */
	if (!MIPS_HAS_USERLOCAL) {
		extern uint32_t mips32r2_cpu_switch_resume[];
		for (uint32_t *insnp = mips32r2_cpu_switch_resume;; insnp++) {
			KASSERT(insnp[0] != JR_RA);
			if (insnp[0] == _LOAD_V0_L_PRIVATE_A0
			    && insnp[1] == _MTC0_V0_USERLOCAL) {
				insnp[0] = JR_RA;
				insnp[1] = 0;		/* NOP */
				break;
			}
		}
	}

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = mips32r2_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);

	mips_watchpoint_init();
}
#endif /* MIPS32R2 */

#if defined(MIPS64)
static void
mips64_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address */
	extern char mips64_exception[];

	/* TLB miss handler addresses */
	extern char mips64_tlb_miss[];
	extern char mips64_xtlb_miss[];

	/* Cache error handler */
	extern char mips64_cache[];

	/* MIPS64 interrupt exception handler */
	extern char mips64_intr[], mips64_intr_end[];

	/*
	 * Copy down exception vector code.
	 */

	if (mips64_xtlb_miss - mips64_tlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (mips64_cache - mips64_xtlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "XTLB");
	if (mips64_exception - mips64_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (mips64_intr - mips64_exception != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "General exception");
	if (mips64_intr_end - mips64_intr > 0x80)
		panic("startup: %s vector code too large",
		    "interrupt exception");

	memcpy((void *)MIPS_UTLB_MISS_EXC_VEC, mips64_tlb_miss,
	      mips64_intr_end - mips64_tlb_miss);

	/*
	 * Copy locore-function vector.
	 */
	mips_locore_jumpvec = mips64_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);

	mips_watchpoint_init();
}
#endif /* MIPS64 */

#if defined(MIPS64R2)
void
mips64r2_vector_init(const struct splsw *splsw)
{
	/* r4000 exception handler address */
	extern char mips64r2_exception[];

	/* TLB miss handler addresses */
	extern char mips64r2_tlb_miss[];
	extern char mips64r2_xtlb_miss[];

	/* Cache error handler */
	extern char mips64r2_cache[];

	/* MIPS64 interrupt exception handler */
	extern char mips64r2_intr[], mips64r2_intr_end[];

	/*
	 * Copy down exception vector code.
	 */

	if (mips64r2_xtlb_miss - mips64r2_tlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "UTLB");
	if (mips64r2_cache - mips64r2_xtlb_miss != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "XTLB");
	if (mips64r2_exception - mips64r2_cache != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "Cache error");
	if (mips64r2_intr - mips64r2_exception != 0x80)
		panic("startup: %s vector code not 128 bytes in length",
		    "General exception");
	if (mips64r2_intr_end - mips64r2_intr > 0x80)
		panic("startup: %s vector code too large",
		    "interrupt exception");

	const intptr_t ebase = (intptr_t)mipsNN_cp0_ebase_read();
	const int cpunum = ebase & MIPS_EBASE_CPUNUM;

	// This may need to be on CPUs other CPU0 so use EBASE to fetch
	// the appropriate address for exception code.  EBASE also contains
	// the cpunum so remove that.
	memcpy((void *)(intptr_t)(ebase & ~MIPS_EBASE_CPUNUM), mips64r2_tlb_miss,
	      mips64r2_intr_end - mips64r2_tlb_miss);

	/*
	 * Let's see if this cpu has USERLOCAL or DSP V2 ASE...
	 */
	if (mipsNN_cp0_config2_read() & MIPSNN_CFG2_M) {
		const uint32_t cfg3 = mipsNN_cp0_config3_read();
		if (cfg3 & MIPSNN_CFG3_ULRI) {
			mips_options.mips_cpu_flags |= CPU_MIPS_HAVE_USERLOCAL;
		}
		if (cfg3 & MIPSNN_CFG3_DSP2P) {
			mips_options.mips_cpu_flags |= CPU_MIPS_HAVE_DSP;
		}
	}

	/*
	 * If this CPU doesn't have a COP0 USERLOCAL register, at the end
	 * of cpu_switch resume overwrite the instructions which update it.
	 */
	if (!MIPS_HAS_USERLOCAL && cpunum == 0) {
		extern uint32_t mips64r2_cpu_switch_resume[];
		for (uint32_t *insnp = mips64r2_cpu_switch_resume;; insnp++) {
			KASSERT(insnp[0] != JR_RA);
			if (insnp[0] == _LOAD_V0_L_PRIVATE_A0
			    && insnp[1] == _MTC0_V0_USERLOCAL) {
				insnp[0] = JR_RA;
				insnp[1] = 0;		/* NOP */
				break;
			}
		}
	}

	/*
	 * Copy locore-function vector.
	 */
	if (cpunum == 0)
		mips_locore_jumpvec = mips64r2_locore_vec;

	mips_icache_sync_all();
	mips_dcache_wbinv_all();

	/* Clear BEV in SR so we start handling our own exceptions */
	mips_cp0_status_write(mips_cp0_status_read() & ~MIPS_SR_BEV);

	mips_watchpoint_init();
}
#endif /* MIPS64R2 */

/*
 * Do all the stuff that locore normally does before calling main(),
 * that is common to all mips-CPU NetBSD ports.
 *
 * The principal purpose of this function is to examine the
 * variable cpu_id, into which the kernel locore start code
 * writes the CPU ID register, and to then copy appropriate
 * code into the CPU exception-vector entries and the jump tables
 * used to hide the differences in cache and TLB handling in
 * different MIPS CPUs.
 *
 * This should be the very first thing called by each port's
 * init_main() function.
 */

/*
 * Initialize the hardware exception vectors, and the jump table used to
 * call locore cache and TLB management functions, based on the kind
 * of CPU the kernel is running on.
 */
void
mips_vector_init(const struct splsw *splsw, bool multicpu_p)
{
	struct mips_options * const opts = &mips_options;
	const struct pridtab *ct;
	const mips_prid_t cpu_id = opts->mips_cpu_id;

	for (ct = cputab; ct->cpu_name != NULL; ct++) {
		if (MIPS_PRID_CID(cpu_id) != ct->cpu_cid ||
		    MIPS_PRID_IMPL(cpu_id) != ct->cpu_pid)
			continue;
		if (ct->cpu_rev >= 0 &&
		    MIPS_PRID_REV(cpu_id) != ct->cpu_rev)
			continue;
		if (ct->cpu_copts >= 0 &&
		    MIPS_PRID_COPTS(cpu_id) != ct->cpu_copts)
			continue;

		opts->mips_cpu = ct;
		opts->mips_cpu_arch = ct->cpu_isa;
		opts->mips_num_tlb_entries = ct->cpu_ntlb;
		break;
	}

	if (opts->mips_cpu == NULL)
		panic("CPU type (0x%x) not supported", cpu_id);

#if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
	if (MIPS_PRID_CID(cpu_id) != 0) {
		/* MIPS32/MIPS64, use coprocessor 0 config registers */
		uint32_t cfg, cfg1, cfg4;

		cfg = mips3_cp0_config_read();
		cfg1 = mipsNN_cp0_config1_read();
		if (opts->mips_cpu->cpu_cp0flags & MIPS_CP0FL_CONFIG4)
			cfg4 = mipsNN_cp0_config4_read();
		else
			cfg4 = 0;

		/* pick CPU type */
		switch (MIPSNN_GET(CFG_AT, cfg)) {
		case MIPSNN_CFG_AT_MIPS32:
			opts->mips_cpu_arch = CPU_ARCH_MIPS32;
			break;
		case MIPSNN_CFG_AT_MIPS64:
			opts->mips_cpu_arch = CPU_ARCH_MIPS64;
			break;
		case MIPSNN_CFG_AT_MIPS64S:
		default:
			panic("MIPS32/64 architecture type %d not supported",
			    MIPSNN_GET(CFG_AT, cfg));
		}

		switch (MIPSNN_GET(CFG_AR, cfg)) {
		case MIPSNN_CFG_AR_REV1:
			break;
		case MIPSNN_CFG_AR_REV2:
			switch (opts->mips_cpu_arch) {
			case CPU_ARCH_MIPS32:
				opts->mips_cpu_arch = CPU_ARCH_MIPS32R2;
				break;
			case CPU_ARCH_MIPS64:
				opts->mips_cpu_arch = CPU_ARCH_MIPS64R2;
				break;
			default:
				printf("WARNING: MIPS32/64 arch %d revision %d "
				    "unknown!\n", opts->mips_cpu_arch,
				    MIPSNN_GET(CFG_AR, cfg));
				break;
			}
			break;
		default:
			printf("WARNING: MIPS32/64 arch revision %d "
			    "unknown!\n", MIPSNN_GET(CFG_AR, cfg));
			break;
		}

		/* figure out MMU type (and number of TLB entries) */
		switch (MIPSNN_GET(CFG_MT, cfg)) {
		case MIPSNN_CFG_MT_TLB:
			/*
			 * Config1[MMUSize-1] defines the number of TLB
			 * entries minus 1, allowing up to 64 TLBs to be
			 * defined.  For MIPS32R2 and MIPS64R2 and later
			 * if the Config4[MMUExtDef] field is 1 then the
			 * Config4[MMUSizeExt] field is an extension of
			 * Config1[MMUSize-1] field.
			 */
			opts->mips_num_tlb_entries = MIPSNN_CFG1_MS(cfg1);
			if (__SHIFTOUT(cfg4, MIPSNN_CFG4_MMU_EXT_DEF) ==
			    MIPSNN_CFG4_MMU_EXT_DEF_MMU) {
				opts->mips_num_tlb_entries +=
				__SHIFTOUT(cfg4, MIPSNN_CFG4_MMU_SIZE_EXT) <<
				    popcount(MIPSNN_CFG1_MS_MASK);
			}
			break;
		case MIPSNN_CFG_MT_NONE:
		case MIPSNN_CFG_MT_BAT:
		case MIPSNN_CFG_MT_FIXED:
		default:
			panic("MIPS32/64 MMU type %d not supported",
			    MIPSNN_GET(CFG_MT, cfg));
		}
	}
#endif /* (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */

	if (opts->mips_cpu_arch < 1)
		panic("Unknown CPU ISA for CPU type 0x%x", cpu_id);
	if (opts->mips_num_tlb_entries < 1)
		panic("Unknown number of TLBs for CPU type 0x%x", cpu_id);

	/*
	 * Check CPU-specific flags.
	 */
	opts->mips_cpu_flags = opts->mips_cpu->cpu_flags;
	opts->mips_has_r4k_mmu = (opts->mips_cpu_flags & CPU_MIPS_R4K_MMU) != 0;
	opts->mips_has_llsc = (opts->mips_cpu_flags & CPU_MIPS_NO_LLSC) == 0;
#if defined(MIPS3_4100)
	if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4100)
		opts->mips3_pg_shift = MIPS3_4100_PG_SHIFT;
	else
#endif
		opts->mips3_pg_shift = MIPS3_DEFAULT_PG_SHIFT;

	opts->mips3_cca_devmem = CCA_UNCACHED;
	if (opts->mips_cpu_flags & CPU_MIPS_HAVE_SPECIAL_CCA) {
		uint32_t cca;

		cca = (opts->mips_cpu_flags & CPU_MIPS_CACHED_CCA_MASK) >>
		    CPU_MIPS_CACHED_CCA_SHIFT;
		opts->mips3_pg_cached = MIPS3_CCA_TO_PG(cca);
#ifndef __mips_o32
		opts->mips3_xkphys_cached = MIPS_PHYS_TO_XKPHYS(cca, 0);
#endif
	} else {
		opts->mips3_pg_cached = MIPS3_DEFAULT_PG_CACHED;
#ifndef __mips_o32
		opts->mips3_xkphys_cached = MIPS3_DEFAULT_XKPHYS_CACHED;
#endif
	}

#ifdef __HAVE_MIPS_MACHDEP_CACHE_CONFIG
	mips_machdep_cache_config();
#endif

	/*
	 * if 'splsw' is NULL, use standard SPL with COP0 status/cause
	 * otherwise use chip-specific splsw
	 */
	if (splsw == NULL) {
		mips_splsw = std_splsw;
#ifdef PARANOIA
		std_splsw_test();	/* only works with std_splsw */
#endif
	} else {
		mips_splsw = *splsw;
	}

	/*
	 * Determine cache configuration and initialize our cache
	 * frobbing routine function pointers.
	 */
	mips_config_cache();

	/*
	 * We default to RAS atomic ops since they are the lowest overhead.
	 */
#ifdef MULTIPROCESSOR
	if (multicpu_p) {
		/*
		 * If we could have multiple CPUs active,
		 * use the ll/sc variants.
		 */
		mips_locore_atomicvec = mips_llsc_locore_atomicvec;
	}
#endif
	/*
	 * Now initialize our ISA-dependent function vector.
	 */
	switch (opts->mips_cpu_arch) {
#if defined(MIPS1)
	case CPU_ARCH_MIPS1:
		(*mips1_locore_vec.ljv_tlb_invalidate_all)();
		mips1_vector_init(splsw);
		mips_locoresw = mips1_locoresw;
		break;
#endif
#if defined(MIPS3)
	case CPU_ARCH_MIPS3:
	case CPU_ARCH_MIPS4:
		mips3_tlb_probe();
#if defined(MIPS3_4100)
		if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4100)
			mips3_cp0_pg_mask_write(MIPS4100_PG_SIZE_TO_MASK(PAGE_SIZE));
		else
#endif
		mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
		mips3_cp0_wired_write(0);
#if defined(MIPS3_LOONGSON2)
		if (opts->mips_cpu_flags & CPU_MIPS_LOONGSON2) {
			(*loongson2_locore_vec.ljv_tlb_invalidate_all)();
			mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
			loongson2_vector_init(splsw);
			mips_locoresw = loongson2_locoresw;
			opts->mips3_cca_devmem = CCA_ACCEL;
			break;
		}
#endif /* MIPS3_LOONGSON2 */
		(*mips3_locore_vec.ljv_tlb_invalidate_all)();
		mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
		mips3_vector_init(splsw);
		mips_locoresw = mips3_locoresw;
		break;

#endif /* MIPS3 */
#if defined(MIPS32)
	case CPU_ARCH_MIPS32:
		mips3_tlb_probe();
		mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
		mips3_cp0_wired_write(0);
		(*mips32_locore_vec.ljv_tlb_invalidate_all)();
		mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
		mips32_vector_init(splsw);
		mips_locoresw = mips32_locoresw;
		break;
#endif
#if defined(MIPS32R2)
	case CPU_ARCH_MIPS32R2:
		mips3_tlb_probe();
		mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
		mips3_cp0_wired_write(0);
		(*mips32r2_locore_vec.ljv_tlb_invalidate_all)();
		mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
		mips32r2_vector_init(splsw);
		mips_locoresw = mips32r2_locoresw;
		break;
#endif
#if defined(MIPS64)
	case CPU_ARCH_MIPS64: {
		mips3_tlb_probe();
		mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
		mips3_cp0_wired_write(0);
		(*mips64_locore_vec.ljv_tlb_invalidate_all)();
		mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
		mips64_vector_init(splsw);
		mips_locoresw = mips64_locoresw;
		break;
	}
#endif
#if defined(MIPS64R2)
	case CPU_ARCH_MIPS64R2: {
		mips3_tlb_probe();
		mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE));
		mips3_cp0_wired_write(0);
		(*mips64r2_locore_vec.ljv_tlb_invalidate_all)();
		mips3_cp0_wired_write(pmap_tlb0_info.ti_wired);
		mips64r2_vector_init(splsw);
		mips_locoresw = mips64r2_locoresw;
		break;
	}
#endif
	default:
		printf("cpu_arch 0x%x: not supported\n", opts->mips_cpu_arch);
		cpu_reboot(RB_HALT, NULL);
	}

	/*
	 * Now that the splsw and locoresw have been filled in, fixup the
	 * jumps to any stubs to actually jump to the real routines.
	 */
	extern uint32_t _ftext[];
	extern uint32_t _etext[];
	mips_fixup_stubs(_ftext, _etext);

#if (MIPS3 + MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
	/*
	 * Install power-saving idle routines.
	 */
	if ((opts->mips_cpu_flags & CPU_MIPS_USE_WAIT) &&
	    !(opts->mips_cpu_flags & CPU_MIPS_NO_WAIT))
		mips_locoresw.lsw_cpu_idle = mips_wait_idle;
#endif /* (MIPS3 + MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */
}

void
mips_set_wbflush(void (*flush_fn)(void))
{
	mips_locoresw.lsw_wbflush = flush_fn;
	(*flush_fn)();
}

#if defined(MIPS3_PLUS)
static void
mips3_tlb_probe(void)
{
	struct mips_options * const opts = &mips_options;
	opts->mips3_tlb_pg_mask = mips3_cp0_tlb_page_mask_probe();
	if (CPUIS64BITS) {
		opts->mips3_tlb_vpn_mask = mips3_cp0_tlb_entry_hi_probe();
		opts->mips3_tlb_vpn_mask |= PAGE_MASK;
		opts->mips3_tlb_vpn_mask <<= 2;
		opts->mips3_tlb_vpn_mask >>= 2;
		opts->mips3_tlb_pfn_mask = mips3_cp0_tlb_entry_lo_probe();
#if defined(_LP64) && defined(ENABLE_MIPS_16KB_PAGE)
		/*
		 * 16KB pages could cause our page table being able to address
		 * a larger address space than the actual chip supports.  So
		 * we need to limit the address space to what it can really
		 * address.
		 */
		if (mips_vm_maxuser_address > opts->mips3_tlb_vpn_mask + 1)
			mips_vm_maxuser_address = opts->mips3_tlb_vpn_mask + 1;
#endif
	}
}
#endif

static const char *
wayname(int ways)
{
	static char buf[sizeof("xxx-way set-associative")];

#ifdef DIAGNOSTIC
	if (ways > 999)
		panic("mips cache - too many ways (%d)", ways);
#endif

	switch (ways) {
	case 0:
		return "fully set-associative";
	case 1:
		return "direct-mapped";
	default:
		snprintf(buf, sizeof(buf), "%d-way set-associative", ways);
		return buf;
	}
}

/*
 * Identify product revision IDs of CPU and FPU.
 */
void
cpu_identify(device_t dev)
{
	const struct mips_options * const opts = &mips_options;
	const struct mips_cache_info * const mci = &mips_cache_info;
	const mips_prid_t cpu_id = opts->mips_cpu_id;
	const mips_prid_t fpu_id = opts->mips_fpu_id;
	static const char * const wtnames[] = {
		"write-back",
		"write-through",
	};
	const char *cpuname, *fpuname;
	int i;

	cpuname = opts->mips_cpu->cpu_name;
#ifdef MIPS64_OCTEON
	if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_CAVIUM) {
		cpuname = octeon_cpu_model(cpu_id);
	}
#endif

	fpuname = NULL;
	for (i = 0; i < sizeof(fputab)/sizeof(fputab[0]); i++) {
		if (MIPS_PRID_CID(fpu_id) == fputab[i].cpu_cid &&
		    MIPS_PRID_IMPL(fpu_id) == fputab[i].cpu_pid) {
			fpuname = fputab[i].cpu_name;
			break;
		}
	}
	if (fpuname == NULL && MIPS_PRID_IMPL(fpu_id) == MIPS_PRID_IMPL(cpu_id))
		fpuname = "built-in FPU";
	if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4700)	/* FPU PRid is 0x20 */
		fpuname = "built-in FPU";
	if (MIPS_PRID_IMPL(cpu_id) == MIPS_RC64470)	/* FPU PRid is 0x21 */
		fpuname = "built-in FPU";
#ifdef MIPSNN
	if (CPUISMIPSNN) {
		uint32_t cfg1;

		switch (MIPS_PRID_CID(cpu_id)) {
		/*
		 * CPUs from the following companies have a built-in
		 * FPU if Config1[FP] is set.
		 */
		case MIPS_PRID_CID_SIBYTE:
		case MIPS_PRID_CID_CAVIUM:
			cfg1 = mipsNN_cp0_config1_read();
			if (cfg1 & MIPSNN_CFG1_FP)
				fpuname = "built-in FPU";
			break;
		}
	}
#endif

	if (opts->mips_cpu->cpu_cid != 0) {
		if (opts->mips_cpu->cpu_cid <= ncidnames)
			aprint_normal("%s ", cidnames[opts->mips_cpu->cpu_cid]);
		else if (opts->mips_cpu->cpu_cid == MIPS_PRID_CID_INGENIC) {
			aprint_normal("Ingenic ");
		} else {
			aprint_normal("Unknown Company ID - 0x%x",
			    opts->mips_cpu->cpu_cid);
			aprint_normal_dev(dev, "");
		}
	}
	if (cpuname != NULL)
		aprint_normal("%s (0x%x)", cpuname, cpu_id);
	else
		aprint_normal("unknown CPU type (0x%x)", cpu_id);
	if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_PREHISTORIC)
		aprint_normal(" Rev. %d.%d", MIPS_PRID_REV_MAJ(cpu_id),
		    MIPS_PRID_REV_MIN(cpu_id));
	else
		aprint_normal(" Rev. %d", MIPS_PRID_REV(cpu_id));

	if (fpuname != NULL)
		aprint_normal(" with %s", fpuname);
	else
		aprint_normal(" with unknown FPC type (0x%x)", fpu_id);
	if (opts->mips_fpu_id != 0) {
		if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_PREHISTORIC)
			aprint_normal(" Rev. %d.%d", MIPS_PRID_REV_MAJ(fpu_id),
			    MIPS_PRID_REV_MIN(fpu_id));
		else
			aprint_normal(" Rev. %d", MIPS_PRID_REV(fpu_id));
	}
	if (opts->mips_cpu_flags & MIPS_HAS_DSP) {
		aprint_normal(" and DSPv2");
	}
	aprint_normal("\n");

	if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_PREHISTORIC &&
	    MIPS_PRID_RSVD(cpu_id) != 0) {
		aprint_normal_dev(dev,
		    "NOTE: top 8 bits of prehistoric PRID not 0!\n");
		aprint_normal_dev(dev, "Please mail port-mips@NetBSD.org "
		    "with %s dmesg lines.\n", device_xname(dev));
	}

	switch (opts->mips_cpu_arch) {
#if defined(MIPS1)
	case CPU_ARCH_MIPS1:
		if (mci->mci_picache_size)
			aprint_normal_dev(dev, "%dKB/%dB %s Instruction cache, "
			    "%d TLB entries\n", mci->mci_picache_size / 1024,
			    mci->mci_picache_line_size,
			    wayname(mci->mci_picache_ways),
			    opts->mips_num_tlb_entries);
		else
			aprint_normal_dev(dev, "%d TLB entries\n",
			    opts->mips_num_tlb_entries);
		if (mci->mci_pdcache_size)
			aprint_normal_dev(dev, "%dKB/%dB %s %s Data cache\n",
			    mci->mci_pdcache_size / 1024,
			    mci->mci_pdcache_line_size,
			    wayname(mci->mci_pdcache_ways),
			    wtnames[mci->mci_pdcache_write_through]);
		break;
#endif /* MIPS1 */
#if (MIPS3 + MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
	case CPU_ARCH_MIPS3:
	case CPU_ARCH_MIPS4:
	case CPU_ARCH_MIPS32:
	case CPU_ARCH_MIPS32R2:
	case CPU_ARCH_MIPS64:
	case CPU_ARCH_MIPS64R2: {
		const char *sufx = "KMGTPE";
		uint32_t pg_mask;
		aprint_normal_dev(dev, "%d TLB entries",
		    opts->mips_num_tlb_entries);
#if !defined(__mips_o32)
		if (CPUIS64BITS) {
			int64_t pfn_mask;
			i = ffs(~(opts->mips3_tlb_vpn_mask >> 31)) + 30;
			aprint_normal(", %d%cB (%d-bit) VAs",
			    1 << (i % 10), sufx[(i / 10) - 1], i);
			for (i = 64, pfn_mask = opts->mips3_tlb_pfn_mask << 6;
			     pfn_mask > 0; i--, pfn_mask <<= 1)
				;
			aprint_normal(", %d%cB (%d-bit) PAs",
			      1 << (i % 10), sufx[(i / 10) - 1], i);
		}
#endif
		for (i = 4, pg_mask = opts->mips3_tlb_pg_mask >> 13;
		     pg_mask != 0; ) {
			if ((pg_mask & 3) != 3)
				break;
			pg_mask >>= 2;
			i *= 4;
			if (i == 1024) {
				i = 1;
				sufx++;
			}
		}
		aprint_normal(", %d%cB max page size\n", i, sufx[0]);
		if (mci->mci_picache_size)
			aprint_normal_dev(dev,
			    "%dKB/%dB %s L1 instruction cache\n",
			    mci->mci_picache_size / 1024,
			    mci->mci_picache_line_size,
			    wayname(mci->mci_picache_ways));
		if (mci->mci_pdcache_size)
			aprint_normal_dev(dev,
			    "%dKB/%dB %s %s %sL1 data cache\n",
			    mci->mci_pdcache_size / 1024,
			    mci->mci_pdcache_line_size,
			    wayname(mci->mci_pdcache_ways),
			    wtnames[mci->mci_pdcache_write_through],
			    ((opts->mips_cpu_flags & CPU_MIPS_D_CACHE_COHERENT)
				? "coherent " : ""));
		if (mci->mci_sdcache_line_size)
			aprint_normal_dev(dev,
			    "%dKB/%dB %s %s L2 %s cache\n",
			    mci->mci_sdcache_size / 1024,
			    mci->mci_sdcache_line_size,
			    wayname(mci->mci_sdcache_ways),
			    wtnames[mci->mci_sdcache_write_through],
			    mci->mci_scache_unified ? "unified" : "data");
		break;
	}
#endif /* (MIPS3 + MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */
	default:
		panic("cpu_identify: impossible");
	}
}

/*
 * Set registers on exec.
 * Clear all registers except sp, pc, and t9.
 * $sp is set to the stack pointer passed in.  $pc is set to the entry
 * point given by the exec_package passed in, as is $t9 (used for PIC
 * code by the MIPS elf abi).
 */
void
setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack)
{
	struct trapframe * const tf = l->l_md.md_utf;
	struct proc * const p = l->l_proc;

	memset(tf, 0, sizeof(struct trapframe));
	tf->tf_regs[_R_SP] = (intptr_t)stack;
	tf->tf_regs[_R_PC] = (intptr_t)pack->ep_entry & ~3;
	tf->tf_regs[_R_T9] = (intptr_t)pack->ep_entry & ~3; /* abicall requirement */
	tf->tf_regs[_R_SR] = PSL_USERSET;
#if !defined(__mips_o32)
	/*
	 * allow 64bit ops in userland for non-O32 ABIs
	 */
	if (p->p_md.md_abi == _MIPS_BSD_API_N32
	    && (CPUISMIPS64 || CPUISMIPS64R2)) {
		tf->tf_regs[_R_SR] |= MIPS_SR_PX;
	} else if (p->p_md.md_abi != _MIPS_BSD_API_O32) {
		tf->tf_regs[_R_SR] |= MIPS_SR_UX;
	}
	if (_MIPS_SIM_NEWABI_P(p->p_md.md_abi))
		tf->tf_regs[_R_SR] |= MIPS3_SR_FR;
#endif
#ifdef _LP64
	/*
	 * If we are using a 32-bit ABI on a 64-bit kernel, mark the process
	 * that way.  If we aren't, clear it.
	 */
	if (p->p_md.md_abi == _MIPS_BSD_API_N32
	    || p->p_md.md_abi == _MIPS_BSD_API_O32) {
		p->p_flag |= PK_32;
	} else {
		p->p_flag &= ~PK_32;
	}
#endif
	/*
	 * Set up arguments for _start():
	 *	_start(stack, obj, cleanup, ps_strings);
	 *
	 * Notes:
	 *	- obj and cleanup are the auxiliary and termination
	 *	  vectors.  They are fixed up by ld.elf_so.
	 *	- ps_strings is a NetBSD extension.
	 */
	tf->tf_regs[_R_A0] = (intptr_t)stack;
	tf->tf_regs[_R_A1] = 0;
	tf->tf_regs[_R_A2] = 0;
	tf->tf_regs[_R_A3] = p->p_psstrp;

	l->l_md.md_ss_addr = 0;
}

#ifdef __HAVE_BOOTINFO_H
/*
 * Machine dependent system variables.
 */
static int
sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)
{
	struct btinfo_bootpath *bibp;
	struct sysctlnode node;

	bibp = lookup_bootinfo(BTINFO_BOOTPATH);
	if(!bibp)
		return(ENOENT); /* ??? */

	node = *rnode;
	node.sysctl_data = bibp->bootpath;
	node.sysctl_size = sizeof(bibp->bootpath);
	return (sysctl_lookup(SYSCTLFN_CALL(&node)));
}
#endif

SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
{

	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT,
		       CTLTYPE_NODE, "machdep", NULL,
		       NULL, 0, NULL, 0,
		       CTL_MACHDEP, CTL_EOL);

	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT,
		       CTLTYPE_STRUCT, "console_device", NULL,
		       sysctl_consdev, 0, NULL, sizeof(dev_t),
		       CTL_MACHDEP, CPU_CONSDEV, CTL_EOL);
#ifdef __HAVE_BOOTINFO_H
	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT,
		       CTLTYPE_STRING, "booted_kernel", NULL,
		       sysctl_machdep_booted_kernel, 0, NULL, 0,
		       CTL_MACHDEP, CPU_BOOTED_KERNEL, CTL_EOL);
#endif
	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT,
		       CTLTYPE_STRING, "root_device", NULL,
		       sysctl_root_device, 0, NULL, 0,
		       CTL_MACHDEP, CPU_ROOT_DEVICE, CTL_EOL);
	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
		       CTLTYPE_INT, "llsc", NULL,
		       NULL, MIPS_HAS_LLSC, NULL, 0,
		       CTL_MACHDEP, CPU_LLSC, CTL_EOL);
#ifdef MIPS3_LOONGSON2
	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
		       CTLTYPE_INT, "loongson-mmi", NULL,
		       NULL, MIPS_HAS_LMMI, NULL, 0,
		       CTL_MACHDEP, CPU_LMMI, CTL_EOL);
#endif
	sysctl_createv(clog, 0, NULL, NULL,
		       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
		       CTLTYPE_INT, "fpu_present", NULL,
		       NULL,
#ifdef NOFPU
		       0,
#else
		       1,
#endif
		       NULL, 0, CTL_MACHDEP, CTL_CREATE, CTL_EOL);
}

/*
 * These are imported from platform-specific code.
 * XXX Should be declared in a header file.
 */
extern phys_ram_seg_t mem_clusters[];
extern int mem_cluster_cnt;

/*
 * These variables are needed by /sbin/savecore.
 */
u_int32_t dumpmag = 0x8fca0101;	/* magic number */
int	dumpsize = 0;		/* pages */
long	dumplo = 0;		/* blocks */

struct pcb dumppcb;

/*
 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers.
 */
int
cpu_dumpsize(void)
{
	int size;

	size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) +
	    ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t));
	if (roundup(size, dbtob(1)) != dbtob(1))
		return (-1);

	return (1);
}

/*
 * cpu_dump_mempagecnt: calculate size of RAM (in pages) to be dumped.
 */
u_long
cpu_dump_mempagecnt(void)
{
	u_long i, n;

	n = 0;
	for (i = 0; i < mem_cluster_cnt; i++)
		n += atop(mem_clusters[i].size);
	return (n);
}

/*
 * cpu_dump: dump machine-dependent kernel core dump headers.
 */
int
cpu_dump(void)
{
	int (*dump)(dev_t, daddr_t, void *, size_t);
	char buf[dbtob(1)];
	kcore_seg_t *segp;
	cpu_kcore_hdr_t *cpuhdrp;
	phys_ram_seg_t *memsegp;
	const struct bdevsw *bdev;
	int i;

	bdev = bdevsw_lookup(dumpdev);
	if (bdev == NULL)
		return (ENXIO);

	dump = bdev->d_dump;

	memset(buf, 0, sizeof buf);
	segp = (kcore_seg_t *)buf;
	cpuhdrp = (cpu_kcore_hdr_t *)&buf[ALIGN(sizeof(*segp))];
	memsegp = (phys_ram_seg_t *)&buf[ ALIGN(sizeof(*segp)) +
	    ALIGN(sizeof(*cpuhdrp))];

	/*
	 * Generate a segment header.
	 */
	CORE_SETMAGIC(*segp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
	segp->c_size = dbtob(1) - ALIGN(sizeof(*segp));

	/*
	 * Add the machine-dependent header info.
	 */
	if (MIPS_HAS_R4K_MMU) {
		cpuhdrp->archlevel = 3;
		cpuhdrp->pg_shift  = MIPS3_PG_SHIFT;
		cpuhdrp->pg_frame  = MIPS3_PG_FRAME;
		cpuhdrp->pg_v      = MIPS3_PG_V;
	} else {
		cpuhdrp->archlevel = 1;
		cpuhdrp->pg_shift  = MIPS1_PG_SHIFT;
		cpuhdrp->pg_frame  = MIPS1_PG_FRAME;
		cpuhdrp->pg_v      = MIPS1_PG_V;
	}
	cpuhdrp->sysmappa   = MIPS_KSEG0_TO_PHYS(curcpu()->ci_pmap_kern_segtab);
	cpuhdrp->nmemsegs   = mem_cluster_cnt;

	/*
	 * Fill in the memory segment descriptors.
	 */
	for (i = 0; i < mem_cluster_cnt; i++) {
		memsegp[i].start = mem_clusters[i].start;
		memsegp[i].size = mem_clusters[i].size;
	}

	return (dump(dumpdev, dumplo, (void *)buf, dbtob(1)));
}

/*
 * This is called by main to set dumplo and dumpsize.
 * Dumps always skip the first CLBYTES of disk space
 * in case there might be a disk label stored there.
 * If there is extra space, put dump at the end to
 * reduce the chance that swapping trashes it.
 */
void
cpu_dumpconf(void)
{
	int nblks, dumpblks;	/* size of dump area */

	if (dumpdev == NODEV)
		goto bad;
	nblks = bdev_size(dumpdev);
	if (nblks <= ctod(1))
		goto bad;

	dumpblks = cpu_dumpsize();
	if (dumpblks < 0)
		goto bad;
	dumpblks += ctod(cpu_dump_mempagecnt());

	/* If dump won't fit (incl. room for possible label), punt. */
	if (dumpblks > (nblks - ctod(1)))
		goto bad;

	/* Put dump at end of partition */
	dumplo = nblks - dumpblks;

	/* dumpsize is in page units, and doesn't include headers. */
	dumpsize = cpu_dump_mempagecnt();
	return;

 bad:
	dumpsize = 0;
}

/*
 * Dump the kernel's image to the swap partition.
 */
#define	BYTES_PER_DUMP	PAGE_SIZE

void
dumpsys(void)
{
	u_long totalbytesleft, bytes, i, n, memcl;
	u_long maddr;
	int psize;
	daddr_t blkno;
	const struct bdevsw *bdev;
	int (*dump)(dev_t, daddr_t, void *, size_t);
	int error;

	/* Save registers. */
	savectx(&dumppcb);

	if (dumpdev == NODEV)
		return;
	bdev = bdevsw_lookup(dumpdev);
	if (bdev == NULL || bdev->d_psize == NULL)
		return;

	/*
	 * For dumps during autoconfiguration,
	 * if dump device has already configured...
	 */
	if (dumpsize == 0)
		cpu_dumpconf();
	if (dumplo <= 0) {
		printf("\ndump to dev %u,%u not possible\n", major(dumpdev),
		    minor(dumpdev));
		return;
	}
	printf("\ndumping to dev %u,%u offset %ld\n", major(dumpdev),
	    minor(dumpdev), dumplo);

	psize = bdev_size(dumpdev);
	printf("dump ");
	if (psize == -1) {
		printf("area unavailable\n");
		return;
	}

	/* XXX should purge all outstanding keystrokes. */

	if ((error = cpu_dump()) != 0)
		goto err;

	totalbytesleft = ptoa(cpu_dump_mempagecnt());
	blkno = dumplo + cpu_dumpsize();
	dump = bdev->d_dump;
	error = 0;

	for (memcl = 0; memcl < mem_cluster_cnt; memcl++) {
		maddr = mem_clusters[memcl].start;
		bytes = mem_clusters[memcl].size;

		for (i = 0; i < bytes; i += n, totalbytesleft -= n) {
			void *maddr_va;

			/* Print out how many MBs we have left to go. */
			if ((totalbytesleft % (1024*1024)) == 0)
				printf_nolog("%ld ",
				    totalbytesleft / (1024 * 1024));

			/* Limit size for next transfer. */
			n = bytes - i;
			if (n > BYTES_PER_DUMP)
				n = BYTES_PER_DUMP;

#ifdef _LP64
			maddr_va = (void *)MIPS_PHYS_TO_XKPHYS_CACHED(maddr);
#else
			maddr_va = (void *)MIPS_PHYS_TO_KSEG0(maddr);
#endif
			error = (*dump)(dumpdev, blkno, maddr_va, n);
			if (error)
				goto err;
			maddr += n;
			blkno += btodb(n);		/* XXX? */

			/* XXX should look for keystrokes, to cancel. */
		}
	}

 err:
	switch (error) {

	case ENXIO:
		printf("device bad\n");
		break;

	case EFAULT:
		printf("device not ready\n");
		break;

	case EINVAL:
		printf("area improper\n");
		break;

	case EIO:
		printf("i/o error\n");
		break;

	case EINTR:
		printf("aborted from console\n");
		break;

	case 0:
		printf("succeeded\n");
		break;

	default:
		printf("error %d\n", error);
		break;
	}
	printf("\n\n");
	delay(5000000);		/* 5 seconds */
}

void
mips_init_msgbuf(void)
{
	vsize_t sz = (vsize_t)round_page(MSGBUFSIZE);
	vsize_t reqsz = sz;
	uvm_physseg_t bank = uvm_physseg_get_last();
#ifndef _LP64
	/*
	 * First the physical segment that can be mapped to KSEG0
	 */
	for (; uvm_physseg_valid_p(bank); bank = uvm_physseg_get_prev(bank)) {
		if (uvm_physseg_get_avail_start(bank) + atop(sz) <= atop(MIPS_PHYS_MASK))
			break;
	}
#endif

	paddr_t start = uvm_physseg_get_start(bank);
	paddr_t end = uvm_physseg_get_end(bank);

	/* shrink so that it'll fit in the last segment */
	if ((end - start) < atop(sz))
		sz = ptoa(end - start);

	end -= atop(sz);
	uvm_physseg_unplug(end, atop(sz));

#ifdef _LP64
	msgbufaddr = (void *) MIPS_PHYS_TO_XKPHYS_CACHED(ptoa(end));
#else
	msgbufaddr = (void *) MIPS_PHYS_TO_KSEG0(ptoa(end));
#endif
	initmsgbuf(msgbufaddr, sz);

	/* warn if the message buffer had to be shrunk */
	if (sz != reqsz)
		printf("WARNING: %"PRIdVSIZE" bytes not available for msgbuf "
		    "in last cluster (%"PRIdVSIZE" used)\n", reqsz, sz);
}

void
mips_init_lwp0_uarea(void)
{
	struct lwp * const l = &lwp0;
	vaddr_t v;

	if (l->l_addr == NULL) {
		v = uvm_pageboot_alloc(USPACE);
		uvm_lwp_setuarea(&lwp0, v);
	} else {
		v = (vaddr_t)l->l_addr;
	}

	l->l_md.md_utf = (struct trapframe *)(v + USPACE) - 1;
	struct pcb * const pcb = lwp_getpcb(l);
	/*
	 * Now zero out the only two areas of the uarea that we care about.
	 */
	memset(l->l_md.md_utf, 0, sizeof(*l->l_md.md_utf));
	memset(pcb, 0, sizeof(*pcb));

	pcb->pcb_context.val[_L_SR] = MIPS_SR_INT_IE
	    | (ipl_sr_map.sr_bits[IPL_SCHED] ^ MIPS_INT_MASK);
#ifdef __mips_n32
	pcb->pcb_context.val[_L_SR] |= MIPS_SR_KX;
	l->l_md.md_utf->tf_regs[_R_SR] = MIPS_SR_KX;
#endif
#ifdef _LP64
	pcb->pcb_context.val[_L_SR] |= MIPS_SR_KX | MIPS_SR_UX;
	l->l_md.md_utf->tf_regs[_R_SR] = MIPS_SR_KX | MIPS_SR_UX;
#endif
}

int mips_poolpage_vmfreelist = VM_FREELIST_DEFAULT;

#define	HALFGIG		((paddr_t)512 * 1024 * 1024)
#define	FOURGIG		((paddr_t)4 * 1024 * 1024 * 1024)

void
mips_page_physload(vaddr_t vkernstart, vaddr_t vkernend,
	const phys_ram_seg_t *segs, size_t nseg,
	const struct mips_vmfreelist *flp, size_t nfl)
{
	const paddr_t kernstart = MIPS_KSEG0_TO_PHYS(trunc_page(vkernstart));
	const paddr_t kernend = MIPS_KSEG0_TO_PHYS(round_page(vkernend));
#if defined(VM_FREELIST_FIRST4G) || defined(VM_FREELIST_FIRST512M)
#ifdef VM_FREELIST_FIRST512M
	bool need512m = false;
#endif
#ifdef VM_FREELIST_FIRST4G
	bool need4g = false;
#endif

	/*
	 * Do a first pass and see what ranges memory we have to deal with.
	 */
	for (size_t i = 0; i < nseg; i++) {
#ifdef VM_FREELIST_FIRST4G
		if (round_page(segs[i].start + segs[i].size) > FOURGIG) {
			need4g = true;
		}
#endif
#ifdef VM_FREELIST_FIRST512M
		if (round_page(segs[i].start + segs[i].size) > HALFGIG) {
			need512m = true;
#if !defined(_LP64)
			mips_poolpage_vmfreelist = VM_FREELIST_FIRST512M;
#endif
		}
#endif
	}
#endif /* VM_FREELIST_FIRST512M || VM_FREELIST_FIRST4G */

	for (; nseg-- > 0; segs++) {
		/*
		 * Make sure everything is in page units.
		 */
		paddr_t segstart = round_page(segs->start);
		const paddr_t segfinish = trunc_page(segs->start + segs->size);

		if (segstart >= segfinish) {
			/*
			 * This is purely cosmetic, to avoid output like
			 *    phys segment: 0xffffffffffffe000 @ 0xffb6000
			 * when a segment starts and finishes in the same page.
			 */
			printf("phys segment: %#"PRIxPADDR" @ %#"PRIxPADDR
			    " (short)\n", (paddr_t)segs->size, segstart);
			continue;
		}

		printf("phys segment: %#"PRIxPADDR" @ %#"PRIxPADDR"\n",
		    segfinish - segstart, segstart);

		/*
		 * Page 0 is reserved for exception vectors.
		 */
		if (segstart == 0) {
			segstart = PAGE_SIZE;
		}
		while (segstart < segfinish) {
			int freelist = -1;	/* unknown freelist */
			paddr_t segend = segfinish;
			for (size_t i = 0; i < nfl; i++) {
				/*
				 * If this segment doesn't overlap the freelist
				 * at all, skip it.
				 */
				if (segstart >= flp[i].fl_end
				    || segend <= flp[i].fl_start)
					continue;
				/*
				 * If the start of this segment starts before
				 * the start of the freelist, then limit the
				 * segment to loaded to the part that doesn't
				 * match this freelist and fall back to normal
				 * freelist matching.
				 */
				if (segstart < flp[i].fl_start) {
					segstart = flp[i].fl_start;
					break;
				}

				/*
				 * We've matched this freelist so remember it.
				 */
				freelist = flp->fl_freelist;

				/*
				 * If this segment extends past the end of this
				 * freelist, bound to segment to the freelist.
				 */
				if (segend > flp[i].fl_end)
					segend = flp[i].fl_end;
				break;
			}
			/*
			 * If we didn't match one of the port dependent
			 * freelists, let's try the common ones.
			 */
			if (freelist == -1) {
#ifdef VM_FREELIST_FIRST512M
				if (need512m && segstart < HALFGIG) {
					freelist = VM_FREELIST_FIRST512M;
					if (segend > HALFGIG)
						segend = HALFGIG;
				} else
#endif
#ifdef VM_FREELIST_FIRST4G
				if (need4g && segstart < FOURGIG) {
					freelist = VM_FREELIST_FIRST4G;
					if (segend > FOURGIG)
						segend = FOURGIG;
				} else
#endif
					freelist = VM_FREELIST_DEFAULT;
			}

			/*
			 * Make sure the memory we provide to uvm doesn't
			 * include the kernel.
			 */
			if (segstart < kernend && segend > kernstart) {
				if (segstart < kernstart) {
					/*
					 * Only add the memory before the
					 * kernel.
					 */
					segend = kernstart;
				} else if (segend > kernend) {
					/*
					 * Only add the memory after the
					 * kernel.
					 */
					segstart = kernend;
				} else {
					/*
					 * Just skip the segment entirely since
					 * it's completely inside the kernel.
					 */
					printf("skipping %#"PRIxPADDR" @ %#"PRIxPADDR" (kernel)\n",
					    segend - segstart, segstart);
					break;
				}
			}

			/*
			 * Now we give this segment to uvm.
			 */
			printf("adding %#"PRIxPADDR" @ %#"PRIxPADDR" to freelist %d\n",
			    segend - segstart, segstart, freelist);
			paddr_t first = atop(segstart);
			paddr_t last = atop(segend);
			uvm_page_physload(first, last, first, last, freelist);

			/*
			 * Start where we finished.
			 */
			segstart = segend;
		}
	}
}

/*
 * Start a new LWP
 */
void
startlwp(void *arg)
{
	ucontext_t * const uc = arg;
	lwp_t * const l = curlwp;
	int error __diagused;

	error = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags);
	KASSERT(error == 0);

	kmem_free(uc, sizeof(ucontext_t));
	userret(l);
}

#ifdef COMPAT_NETBSD32
/*
 * Start a new LWP
 */
void
startlwp32(void *arg)
{
	ucontext32_t * const uc = arg;
	lwp_t * const l = curlwp;
	int error __diagused;

	error = cpu_setmcontext32(l, &uc->uc_mcontext, uc->uc_flags);
	KASSERT(error == 0);

	/* Note: we are freeing ucontext_t, not ucontext32_t. */
	kmem_free(uc, sizeof(ucontext_t));
	userret(l);
}
#endif /* COMPAT_NETBSD32 */

#ifdef PARANOIA
void
std_splsw_test(void)
{
	struct cpu_info * const ci = curcpu();
	const uint32_t * const sr_map = ipl_sr_map.sr_bits;
	uint32_t status = mips_cp0_status_read();
	uint32_t sr_bits;
	int s;

	KASSERT((status & MIPS_SR_INT_IE) == 0);

	sr_bits = sr_map[IPL_NONE];

	splx(IPL_NONE);
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT(status == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_NONE);

	s = splsoftclock();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_SOFTCLOCK]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_SOFTCLOCK);
	KASSERT(s == IPL_NONE);

	s = splsoftbio();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_SOFTBIO]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_SOFTBIO);
	KASSERT(s == IPL_SOFTCLOCK);

	s = splsoftnet();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_SOFTNET]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_SOFTNET);
	KASSERT(s == IPL_SOFTBIO);

	s = splsoftserial();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_SOFTSERIAL]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_SOFTSERIAL);
	KASSERT(s == IPL_SOFTNET);

	s = splvm();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_VM]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_VM);
	KASSERT(s == IPL_SOFTSERIAL);

	s = splsched();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_SCHED]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_SCHED);
	KASSERT(s == IPL_VM);

	s = splhigh();
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT((status ^ sr_map[IPL_HIGH]) == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_HIGH);
	KASSERT(s == IPL_SCHED);

	splx(IPL_NONE);
	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT(status == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_NONE);

	for (int r = IPL_SOFTCLOCK; r <= IPL_HIGH; r++) {
		/*
		 * As IPL increases, more intrs may be masked but no intrs
		 * may become unmasked.
		 */
		KASSERT((sr_map[r] & sr_bits) == sr_bits);
		sr_bits |= sr_map[r];
		s = splraise(r);
		KASSERT(s == IPL_NONE);

		for (int t = r; t <= IPL_HIGH; t++) {
			int o = splraise(t);
			status = mips_cp0_status_read() & MIPS_INT_MASK;
			KASSERT((status ^ sr_map[t]) == MIPS_INT_MASK);
			KASSERT(ci->ci_cpl == t);
			KASSERT(o == r);

			splx(o);
			status = mips_cp0_status_read() & MIPS_INT_MASK;
			KASSERT((status ^ sr_map[r]) == MIPS_INT_MASK);
			KASSERT(ci->ci_cpl == r);
		}

		splx(s);
		status = mips_cp0_status_read() & MIPS_INT_MASK;
		KASSERT((status ^ sr_map[s]) == MIPS_INT_MASK);
		KASSERT(ci->ci_cpl == s);
	}

	status = mips_cp0_status_read() & MIPS_INT_MASK;
	KASSERT(status == MIPS_INT_MASK);
	KASSERT(ci->ci_cpl == IPL_NONE);
}

#endif /* PARANOIA */

#ifdef MODULAR
/*
 * Push any modules loaded by the boot loader.
 */
void
module_init_md(void)
{

	/* XXX Do something board/machine specific here one day... */
}
#endif /* MODULAR */

bool
mm_md_direct_mapped_phys(paddr_t pa, vaddr_t *vap)
{
#ifdef _LP64
	if (MIPS_XKSEG_P(pa)) {
		*vap = MIPS_PHYS_TO_XKPHYS_CACHED(pa);
		return true;
	}
#endif
	if (MIPS_KSEG0_P(pa)) {
		*vap = MIPS_PHYS_TO_KSEG0(pa);
		return true;
	}
	return false;
}

bool
mm_md_page_color(paddr_t pa, int *colorp)
{
	if (MIPS_CACHE_VIRTUAL_ALIAS) {
		struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
		KASSERT(pg != NULL);
		struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
		*colorp = atop(mdpg->mdpg_first.pv_va);
		return !mips_cache_badalias(pa, mdpg->mdpg_first.pv_va);
	}
	*colorp = 0;
	return true;
}

int
mm_md_physacc(paddr_t pa, vm_prot_t prot)
{

	return (pa < ctob(physmem)) ? 0 : EFAULT;
}

int
mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled)
{
	const vaddr_t v = (vaddr_t)ptr;

#ifdef _LP64
	extern char end[];

	/* For any address < XKPHYS cached address 0, fault */
	if (v < MIPS_PHYS_TO_XKPHYS_CACHED(0)) {
		return EFAULT;
	}

	/* If address < XKPHY(end of message buffer), good! */
	if (v < MIPS_PHYS_TO_XKPHYS_CACHED(pmap_limits.avail_end +
	    mips_round_page(MSGBUFSIZE))) {
		/* XXX holes in RAM (eg, EdgeRouter 4) */
		*handled = true;
		return 0;
	}

	/* If address in KSEG0 and is before end of kernel, good! */
	if (MIPS_KSEG0_P(v) && v < (vaddr_t)end) {
		*handled = true;
		return 0;
	}

	/* Otherwise, fall back to the uvm_kernacc() check. */
#else
	if (v < MIPS_KSEG0_START) {
		return EFAULT;
	}
	if (v < MIPS_PHYS_TO_KSEG0(pmap_limits.avail_end +
	    mips_round_page(MSGBUFSIZE))) {
		*handled = true;
		return 0;
	}
	if (v < MIPS_KSEG2_START) {
		return EFAULT;
	}
#endif
	*handled = false;
	return 0;
}

#if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
static void
mips_watchpoint_init(void)
{
	/*
	 * determine number of CPU watchpoints
	 */
	curcpu()->ci_cpuwatch_count = cpuwatch_discover();
}
#endif


/*
 * Process the tail end of a posix_spawn() for the child.
 */
void
cpu_spawn_return(struct lwp *l)
{
	userret(l);
}