1 /* $NetBSD: pmap.h,v 1.46 2018/05/19 15:03:26 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Copyright (c) 2001 Wasabi Systems, Inc. 30 * All rights reserved. 31 * 32 * Written by Frank van der Linden for Wasabi Systems, Inc. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 3. All advertising materials mentioning features or use of this software 43 * must display the following acknowledgement: 44 * This product includes software developed for the NetBSD Project by 45 * Wasabi Systems, Inc. 46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 47 * or promote products derived from this software without specific prior 48 * written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 60 * POSSIBILITY OF SUCH DAMAGE. 61 */ 62 63 #ifndef _AMD64_PMAP_H_ 64 #define _AMD64_PMAP_H_ 65 66 #ifdef __x86_64__ 67 68 #if defined(_KERNEL_OPT) 69 #include "opt_xen.h" 70 #endif 71 72 #include <sys/atomic.h> 73 74 #include <machine/pte.h> 75 #include <machine/segments.h> 76 #ifdef _KERNEL 77 #include <machine/cpufunc.h> 78 #endif 79 80 #include <uvm/uvm_object.h> 81 #ifdef XEN 82 #include <xen/xenfunc.h> 83 #include <xen/xenpmap.h> 84 #endif /* XEN */ 85 86 /* 87 * The x86_64 pmap module closely resembles the i386 one and it 88 * uses the same recursive entry scheme. See the i386 pmap.h 89 * for a description. The obvious difference is that 3 extra 90 * levels of page table need to be dealt with. The level 1 page 91 * table pages are at: 92 * 93 * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry) 94 * 95 * The rest is kept as physical pages in 3 UVM objects, and is 96 * temporarily mapped for virtual access when needed. 97 * 98 * Note that address space is signed, so the layout for 48 bits is: 99 * 100 * +---------------------------------+ 0xffffffffffffffff 101 * | | 102 * | Unused | 103 * | | 104 * +---------------------------------+ 0xffffff8000000000 105 * ~ ~ 106 * | | 107 * | Kernel Space | 108 * | | 109 * | | 110 * +---------------------------------+ 0xffff800000000000 = 0x0000800000000000 111 * | | 112 * | alt.L1 table (PTE pages) | 113 * | | 114 * +---------------------------------+ 0x00007f8000000000 115 * ~ ~ 116 * | | 117 * | User Space | 118 * | | 119 * | | 120 * +---------------------------------+ 0x0000000000000000 121 * 122 * In other words, there is a 'VA hole' at 0x0000800000000000 - 123 * 0xffff800000000000 which will trap, just as on, for example, 124 * sparcv9. 125 * 126 * The unused space can be used if needed, but it adds a little more 127 * complexity to the calculations. 128 */ 129 130 /* 131 * Mask to get rid of the sign-extended part of addresses. 132 */ 133 #define VA_SIGN_MASK 0xffff000000000000 134 #define VA_SIGN_NEG(va) ((va) | VA_SIGN_MASK) 135 /* 136 * XXXfvdl this one's not right. 137 */ 138 #define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK) 139 140 #define L4_SLOT_PTE 255 141 #ifndef XEN 142 #define L4_SLOT_KERN 256 /* pl4_i(VM_MIN_KERNEL_ADDRESS) */ 143 #else 144 /* Xen use slots 256-272, let's move farther */ 145 #define L4_SLOT_KERN 320 /* pl4_i(VM_MIN_KERNEL_ADDRESS) */ 146 #endif 147 #define L4_SLOT_KERNBASE 511 /* pl4_i(KERNBASE) */ 148 149 #define PDIR_SLOT_KERN L4_SLOT_KERN 150 #define PDIR_SLOT_PTE L4_SLOT_PTE 151 152 /* 153 * The following defines give the virtual addresses of various MMU 154 * data structures: 155 * PTE_BASE: the base VA of the linear PTE mappings 156 * PDP_BASE: the base VA of the recursive mapping of the PTD 157 * PDP_PDE: the VA of the PDE that points back to the PDP 158 */ 159 160 #define PTE_BASE ((pt_entry_t *)(L4_SLOT_PTE * NBPD_L4)) 161 #define KERN_BASE ((pt_entry_t *)(L4_SLOT_KERN * NBPD_L4)) 162 163 #define L1_BASE PTE_BASE 164 #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3)) 165 #define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2)) 166 #define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1)) 167 168 #define PDP_PDE (L4_BASE + PDIR_SLOT_PTE) 169 170 #define PDP_BASE L4_BASE 171 172 #define NKL4_MAX_ENTRIES (unsigned long)64 173 #define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512) 174 #define NKL2_MAX_ENTRIES (unsigned long)(NKL3_MAX_ENTRIES * 512) 175 #define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * 512) 176 177 #define NKL4_KIMG_ENTRIES 1 178 #define NKL3_KIMG_ENTRIES 1 179 #define NKL2_KIMG_ENTRIES 48 180 181 /* 182 * Since kva space is below the kernel in its entirety, we start off 183 * with zero entries on each level. 184 */ 185 #define NKL4_START_ENTRIES 0 186 #define NKL3_START_ENTRIES 0 187 #define NKL2_START_ENTRIES 0 188 #define NKL1_START_ENTRIES 0 /* XXX */ 189 190 #define NTOPLEVEL_PDES (PAGE_SIZE / (sizeof (pd_entry_t))) 191 192 #define NPDPG (PAGE_SIZE / sizeof (pd_entry_t)) 193 194 #define PTP_MASK_INITIALIZER { L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME } 195 #define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT } 196 #define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES, \ 197 NKL3_START_ENTRIES, NKL4_START_ENTRIES } 198 #define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \ 199 NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES } 200 #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 } 201 #define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE } 202 203 #define PTP_LEVELS 4 204 205 /* 206 * PG_AVAIL usage: we make use of the ignored bits of the PTE 207 */ 208 209 #define PG_W PG_AVAIL1 /* "wired" mapping */ 210 #define PG_PVLIST PG_AVAIL2 /* mapping has entry on pvlist */ 211 /* PG_AVAIL3 not used */ 212 213 #define PG_X 0 /* XXX dummy */ 214 215 /* 216 * Number of PTE's per cache line. 8 byte pte, 64-byte cache line 217 * Used to avoid false sharing of cache lines. 218 */ 219 #define NPTECL 8 220 221 void svs_pmap_sync(struct pmap *, int); 222 void svs_lwp_switch(struct lwp *, struct lwp *); 223 void svs_pdir_switch(struct pmap *); 224 void svs_init(void); 225 extern bool svs_enabled; 226 227 #include <x86/pmap.h> 228 229 #ifndef XEN 230 #define pmap_pa2pte(a) (a) 231 #define pmap_pte2pa(a) ((a) & PG_FRAME) 232 #define pmap_pte_set(p, n) do { *(p) = (n); } while (0) 233 #define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n)) 234 #define pmap_pte_testset(p, n) \ 235 atomic_swap_ulong((volatile unsigned long *)p, n) 236 #define pmap_pte_setbits(p, b) \ 237 atomic_or_ulong((volatile unsigned long *)p, b) 238 #define pmap_pte_clearbits(p, b) \ 239 atomic_and_ulong((volatile unsigned long *)p, ~(b)) 240 #define pmap_pte_flush() /* nothing */ 241 #else 242 extern kmutex_t pte_lock; 243 244 static __inline pt_entry_t 245 pmap_pa2pte(paddr_t pa) 246 { 247 return (pt_entry_t)xpmap_ptom_masked(pa); 248 } 249 250 static __inline paddr_t 251 pmap_pte2pa(pt_entry_t pte) 252 { 253 return xpmap_mtop_masked(pte & PG_FRAME); 254 } 255 256 static __inline void 257 pmap_pte_set(pt_entry_t *pte, pt_entry_t npte) 258 { 259 int s = splvm(); 260 xpq_queue_pte_update(xpmap_ptetomach(pte), npte); 261 splx(s); 262 } 263 264 static __inline pt_entry_t 265 pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n) 266 { 267 pt_entry_t opte; 268 269 mutex_enter(&pte_lock); 270 opte = *ptep; 271 if (opte == o) { 272 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n); 273 xpq_flush_queue(); 274 } 275 276 mutex_exit(&pte_lock); 277 return opte; 278 } 279 280 static __inline pt_entry_t 281 pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte) 282 { 283 pt_entry_t opte; 284 285 mutex_enter(&pte_lock); 286 opte = *pte; 287 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte); 288 xpq_flush_queue(); 289 mutex_exit(&pte_lock); 290 return opte; 291 } 292 293 static __inline void 294 pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits) 295 { 296 mutex_enter(&pte_lock); 297 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits); 298 xpq_flush_queue(); 299 mutex_exit(&pte_lock); 300 } 301 302 static __inline void 303 pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits) 304 { 305 mutex_enter(&pte_lock); 306 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), 307 (*pte) & ~bits); 308 xpq_flush_queue(); 309 mutex_exit(&pte_lock); 310 } 311 312 static __inline void 313 pmap_pte_flush(void) 314 { 315 int s = splvm(); 316 xpq_flush_queue(); 317 splx(s); 318 } 319 #endif 320 321 #ifdef __HAVE_DIRECT_MAP 322 #define PMAP_DIRECT 323 324 static __inline int 325 pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len, 326 int (*process)(void *, size_t, void *), void *arg) 327 { 328 vaddr_t va = PMAP_DIRECT_MAP(pa); 329 330 return process((void *)(va + pgoff), len, arg); 331 } 332 333 #endif /* __HAVE_DIRECT_MAP */ 334 335 void pmap_changeprot_local(vaddr_t, vm_prot_t); 336 337 #include <x86/pmap_pv.h> 338 339 #define __HAVE_VM_PAGE_MD 340 #define VM_MDPAGE_INIT(pg) \ 341 memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \ 342 PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp) 343 344 struct vm_page_md { 345 struct pmap_page mp_pp; 346 }; 347 348 #else /* !__x86_64__ */ 349 350 #include <i386/pmap.h> 351 352 #endif /* __x86_64__ */ 353 354 #endif /* _AMD64_PMAP_H_ */ 355