1 /* $NetBSD: pmap.h,v 1.108 2010/11/14 13:33:21 uebayasi Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgment: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /* 36 * Copyright (c) 2001 Wasabi Systems, Inc. 37 * All rights reserved. 38 * 39 * Written by Frank van der Linden for Wasabi Systems, Inc. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. All advertising materials mentioning features or use of this software 50 * must display the following acknowledgement: 51 * This product includes software developed for the NetBSD Project by 52 * Wasabi Systems, Inc. 53 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 54 * or promote products derived from this software without specific prior 55 * written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 58 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 59 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 60 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 61 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 62 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 63 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 64 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 65 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 66 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 67 * POSSIBILITY OF SUCH DAMAGE. 68 */ 69 70 #ifndef _I386_PMAP_H_ 71 #define _I386_PMAP_H_ 72 73 #if defined(_KERNEL_OPT) 74 #include "opt_user_ldt.h" 75 #include "opt_xen.h" 76 #endif 77 78 #include <sys/atomic.h> 79 80 #include <i386/pte.h> 81 #include <machine/segments.h> 82 #if defined(_KERNEL) 83 #include <machine/cpufunc.h> 84 #endif 85 86 #include <uvm/uvm_object.h> 87 #ifdef XEN 88 #include <xen/xenfunc.h> 89 #include <xen/xenpmap.h> 90 #endif /* XEN */ 91 92 /* 93 * see pte.h for a description of i386 MMU terminology and hardware 94 * interface. 95 * 96 * a pmap describes a processes' 4GB virtual address space. when PAE 97 * is not in use, this virtual address space can be broken up into 1024 4MB 98 * regions which are described by PDEs in the PDP. the PDEs are defined as 99 * follows: 100 * 101 * (ranges are inclusive -> exclusive, just like vm_map_entry start/end) 102 * (the following assumes that KERNBASE is 0xc0000000) 103 * 104 * PDE#s VA range usage 105 * 0->766 0x0 -> 0xbfc00000 user address space 106 * 767 0xbfc00000-> recursive mapping of PDP (used for 107 * 0xc0000000 linear mapping of PTPs) 108 * 768->1023 0xc0000000-> kernel address space (constant 109 * 0xffc00000 across all pmap's/processes) 110 * 1023 0xffc00000-> "alternate" recursive PDP mapping 111 * <end> (for other pmaps) 112 * 113 * 114 * note: a recursive PDP mapping provides a way to map all the PTEs for 115 * a 4GB address space into a linear chunk of virtual memory. in other 116 * words, the PTE for page 0 is the first int mapped into the 4MB recursive 117 * area. the PTE for page 1 is the second int. the very last int in the 118 * 4MB range is the PTE that maps VA 0xfffff000 (the last page in a 4GB 119 * address). 120 * 121 * all pmap's PD's must have the same values in slots 768->1023 so that 122 * the kernel is always mapped in every process. these values are loaded 123 * into the PD at pmap creation time. 124 * 125 * at any one time only one pmap can be active on a processor. this is 126 * the pmap whose PDP is pointed to by processor register %cr3. this pmap 127 * will have all its PTEs mapped into memory at the recursive mapping 128 * point (slot #767 as show above). when the pmap code wants to find the 129 * PTE for a virtual address, all it has to do is the following: 130 * 131 * address of PTE = (767 * 4MB) + (VA / PAGE_SIZE) * sizeof(pt_entry_t) 132 * = 0xbfc00000 + (VA / 4096) * 4 133 * 134 * what happens if the pmap layer is asked to perform an operation 135 * on a pmap that is not the one which is currently active? in that 136 * case we take the PA of the PDP of non-active pmap and put it in 137 * slot 1023 of the active pmap. this causes the non-active pmap's 138 * PTEs to get mapped in the final 4MB of the 4GB address space 139 * (e.g. starting at 0xffc00000). 140 * 141 * the following figure shows the effects of the recursive PDP mapping: 142 * 143 * PDP (%cr3) 144 * +----+ 145 * | 0| -> PTP#0 that maps VA 0x0 -> 0x400000 146 * | | 147 * | | 148 * | 767| -> points back to PDP (%cr3) mapping VA 0xbfc00000 -> 0xc0000000 149 * | 768| -> first kernel PTP (maps 0xc0000000 -> 0xc0400000) 150 * | | 151 * |1023| -> points to alternate pmap's PDP (maps 0xffc00000 -> end) 152 * +----+ 153 * 154 * note that the PDE#767 VA (0xbfc00000) is defined as "PTE_BASE" 155 * note that the PDE#1023 VA (0xffc00000) is defined as "APTE_BASE" 156 * 157 * starting at VA 0xbfc00000 the current active PDP (%cr3) acts as a 158 * PTP: 159 * 160 * PTP#767 == PDP(%cr3) => maps VA 0xbfc00000 -> 0xc0000000 161 * +----+ 162 * | 0| -> maps the contents of PTP#0 at VA 0xbfc00000->0xbfc01000 163 * | | 164 * | | 165 * | 767| -> maps contents of PTP#767 (the PDP) at VA 0xbfeff000 166 * | 768| -> maps contents of first kernel PTP 167 * | | 168 * |1023| 169 * +----+ 170 * 171 * note that mapping of the PDP at PTP#767's VA (0xbfeff000) is 172 * defined as "PDP_BASE".... within that mapping there are two 173 * defines: 174 * "PDP_PDE" (0xbfeffbfc) is the VA of the PDE in the PDP 175 * which points back to itself. 176 * "APDP_PDE" (0xbfeffffc) is the VA of the PDE in the PDP which 177 * establishes the recursive mapping of the alternate pmap. 178 * to set the alternate PDP, one just has to put the correct 179 * PA info in *APDP_PDE. 180 * 181 * note that in the APTE_BASE space, the APDP appears at VA 182 * "APDP_BASE" (0xfffff000). 183 * 184 * - PAE support - 185 * --------------- 186 * 187 * PAE adds another layer of indirection during address translation, breaking 188 * up the translation process in 3 different levels: 189 * - L3 page directory, containing 4 * 64-bits addresses (index determined by 190 * bits [31:30] from the virtual address). This breaks up the address space 191 * in 4 1GB regions. 192 * - the PD (L2), containing 512 64-bits addresses, breaking each L3 region 193 * in 512 * 2MB regions. 194 * - the PT (L1), also containing 512 64-bits addresses (at L1, the size of 195 * the pages is still 4K). 196 * 197 * The kernel virtual space is mapped by the last entry in the L3 page, 198 * the first 3 entries mapping the user VA space. 199 * 200 * Because the L3 has only 4 entries of 1GB each, we can't use recursive 201 * mappings at this level for PDP_PDE and APDP_PDE (this would eat up 2 of 202 * the 4GB virtual space). There are also restrictions imposed by Xen on the 203 * last entry of the L3 PD (reference count to this page cannot be bigger 204 * than 1), which makes it hard to use one L3 page per pmap to switch 205 * between pmaps using %cr3. 206 * 207 * As such, each CPU gets its own L3 page that is always loaded into its %cr3 208 * (ci_pae_l3_pd in the associated cpu_info struct). We claim that the VM has 209 * only a 2-level PTP (similar to the non-PAE case). L2 PD is now 4 contiguous 210 * pages long (corresponding to the 4 entries of the L3), and the different 211 * index/slots (like PDP_PDE) are adapted accordingly. 212 * 213 * Kernel space remains in L3[3], L3[0-2] maps the user VA space. Switching 214 * between pmaps consists in modifying the first 3 entries of the CPU's L3 page. 215 * 216 * PTE_BASE and APTE_BASE will need 4 entries in the L2 PD pages to map the 217 * L2 pages recursively. 218 * 219 * In addition, for Xen, we can't recursively map L3[3] (Xen wants the ref 220 * count on this page to be exactly one), so we use a shadow PD page for 221 * the last L2 PD. The shadow page could be static too, but to make pm_pdir[] 222 * contiguous we'll allocate/copy one page per pmap. 223 */ 224 /* XXX MP should we allocate one APDP_PDE per processor?? */ 225 226 /* 227 * Mask to get rid of the sign-extended part of addresses. 228 */ 229 #define VA_SIGN_MASK 0 230 #define VA_SIGN_NEG(va) ((va) | VA_SIGN_MASK) 231 /* 232 * XXXfvdl this one's not right. 233 */ 234 #define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK) 235 236 /* 237 * the following defines identify the slots used as described above. 238 */ 239 #ifdef PAE 240 #define L2_SLOT_PTE (KERNBASE/NBPD_L2-4) /* 1532: for recursive PDP map */ 241 #define L2_SLOT_KERN (KERNBASE/NBPD_L2) /* 1536: start of kernel space */ 242 #ifndef XEN 243 #define L2_SLOT_APTE 2044 /* 2044: alternative recursive slot */ 244 #else 245 #define L2_SLOT_APTE 1960 /* 1964-2047 reserved by Xen */ 246 #endif 247 #else /* PAE */ 248 #define L2_SLOT_PTE (KERNBASE/NBPD_L2-1) /* 767: for recursive PDP map */ 249 #define L2_SLOT_KERN (KERNBASE/NBPD_L2) /* 768: start of kernel space */ 250 #ifndef XEN 251 #define L2_SLOT_APTE 1023 /* 1023: alternative recursive slot */ 252 #else 253 #define L2_SLOT_APTE 1007 /* 1008-1023 reserved by Xen */ 254 #endif 255 #endif /* PAE */ 256 257 #define L2_SLOT_KERNBASE L2_SLOT_KERN 258 259 #define PDIR_SLOT_KERN L2_SLOT_KERN 260 #define PDIR_SLOT_PTE L2_SLOT_PTE 261 #define PDIR_SLOT_APTE L2_SLOT_APTE 262 263 /* 264 * the following defines give the virtual addresses of various MMU 265 * data structures: 266 * PTE_BASE and APTE_BASE: the base VA of the linear PTE mappings 267 * PDP_BASE and APDP_BASE: the base VA of the recursive mapping of the PDP 268 * PDP_PDE and APDP_PDE: the VA of the PDE that points back to the PDP/APDP 269 */ 270 271 #define PTE_BASE ((pt_entry_t *) (PDIR_SLOT_PTE * NBPD_L2)) 272 #define APTE_BASE ((pt_entry_t *) (VA_SIGN_NEG((PDIR_SLOT_APTE * NBPD_L2)))) 273 274 #define L1_BASE PTE_BASE 275 #define AL1_BASE APTE_BASE 276 277 #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L2_SLOT_PTE * NBPD_L1)) 278 #define AL2_BASE ((pd_entry_t *)((char *)AL1_BASE + L2_SLOT_PTE * NBPD_L1)) 279 280 #define PDP_PDE (L2_BASE + PDIR_SLOT_PTE) 281 #if defined(PAE) && defined(XEN) 282 /* 283 * when PAE is in use under Xen, we can't write APDP_PDE through the recursive 284 * mapping, because it points to the shadow PD. Use the kernel PD instead, 285 * which is static 286 */ 287 #define APDP_PDE (&pmap_kl2pd[l2tol2(PDIR_SLOT_APTE)]) 288 #define APDP_PDE_SHADOW (L2_BASE + PDIR_SLOT_APTE) 289 #else /* PAE && XEN */ 290 #define APDP_PDE (L2_BASE + PDIR_SLOT_APTE) 291 #endif /* PAE && XEN */ 292 293 #define PDP_BASE L2_BASE 294 #define APDP_BASE AL2_BASE 295 296 /* largest value (-1 for APTP space) */ 297 #define NKL2_MAX_ENTRIES (NTOPLEVEL_PDES - (KERNBASE/NBPD_L2) - 1) 298 #define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * NPDPG) 299 300 #define NKL2_KIMG_ENTRIES 0 /* XXX unused */ 301 302 #define NKL2_START_ENTRIES 0 /* XXX computed on runtime */ 303 #define NKL1_START_ENTRIES 0 /* XXX unused */ 304 305 #define NTOPLEVEL_PDES (PAGE_SIZE * PDP_SIZE / (sizeof (pd_entry_t))) 306 307 #define NPDPG (PAGE_SIZE / sizeof (pd_entry_t)) 308 309 #define PTP_MASK_INITIALIZER { L1_FRAME, L2_FRAME } 310 #define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT } 311 #define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES } 312 #define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES } 313 #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2 } 314 #define PDES_INITIALIZER { L2_BASE } 315 #define APDES_INITIALIZER { AL2_BASE } 316 317 #define PTP_LEVELS 2 318 319 /* 320 * PG_AVAIL usage: we make use of the ignored bits of the PTE 321 */ 322 323 #define PG_W PG_AVAIL1 /* "wired" mapping */ 324 #define PG_PVLIST PG_AVAIL2 /* mapping has entry on pvlist */ 325 #define PG_X PG_AVAIL3 /* executable mapping */ 326 327 /* 328 * Number of PTE's per cache line. 4 byte pte, 32-byte cache line 329 * Used to avoid false sharing of cache lines. 330 */ 331 #ifdef PAE 332 #define NPTECL 4 333 #else 334 #define NPTECL 8 335 #endif 336 337 #include <x86/pmap.h> 338 339 #ifndef XEN 340 #define pmap_pa2pte(a) (a) 341 #define pmap_pte2pa(a) ((a) & PG_FRAME) 342 #define pmap_pte_set(p, n) do { *(p) = (n); } while (0) 343 #define pmap_pte_flush() /* nothing */ 344 345 #ifdef PAE 346 #define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n)) 347 #define pmap_pte_testset(p, n) \ 348 atomic_swap_64((volatile uint64_t *)p, n) 349 #define pmap_pte_setbits(p, b) \ 350 atomic_or_64((volatile uint64_t *)p, b) 351 #define pmap_pte_clearbits(p, b) \ 352 atomic_and_64((volatile uint64_t *)p, ~(b)) 353 #else /* PAE */ 354 #define pmap_pte_cas(p, o, n) atomic_cas_32((p), (o), (n)) 355 #define pmap_pte_testset(p, n) \ 356 atomic_swap_ulong((volatile unsigned long *)p, n) 357 #define pmap_pte_setbits(p, b) \ 358 atomic_or_ulong((volatile unsigned long *)p, b) 359 #define pmap_pte_clearbits(p, b) \ 360 atomic_and_ulong((volatile unsigned long *)p, ~(b)) 361 #endif /* PAE */ 362 363 #else /* XEN */ 364 static __inline pt_entry_t 365 pmap_pa2pte(paddr_t pa) 366 { 367 return (pt_entry_t)xpmap_ptom_masked(pa); 368 } 369 370 static __inline paddr_t 371 pmap_pte2pa(pt_entry_t pte) 372 { 373 return xpmap_mtop_masked(pte & PG_FRAME); 374 } 375 static __inline void 376 pmap_pte_set(pt_entry_t *pte, pt_entry_t npte) 377 { 378 int s = splvm(); 379 xpq_queue_pte_update(xpmap_ptetomach(pte), npte); 380 splx(s); 381 } 382 383 static __inline pt_entry_t 384 pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n) 385 { 386 int s = splvm(); 387 pt_entry_t opte = *ptep; 388 389 if (opte == o) { 390 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n); 391 xpq_flush_queue(); 392 } 393 splx(s); 394 return opte; 395 } 396 397 static __inline pt_entry_t 398 pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte) 399 { 400 int s = splvm(); 401 pt_entry_t opte = *pte; 402 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), 403 npte); 404 xpq_flush_queue(); 405 splx(s); 406 return opte; 407 } 408 409 static __inline void 410 pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits) 411 { 412 int s = splvm(); 413 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits); 414 xpq_flush_queue(); 415 splx(s); 416 } 417 418 static __inline void 419 pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits) 420 { 421 int s = splvm(); 422 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), 423 (*pte) & ~bits); 424 xpq_flush_queue(); 425 splx(s); 426 } 427 428 static __inline void 429 pmap_pte_flush(void) 430 { 431 int s = splvm(); 432 xpq_flush_queue(); 433 splx(s); 434 } 435 436 #endif 437 438 #ifdef PAE 439 /* Address of the static kernel's L2 page */ 440 pd_entry_t *pmap_kl2pd; 441 paddr_t pmap_kl2paddr; 442 #endif 443 444 445 struct trapframe; 446 447 int pmap_exec_fixup(struct vm_map *, struct trapframe *, struct pcb *); 448 void pmap_ldt_cleanup(struct lwp *); 449 450 #include <x86/pmap_pv.h> 451 452 #define __HAVE_VM_PAGE_MD 453 #define VM_MDPAGE_INIT(pg) \ 454 memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \ 455 PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp) 456 457 struct vm_page_md { 458 struct pmap_page mp_pp; 459 }; 460 461 #endif /* _I386_PMAP_H_ */ 462