1 /* $OpenBSD: pmap7.c,v 1.68 2024/11/07 08:12:12 miod Exp $ */ 2 /* $NetBSD: pmap.c,v 1.147 2004/01/18 13:03:50 scw Exp $ */ 3 4 /* 5 * Copyright 2003 Wasabi Systems, Inc. 6 * All rights reserved. 7 * 8 * Written by Steve C. Woodford for Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed for the NetBSD Project by 21 * Wasabi Systems, Inc. 22 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 23 * or promote products derived from this software without specific prior 24 * written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 41 * Copyright (c) 2001 Richard Earnshaw 42 * Copyright (c) 2001-2002 Christopher Gilbert 43 * All rights reserved. 44 * 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. The name of the company nor the name of the author may be used to 51 * endorse or promote products derived from this software without specific 52 * prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 55 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 56 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 57 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 58 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 59 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 60 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 */ 66 67 /*- 68 * Copyright (c) 1999 The NetBSD Foundation, Inc. 69 * All rights reserved. 70 * 71 * This code is derived from software contributed to The NetBSD Foundation 72 * by Charles M. Hannum. 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 83 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 84 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 85 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 86 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 87 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 88 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 89 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 90 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 91 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 92 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 93 * POSSIBILITY OF SUCH DAMAGE. 94 */ 95 96 /* 97 * Copyright (c) 1994-1998 Mark Brinicombe. 98 * Copyright (c) 1994 Brini. 99 * All rights reserved. 100 * 101 * This code is derived from software written for Brini by Mark Brinicombe 102 * 103 * Redistribution and use in source and binary forms, with or without 104 * modification, are permitted provided that the following conditions 105 * are met: 106 * 1. Redistributions of source code must retain the above copyright 107 * notice, this list of conditions and the following disclaimer. 108 * 2. Redistributions in binary form must reproduce the above copyright 109 * notice, this list of conditions and the following disclaimer in the 110 * documentation and/or other materials provided with the distribution. 111 * 3. All advertising materials mentioning features or use of this software 112 * must display the following acknowledgement: 113 * This product includes software developed by Mark Brinicombe. 114 * 4. The name of the author may not be used to endorse or promote products 115 * derived from this software without specific prior written permission. 116 * 117 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 118 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 119 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 120 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 121 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 122 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 123 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 124 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 125 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 126 * 127 * RiscBSD kernel project 128 * 129 * pmap.c 130 * 131 * Machine dependant vm stuff 132 * 133 * Created : 20/09/94 134 */ 135 136 /* 137 * Performance improvements, UVM changes, overhauls and part-rewrites 138 * were contributed by Neil A. Carson <neil@causality.com>. 139 */ 140 141 /* 142 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 143 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 144 * Systems, Inc. 145 * 146 * There are still a few things outstanding at this time: 147 * 148 * - There are some unresolved issues for MP systems: 149 * 150 * o The L1 metadata needs a lock, or more specifically, some places 151 * need to acquire an exclusive lock when modifying L1 translation 152 * table entries. 153 * 154 * o When one cpu modifies an L1 entry, and that L1 table is also 155 * being used by another cpu, then the latter will need to be told 156 * that a tlb invalidation may be necessary. (But only if the old 157 * domain number in the L1 entry being over-written is currently 158 * the active domain on that cpu). I guess there are lots more tlb 159 * shootdown issues too... 160 * 161 * o If the vector_page is at 0x00000000 instead of 0xffff0000, then 162 * MP systems will lose big-time because of the MMU domain hack. 163 * The only way this can be solved (apart from moving the vector 164 * page to 0xffff0000) is to reserve the first 1MB of user address 165 * space for kernel use only. This would require re-linking all 166 * applications so that the text section starts above this 1MB 167 * boundary. 168 * 169 * o Tracking which VM space is resident in the cache/tlb has not yet 170 * been implemented for MP systems. 171 * 172 * o Finally, there is a pathological condition where two cpus running 173 * two separate processes (not procs) which happen to share an L1 174 * can get into a fight over one or more L1 entries. This will result 175 * in a significant slow-down if both processes are in tight loops. 176 */ 177 178 #include <sys/param.h> 179 #include <sys/systm.h> 180 #include <sys/proc.h> 181 #include <sys/malloc.h> 182 #include <sys/user.h> 183 #include <sys/pool.h> 184 185 #include <uvm/uvm.h> 186 187 #include <machine/pmap.h> 188 #include <machine/pcb.h> 189 #include <machine/param.h> 190 #include <arm/cpufunc.h> 191 192 /* 193 * XXX We want to use proper TEX settings eventually. 194 */ 195 196 #define PTE_L1_S_CACHE_MODE (L1_S_B | L1_S_C) 197 #define PTE_L1_S_CACHE_MODE_PT (L1_S_B | L1_S_C) 198 199 /* write-allocate should be tested */ 200 #define PTE_L2_L_CACHE_MODE (L2_B | L2_C) 201 #define PTE_L2_S_CACHE_MODE (L2_B | L2_C) 202 203 #define PTE_L2_L_CACHE_MODE_PT (L2_B | L2_C) 204 #define PTE_L2_S_CACHE_MODE_PT (L2_B | L2_C) 205 206 //#define PMAP_DEBUG 207 #ifdef PMAP_DEBUG 208 209 /* 210 * for switching to potentially finer grained debugging 211 */ 212 #define PDB_FOLLOW 0x0001 213 #define PDB_INIT 0x0002 214 #define PDB_ENTER 0x0004 215 #define PDB_REMOVE 0x0008 216 #define PDB_CREATE 0x0010 217 #define PDB_PTPAGE 0x0020 218 #define PDB_GROWKERN 0x0040 219 #define PDB_BITS 0x0080 220 #define PDB_COLLECT 0x0100 221 #define PDB_PROTECT 0x0200 222 #define PDB_MAP_L1 0x0400 223 #define PDB_BOOTSTRAP 0x1000 224 #define PDB_PARANOIA 0x2000 225 #define PDB_WIRING 0x4000 226 #define PDB_PVDUMP 0x8000 227 #define PDB_KENTER 0x20000 228 #define PDB_KREMOVE 0x40000 229 230 #define pmapdebug (cold ? 0 : 0xffffffff) 231 #define NPDEBUG(_lev_,_stat_) \ 232 if (pmapdebug & (_lev_)) \ 233 ((_stat_)) 234 235 #else /* PMAP_DEBUG */ 236 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 237 #endif /* PMAP_DEBUG */ 238 239 /* 240 * pmap_kernel() points here 241 */ 242 struct pmap kernel_pmap_store; 243 244 /* 245 * Pool and cache that pmap structures are allocated from. 246 * We use a cache to avoid clearing the pm_l2[] array (1KB) 247 * in pmap_create(). 248 */ 249 struct pool pmap_pmap_pool; 250 251 /* 252 * Pool of PV structures 253 */ 254 struct pool pmap_pv_pool; 255 void *pmap_pv_page_alloc(struct pool *, int, int *); 256 void pmap_pv_page_free(struct pool *, void *); 257 struct pool_allocator pmap_pv_allocator = { 258 pmap_pv_page_alloc, pmap_pv_page_free 259 }; 260 261 /* 262 * Pool and cache of l2_dtable structures. 263 * We use a cache to avoid clearing the structures when they're 264 * allocated. (196 bytes) 265 */ 266 struct pool pmap_l2dtable_pool; 267 vaddr_t pmap_kernel_l2dtable_kva; 268 269 /* 270 * Pool and cache of L2 page descriptors. 271 * We use a cache to avoid clearing the descriptor table 272 * when they're allocated. (1KB) 273 */ 274 struct pool pmap_l2ptp_pool; 275 vaddr_t pmap_kernel_l2ptp_kva; 276 paddr_t pmap_kernel_l2ptp_phys; 277 278 /* 279 * pmap copy/zero page, wb page, and mem(5) hook point 280 */ 281 pt_entry_t *csrc_pte, *cdst_pte; 282 vaddr_t csrcp, cdstp; 283 char *memhook; 284 extern caddr_t msgbufaddr; 285 286 /* 287 * Flag to indicate if pmap_init() has done its thing 288 */ 289 int pmap_initialized; 290 291 /* 292 * Metadata for L1 translation tables. 293 */ 294 struct l1_ttable { 295 /* Entry on the L1 Table list */ 296 TAILQ_ENTRY(l1_ttable) l1_link; 297 298 /* Physical address of this L1 page table */ 299 paddr_t l1_physaddr; 300 301 /* KVA of this L1 page table */ 302 pd_entry_t *l1_kva; 303 }; 304 305 /* 306 * Convert a virtual address into its L1 table index. That is, the 307 * index used to locate the L2 descriptor table pointer in an L1 table. 308 * This is basically used to index l1->l1_kva[]. 309 * 310 * Each L2 descriptor table represents 1MB of VA space. 311 */ 312 #define L1_IDX(va) (((vaddr_t)(va)) >> L1_S_SHIFT) 313 314 /* 315 * Set if the PXN bit is supported. 316 */ 317 pd_entry_t l1_c_pxn; 318 319 /* 320 * A list of all L1 tables 321 */ 322 TAILQ_HEAD(, l1_ttable) l1_list; 323 324 /* 325 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 326 * 327 * This is normally 16MB worth L2 page descriptors for any given pmap. 328 * Reference counts are maintained for L2 descriptors so they can be 329 * freed when empty. 330 */ 331 struct l2_dtable { 332 /* The number of L2 page descriptors allocated to this l2_dtable */ 333 u_int l2_occupancy; 334 335 /* List of L2 page descriptors */ 336 struct l2_bucket { 337 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 338 paddr_t l2b_phys; /* Physical address of same */ 339 u_short l2b_l1idx; /* This L2 table's L1 index */ 340 u_short l2b_occupancy; /* How many active descriptors */ 341 } l2_bucket[L2_BUCKET_SIZE]; 342 }; 343 344 /* 345 * Given an L1 table index, calculate the corresponding l2_dtable index 346 * and bucket index within the l2_dtable. 347 */ 348 #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 349 (L2_SIZE - 1)) 350 #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 351 352 /* 353 * Given a virtual address, this macro returns the 354 * virtual address required to drop into the next L2 bucket. 355 */ 356 #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 357 358 /* 359 * L2 allocation. 360 */ 361 #define pmap_alloc_l2_dtable() \ 362 pool_get(&pmap_l2dtable_pool, PR_NOWAIT|PR_ZERO) 363 #define pmap_free_l2_dtable(l2) \ 364 pool_put(&pmap_l2dtable_pool, (l2)) 365 366 /* 367 * We try to map the page tables write-through, if possible. However, not 368 * all CPUs have a write-through cache mode, so on those we have to sync 369 * the cache when we frob page tables. 370 * 371 * We try to evaluate this at compile time, if possible. However, it's 372 * not always possible to do that, hence this run-time var. 373 */ 374 int pmap_needs_pte_sync; 375 376 /* 377 * Real definition of pv_entry. 378 */ 379 struct pv_entry { 380 struct pv_entry *pv_next; /* next pv_entry */ 381 pmap_t pv_pmap; /* pmap where mapping lies */ 382 vaddr_t pv_va; /* virtual address for mapping */ 383 u_int pv_flags; /* flags */ 384 }; 385 386 /* 387 * Macro to determine if a mapping might be resident in the 388 * instruction cache and/or TLB 389 */ 390 #define PV_BEEN_EXECD(f) (((f) & PVF_EXEC) != 0) 391 392 /* 393 * Local prototypes 394 */ 395 void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 396 pt_entry_t **); 397 static int pmap_is_current(pmap_t); 398 void pmap_enter_pv(struct vm_page *, struct pv_entry *, 399 pmap_t, vaddr_t, u_int); 400 static struct pv_entry *pmap_find_pv(struct vm_page *, pmap_t, vaddr_t); 401 struct pv_entry *pmap_remove_pv(struct vm_page *, pmap_t, vaddr_t); 402 u_int pmap_modify_pv(struct vm_page *, pmap_t, vaddr_t, 403 u_int, u_int); 404 405 void pmap_alloc_l1(pmap_t); 406 void pmap_free_l1(pmap_t); 407 408 struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 409 struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 410 void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 411 412 void pmap_clearbit(struct vm_page *, u_int); 413 void pmap_clean_page(struct vm_page *); 414 void pmap_page_remove(struct vm_page *); 415 416 void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 417 vaddr_t kernel_pt_lookup(paddr_t); 418 419 420 /* 421 * External function prototypes 422 */ 423 extern void bzero_page(vaddr_t); 424 extern void bcopy_page(vaddr_t, vaddr_t); 425 426 /* 427 * Misc variables 428 */ 429 vaddr_t virtual_avail; 430 vaddr_t virtual_end; 431 vaddr_t pmap_curmaxkvaddr; 432 433 extern pv_addr_t systempage; 434 435 static __inline int 436 pmap_is_current(pmap_t pm) 437 { 438 if (pm == pmap_kernel() || 439 (curproc && curproc->p_vmspace->vm_map.pmap == pm)) 440 return 1; 441 442 return 0; 443 } 444 445 /* 446 * A bunch of routines to conditionally flush the caches/TLB depending 447 * on whether the specified pmap actually needs to be flushed at any 448 * given time. 449 */ 450 static __inline void 451 pmap_tlb_flushID_SE(pmap_t pm, vaddr_t va) 452 { 453 if (pmap_is_current(pm)) 454 cpu_tlb_flushID_SE(va); 455 } 456 457 static __inline void 458 pmap_tlb_flushID(pmap_t pm) 459 { 460 if (pmap_is_current(pm)) 461 cpu_tlb_flushID(); 462 } 463 464 /* 465 * Returns a pointer to the L2 bucket associated with the specified pmap 466 * and VA, or NULL if no L2 bucket exists for the address. 467 */ 468 struct l2_bucket * 469 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 470 { 471 struct l2_dtable *l2; 472 struct l2_bucket *l2b; 473 u_short l1idx; 474 475 l1idx = L1_IDX(va); 476 477 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 478 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 479 return (NULL); 480 481 return (l2b); 482 } 483 484 /* 485 * main pv_entry manipulation functions: 486 * pmap_enter_pv: enter a mapping onto a vm_page list 487 * pmap_remove_pv: remove a mapping from a vm_page list 488 * 489 * NOTE: pmap_enter_pv expects to lock the pvh itself 490 * pmap_remove_pv expects te caller to lock the pvh before calling 491 */ 492 493 /* 494 * pmap_enter_pv: enter a mapping onto a vm_page lst 495 * 496 * => caller should have pmap locked 497 * => we will gain the lock on the vm_page and allocate the new pv_entry 498 * => caller should adjust ptp's wire_count before calling 499 * => caller should not adjust pmap's wire_count 500 */ 501 void 502 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 503 vaddr_t va, u_int flags) 504 { 505 506 NPDEBUG(PDB_PVDUMP, 507 printf("pmap_enter_pv: pm %p, pg %p, flags 0x%x\n", pm, pg, flags)); 508 509 pve->pv_pmap = pm; 510 pve->pv_va = va; 511 pve->pv_flags = flags; 512 513 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 514 pg->mdpage.pvh_list = pve; /* ... locked list */ 515 pg->mdpage.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 516 517 if (pve->pv_flags & PVF_WIRED) 518 ++pm->pm_stats.wired_count; 519 } 520 521 /* 522 * 523 * pmap_find_pv: Find a pv entry 524 * 525 * => caller should hold lock on vm_page 526 */ 527 static __inline struct pv_entry * 528 pmap_find_pv(struct vm_page *pg, pmap_t pm, vaddr_t va) 529 { 530 struct pv_entry *pv; 531 532 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 533 if (pm == pv->pv_pmap && va == pv->pv_va) 534 break; 535 } 536 537 return (pv); 538 } 539 540 /* 541 * pmap_remove_pv: try to remove a mapping from a pv_list 542 * 543 * => pmap should be locked 544 * => caller should hold lock on vm_page [so that attrs can be adjusted] 545 * => caller should adjust ptp's wire_count and free PTP if needed 546 * => caller should NOT adjust pmap's wire_count 547 * => we return the removed pve 548 */ 549 struct pv_entry * 550 pmap_remove_pv(struct vm_page *pg, pmap_t pm, vaddr_t va) 551 { 552 struct pv_entry *pve, **prevptr; 553 554 NPDEBUG(PDB_PVDUMP, 555 printf("pmap_remove_pv: pm %p, pg %p, va 0x%08lx\n", pm, pg, va)); 556 557 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 558 pve = *prevptr; 559 560 while (pve) { 561 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 562 NPDEBUG(PDB_PVDUMP, 563 printf("pmap_remove_pv: pm %p, pg %p, flags 0x%x\n", pm, pg, pve->pv_flags)); 564 *prevptr = pve->pv_next; /* remove it! */ 565 if (pve->pv_flags & PVF_WIRED) 566 --pm->pm_stats.wired_count; 567 break; 568 } 569 prevptr = &pve->pv_next; /* previous pointer */ 570 pve = pve->pv_next; /* advance */ 571 } 572 573 return(pve); /* return removed pve */ 574 } 575 576 /* 577 * 578 * pmap_modify_pv: Update pv flags 579 * 580 * => caller should hold lock on vm_page [so that attrs can be adjusted] 581 * => caller should NOT adjust pmap's wire_count 582 * => we return the old flags 583 * 584 * Modify a physical-virtual mapping in the pv table 585 */ 586 u_int 587 pmap_modify_pv(struct vm_page *pg, pmap_t pm, vaddr_t va, 588 u_int clr_mask, u_int set_mask) 589 { 590 struct pv_entry *npv; 591 u_int flags, oflags; 592 593 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 594 return (0); 595 596 NPDEBUG(PDB_PVDUMP, 597 printf("pmap_modify_pv: pm %p, pg %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, pg, clr_mask, set_mask, npv->pv_flags)); 598 599 /* 600 * There is at least one VA mapping this page. 601 */ 602 603 if (clr_mask & (PVF_REF | PVF_MOD)) 604 pg->mdpage.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 605 606 oflags = npv->pv_flags; 607 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 608 609 if ((flags ^ oflags) & PVF_WIRED) { 610 if (flags & PVF_WIRED) 611 ++pm->pm_stats.wired_count; 612 else 613 --pm->pm_stats.wired_count; 614 } 615 616 return (oflags); 617 } 618 619 uint nl1; 620 /* 621 * Allocate an L1 translation table for the specified pmap. 622 * This is called at pmap creation time. 623 */ 624 void 625 pmap_alloc_l1(pmap_t pm) 626 { 627 struct l1_ttable *l1; 628 struct pglist plist; 629 struct vm_page *m; 630 pd_entry_t *pl1pt; 631 vaddr_t va, eva; 632 int error; 633 634 #ifdef PMAP_DEBUG 635 printf("%s: %d\n", __func__, ++nl1); 636 #endif 637 /* XXX use a pool? or move to inside struct pmap? */ 638 l1 = malloc(sizeof(*l1), M_VMPMAP, M_WAITOK); 639 640 /* Allocate a L1 page table */ 641 for (;;) { 642 va = (vaddr_t)km_alloc(L1_TABLE_SIZE, &kv_any, &kp_none, 643 &kd_nowait); 644 if (va != 0) 645 break; 646 uvm_wait("alloc_l1_va"); 647 } 648 649 for (;;) { 650 TAILQ_INIT(&plist); 651 error = uvm_pglistalloc(L1_TABLE_SIZE, 0, (paddr_t)-1, 652 L1_TABLE_SIZE, 0, &plist, 1, UVM_PLA_WAITOK); 653 if (error == 0) 654 break; 655 uvm_wait("alloc_l1_pg"); 656 } 657 658 pl1pt = (pd_entry_t *)va; 659 m = TAILQ_FIRST(&plist); 660 for (eva = va + L1_TABLE_SIZE; va < eva; va += PAGE_SIZE) { 661 paddr_t pa = VM_PAGE_TO_PHYS(m); 662 663 pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE); 664 m = TAILQ_NEXT(m, pageq); 665 } 666 667 pmap_init_l1(l1, pl1pt); 668 669 pm->pm_l1 = l1; 670 } 671 672 /* 673 * Free an L1 translation table. 674 * This is called at pmap destruction time. 675 */ 676 void 677 pmap_free_l1(pmap_t pm) 678 { 679 struct l1_ttable *l1 = pm->pm_l1; 680 struct pglist mlist; 681 struct vm_page *pg; 682 struct l2_bucket *l2b; 683 pt_entry_t *ptep; 684 vaddr_t va; 685 uint npg; 686 687 pm->pm_l1 = NULL; 688 TAILQ_REMOVE(&l1_list, l1, l1_link); 689 690 /* free backing pages */ 691 TAILQ_INIT(&mlist); 692 va = (vaddr_t)l1->l1_kva; 693 for (npg = atop(L1_TABLE_SIZE); npg != 0; npg--) { 694 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 695 ptep = &l2b->l2b_kva[l2pte_index(va)]; 696 pg = PHYS_TO_VM_PAGE(l2pte_pa(*ptep)); 697 TAILQ_INSERT_TAIL(&mlist, pg, pageq); 698 va += PAGE_SIZE; 699 } 700 pmap_kremove((vaddr_t)l1->l1_kva, L1_TABLE_SIZE); 701 uvm_pglistfree(&mlist); 702 703 /* free backing va */ 704 km_free(l1->l1_kva, L1_TABLE_SIZE, &kv_any, &kp_none); 705 706 free(l1, M_VMPMAP, 0); 707 } 708 709 /* 710 * void pmap_free_l2_ptp(pt_entry_t *) 711 * 712 * Free an L2 descriptor table. 713 */ 714 static __inline void 715 pmap_free_l2_ptp(pt_entry_t *l2) 716 { 717 pool_put(&pmap_l2ptp_pool, (void *)l2); 718 } 719 720 /* 721 * Returns a pointer to the L2 bucket associated with the specified pmap 722 * and VA. 723 * 724 * If no L2 bucket exists, perform the necessary allocations to put an L2 725 * bucket/page table in place. 726 * 727 * Note that if a new L2 bucket/page was allocated, the caller *must* 728 * increment the bucket occupancy counter appropriately *before* 729 * releasing the pmap's lock to ensure no other thread or cpu deallocates 730 * the bucket/page in the meantime. 731 */ 732 struct l2_bucket * 733 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 734 { 735 struct l2_dtable *l2; 736 struct l2_bucket *l2b; 737 u_short l1idx; 738 739 l1idx = L1_IDX(va); 740 741 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 742 /* 743 * No mapping at this address, as there is 744 * no entry in the L1 table. 745 * Need to allocate a new l2_dtable. 746 */ 747 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 748 return (NULL); 749 750 /* 751 * Link it into the parent pmap 752 */ 753 pm->pm_l2[L2_IDX(l1idx)] = l2; 754 } 755 756 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 757 758 /* 759 * Fetch pointer to the L2 page table associated with the address. 760 */ 761 if (l2b->l2b_kva == NULL) { 762 pt_entry_t *ptep; 763 764 /* 765 * No L2 page table has been allocated. Chances are, this 766 * is because we just allocated the l2_dtable, above. 767 */ 768 ptep = pool_get(&pmap_l2ptp_pool, PR_NOWAIT|PR_ZERO); 769 if (ptep == NULL) { 770 /* 771 * Oops, no more L2 page tables available at this 772 * time. We may need to deallocate the l2_dtable 773 * if we allocated a new one above. 774 */ 775 if (l2->l2_occupancy == 0) { 776 pm->pm_l2[L2_IDX(l1idx)] = NULL; 777 pmap_free_l2_dtable(l2); 778 } 779 return (NULL); 780 } 781 PTE_SYNC_RANGE(ptep, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 782 pmap_extract(pmap_kernel(), (vaddr_t)ptep, &l2b->l2b_phys); 783 784 l2->l2_occupancy++; 785 l2b->l2b_kva = ptep; 786 l2b->l2b_l1idx = l1idx; 787 } 788 789 return (l2b); 790 } 791 792 /* 793 * One or more mappings in the specified L2 descriptor table have just been 794 * invalidated. 795 * 796 * Garbage collect the metadata and descriptor table itself if necessary. 797 * 798 * The pmap lock must be acquired when this is called (not necessary 799 * for the kernel pmap). 800 */ 801 void 802 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 803 { 804 struct l2_dtable *l2; 805 pd_entry_t *pl1pd; 806 pt_entry_t *ptep; 807 u_short l1idx; 808 809 KDASSERT(count <= l2b->l2b_occupancy); 810 811 /* 812 * Update the bucket's reference count according to how many 813 * PTEs the caller has just invalidated. 814 */ 815 l2b->l2b_occupancy -= count; 816 817 /* 818 * Note: 819 * 820 * Level 2 page tables allocated to the kernel pmap are never freed 821 * as that would require checking all Level 1 page tables and 822 * removing any references to the Level 2 page table. See also the 823 * comment elsewhere about never freeing bootstrap L2 descriptors. 824 * 825 * We make do with just invalidating the mapping in the L2 table. 826 * 827 * This isn't really a big deal in practice and, in fact, leads 828 * to a performance win over time as we don't need to continually 829 * alloc/free. 830 */ 831 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 832 return; 833 834 /* 835 * There are no more valid mappings in this level 2 page table. 836 * Go ahead and NULL-out the pointer in the bucket, then 837 * free the page table. 838 */ 839 l1idx = l2b->l2b_l1idx; 840 ptep = l2b->l2b_kva; 841 l2b->l2b_kva = NULL; 842 843 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 844 845 /* 846 * Invalidate the L1 slot. 847 */ 848 *pl1pd = L1_TYPE_INV; 849 PTE_SYNC(pl1pd); 850 pmap_tlb_flushID_SE(pm, l1idx << L1_S_SHIFT); 851 852 /* 853 * Release the L2 descriptor table back to the pool cache. 854 */ 855 pmap_free_l2_ptp(ptep); 856 857 /* 858 * Update the reference count in the associated l2_dtable 859 */ 860 l2 = pm->pm_l2[L2_IDX(l1idx)]; 861 if (--l2->l2_occupancy > 0) 862 return; 863 864 /* 865 * There are no more valid mappings in any of the Level 1 866 * slots managed by this l2_dtable. Go ahead and NULL-out 867 * the pointer in the parent pmap and free the l2_dtable. 868 */ 869 pm->pm_l2[L2_IDX(l1idx)] = NULL; 870 pmap_free_l2_dtable(l2); 871 } 872 873 /* 874 * Modify pte bits for all ptes corresponding to the given physical address. 875 * We use `maskbits' rather than `clearbits' because we're always passing 876 * constants and the latter would require an extra inversion at run-time. 877 */ 878 void 879 pmap_clearbit(struct vm_page *pg, u_int maskbits) 880 { 881 struct l2_bucket *l2b; 882 struct pv_entry *pv; 883 pt_entry_t *ptep, npte, opte; 884 pmap_t pm; 885 vaddr_t va; 886 u_int oflags; 887 888 NPDEBUG(PDB_BITS, 889 printf("pmap_clearbit: pg %p (0x%08lx) mask 0x%x\n", 890 pg, pg->phys_addr, maskbits)); 891 892 /* 893 * Clear saved attributes (modify, reference) 894 */ 895 pg->mdpage.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 896 897 if (pg->mdpage.pvh_list == NULL) 898 return; 899 900 /* 901 * Loop over all current mappings setting/clearing as appropriate 902 */ 903 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 904 va = pv->pv_va; 905 pm = pv->pv_pmap; 906 oflags = pv->pv_flags; 907 pv->pv_flags &= ~maskbits; 908 909 l2b = pmap_get_l2_bucket(pm, va); 910 KDASSERT(l2b != NULL); 911 912 ptep = &l2b->l2b_kva[l2pte_index(va)]; 913 npte = opte = *ptep; 914 NPDEBUG(PDB_BITS, 915 printf( 916 "pmap_clearbit: pv %p, pm %p, va 0x%08lx, flag 0x%x\n", 917 pv, pv->pv_pmap, pv->pv_va, oflags)); 918 919 if (maskbits & (PVF_WRITE|PVF_MOD)) { 920 /* Disable write access. */ 921 npte |= L2_V7_AP(0x4); 922 } 923 924 if (maskbits & PVF_REF) { 925 /* 926 * Clear the Access Flag such that we will 927 * take a page fault the next time the mapping 928 * is referenced. 929 */ 930 npte &= ~L2_V7_AF; 931 } 932 933 if (npte != opte) { 934 *ptep = npte; 935 PTE_SYNC(ptep); 936 /* Flush the TLB entry if a current pmap. */ 937 if (opte & L2_V7_AF) 938 pmap_tlb_flushID_SE(pm, pv->pv_va); 939 } 940 941 NPDEBUG(PDB_BITS, 942 printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n", 943 pm, va, opte, npte)); 944 } 945 } 946 947 /* 948 * pmap_clean_page() 949 * 950 * Invalidate all I$ aliases for a single page. 951 */ 952 void 953 pmap_clean_page(struct vm_page *pg) 954 { 955 pmap_t pm; 956 struct pv_entry *pv; 957 958 if (curproc) 959 pm = curproc->p_vmspace->vm_map.pmap; 960 else 961 pm = pmap_kernel(); 962 963 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 964 /* inline !pmap_is_current(pv->pv_pmap) */ 965 if (pv->pv_pmap != pmap_kernel() && pv->pv_pmap != pm) 966 continue; 967 968 if (PV_BEEN_EXECD(pv->pv_flags)) 969 cpu_icache_sync_range(pv->pv_va, PAGE_SIZE); 970 } 971 } 972 973 /* 974 * Routine: pmap_page_remove 975 * Function: 976 * Removes this physical page from 977 * all physical maps in which it resides. 978 * Reflects back modify bits to the pager. 979 */ 980 void 981 pmap_page_remove(struct vm_page *pg) 982 { 983 struct l2_bucket *l2b; 984 struct pv_entry *pv, *npv; 985 pmap_t pm, curpm; 986 pt_entry_t *ptep, opte; 987 int flush; 988 989 NPDEBUG(PDB_FOLLOW, 990 printf("pmap_page_remove: pg %p (0x%08lx)\n", pg, pg->phys_addr)); 991 992 pv = pg->mdpage.pvh_list; 993 if (pv == NULL) 994 return; 995 996 flush = 0; 997 if (curproc) 998 curpm = curproc->p_vmspace->vm_map.pmap; 999 else 1000 curpm = pmap_kernel(); 1001 1002 while (pv) { 1003 pm = pv->pv_pmap; 1004 1005 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 1006 KDASSERT(l2b != NULL); 1007 1008 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1009 opte = *ptep; 1010 if (opte != L2_TYPE_INV) { 1011 /* inline pmap_is_current(pm) */ 1012 if ((opte & L2_V7_AF) && 1013 (pm == curpm || pm == pmap_kernel())) { 1014 if (PV_BEEN_EXECD(pv->pv_flags)) 1015 cpu_icache_sync_range(pv->pv_va, PAGE_SIZE); 1016 flush = 1; 1017 } 1018 1019 /* 1020 * Update statistics 1021 */ 1022 --pm->pm_stats.resident_count; 1023 1024 /* Wired bit */ 1025 if (pv->pv_flags & PVF_WIRED) 1026 --pm->pm_stats.wired_count; 1027 1028 /* 1029 * Invalidate the PTEs. 1030 */ 1031 *ptep = L2_TYPE_INV; 1032 PTE_SYNC(ptep); 1033 if (flush) 1034 cpu_tlb_flushID_SE(pv->pv_va); 1035 1036 pmap_free_l2_bucket(pm, l2b, 1); 1037 } 1038 1039 npv = pv->pv_next; 1040 pool_put(&pmap_pv_pool, pv); 1041 pv = npv; 1042 } 1043 pg->mdpage.pvh_list = NULL; 1044 } 1045 1046 /* 1047 * pmap_t pmap_create(void) 1048 * 1049 * Create a new pmap structure from scratch. 1050 */ 1051 pmap_t 1052 pmap_create(void) 1053 { 1054 pmap_t pm; 1055 1056 pm = pool_get(&pmap_pmap_pool, PR_WAITOK|PR_ZERO); 1057 1058 pm->pm_refs = 1; 1059 pm->pm_stats.wired_count = 0; 1060 pmap_alloc_l1(pm); 1061 1062 return (pm); 1063 } 1064 1065 /* 1066 * void pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 1067 * int flags) 1068 * 1069 * Insert the given physical page (p) at 1070 * the specified virtual address (v) in the 1071 * target physical map with the protection requested. 1072 * 1073 * NB: This is the only routine which MAY NOT lazy-evaluate 1074 * or lose information. That is, this routine must actually 1075 * insert this page into the given map NOW. 1076 */ 1077 int 1078 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 1079 { 1080 struct l2_bucket *l2b; 1081 struct vm_page *pg, *opg; 1082 struct pv_entry *pve; 1083 pt_entry_t *ptep, npte, opte; 1084 u_int nflags; 1085 u_int oflags; 1086 int mapped = 1; 1087 1088 NPDEBUG(PDB_ENTER, printf("pmap_enter: pm %p va 0x%lx pa 0x%lx prot %x flag %x\n", pm, va, pa, prot, flags)); 1089 1090 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & PROT_MASK) != 0); 1091 KDASSERT(((va | pa) & PGOFSET) == 0); 1092 1093 /* 1094 * Get a pointer to the page. Later on in this function, we 1095 * test for a managed page by checking pg != NULL. 1096 */ 1097 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 1098 1099 nflags = 0; 1100 if (prot & PROT_WRITE) 1101 nflags |= PVF_WRITE; 1102 if (prot & PROT_EXEC) 1103 nflags |= PVF_EXEC; 1104 if (flags & PMAP_WIRED) 1105 nflags |= PVF_WIRED; 1106 1107 /* 1108 * Fetch the L2 bucket which maps this page, allocating one if 1109 * necessary for user pmaps. 1110 */ 1111 if (pm == pmap_kernel()) 1112 l2b = pmap_get_l2_bucket(pm, va); 1113 else 1114 l2b = pmap_alloc_l2_bucket(pm, va); 1115 if (l2b == NULL) { 1116 if (flags & PMAP_CANFAIL) 1117 return (ENOMEM); 1118 1119 panic("pmap_enter: failed to allocate L2 bucket"); 1120 } 1121 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1122 opte = *ptep; 1123 npte = L2_S_PROTO | pa; 1124 1125 if (opte != L2_TYPE_INV) { 1126 /* 1127 * There is already a mapping at this address. 1128 * If the physical address is different, lookup the 1129 * vm_page. 1130 */ 1131 if (l2pte_pa(opte) != pa) 1132 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 1133 else 1134 opg = pg; 1135 } else 1136 opg = NULL; 1137 1138 if (pg) { 1139 /* 1140 * This has to be a managed mapping. 1141 */ 1142 if ((flags & PROT_MASK) || 1143 (pg->mdpage.pvh_attrs & PVF_REF)) { 1144 /* 1145 * - The access type indicates that we don't need 1146 * to do referenced emulation. 1147 * OR 1148 * - The physical page has already been referenced 1149 * so no need to re-do referenced emulation here. 1150 */ 1151 nflags |= PVF_REF; 1152 npte |= L2_V7_AF; 1153 1154 if ((flags & PROT_WRITE) || 1155 (pg->mdpage.pvh_attrs & PVF_MOD)) { 1156 /* 1157 * This is a writable mapping, and the 1158 * page's mod state indicates it has 1159 * already been modified. Make it 1160 * writable from the outset. 1161 */ 1162 nflags |= PVF_MOD; 1163 } else { 1164 prot &= ~PROT_WRITE; 1165 } 1166 } else { 1167 /* 1168 * Need to do page referenced emulation. 1169 */ 1170 prot &= ~PROT_WRITE; 1171 mapped = 0; 1172 } 1173 1174 npte |= PTE_L2_S_CACHE_MODE; 1175 1176 if (pg == opg) { 1177 /* 1178 * We're changing the attrs of an existing mapping. 1179 */ 1180 oflags = pmap_modify_pv(pg, pm, va, 1181 PVF_WRITE | PVF_EXEC | PVF_WIRED | 1182 PVF_MOD | PVF_REF, nflags); 1183 } else { 1184 /* 1185 * New mapping, or changing the backing page 1186 * of an existing mapping. 1187 */ 1188 if (opg) { 1189 /* 1190 * Replacing an existing mapping with a new one. 1191 * It is part of our managed memory so we 1192 * must remove it from the PV list 1193 */ 1194 pve = pmap_remove_pv(opg, pm, va); 1195 } else 1196 if ((pve = pool_get(&pmap_pv_pool, PR_NOWAIT)) == NULL){ 1197 if ((flags & PMAP_CANFAIL) == 0) 1198 panic("pmap_enter: no pv entries"); 1199 1200 if (pm != pmap_kernel()) 1201 pmap_free_l2_bucket(pm, l2b, 0); 1202 1203 NPDEBUG(PDB_ENTER, 1204 printf("pmap_enter: ENOMEM\n")); 1205 return (ENOMEM); 1206 } 1207 1208 pmap_enter_pv(pg, pve, pm, va, nflags); 1209 } 1210 } else { 1211 /* 1212 * We're mapping an unmanaged page. 1213 * These are always readable, and possibly writable, from 1214 * the get go as we don't need to track ref/mod status. 1215 */ 1216 npte |= L2_V7_AF; 1217 1218 if (opg) { 1219 /* 1220 * Looks like there's an existing 'managed' mapping 1221 * at this address. 1222 */ 1223 pve = pmap_remove_pv(opg, pm, va); 1224 pool_put(&pmap_pv_pool, pve); 1225 } 1226 } 1227 1228 /* 1229 * Make sure userland mappings get the right permissions 1230 */ 1231 npte |= L2_S_PROT(pm == pmap_kernel() ? PTE_KERNEL : PTE_USER, prot); 1232 1233 /* 1234 * Keep the stats up to date 1235 */ 1236 if (opte == L2_TYPE_INV) { 1237 l2b->l2b_occupancy++; 1238 pm->pm_stats.resident_count++; 1239 } 1240 1241 NPDEBUG(PDB_ENTER, 1242 printf("pmap_enter: opte 0x%08x npte 0x%08x\n", opte, npte)); 1243 1244 /* 1245 * If this is just a wiring change, the two PTEs will be 1246 * identical, so there's no need to update the page table. 1247 */ 1248 if (npte != opte) { 1249 *ptep = npte; 1250 /* 1251 * We only need to frob the cache/tlb if this pmap 1252 * is current 1253 */ 1254 PTE_SYNC(ptep); 1255 if (npte & L2_V7_AF) { 1256 /* 1257 * This mapping is likely to be accessed as 1258 * soon as we return to userland. Fix up the 1259 * L1 entry to avoid taking another page fault. 1260 */ 1261 pd_entry_t *pl1pd, l1pd; 1262 1263 pl1pd = &pm->pm_l1->l1_kva[L1_IDX(va)]; 1264 l1pd = L1_C_PROTO | l2b->l2b_phys | l1_c_pxn; 1265 if (*pl1pd != l1pd) { 1266 *pl1pd = l1pd; 1267 PTE_SYNC(pl1pd); 1268 } 1269 } 1270 1271 if (opte & L2_V7_AF) 1272 pmap_tlb_flushID_SE(pm, va); 1273 } 1274 1275 /* 1276 * Make sure executable pages do not have stale data in I$, 1277 * which is VIPT. 1278 */ 1279 if (mapped && (prot & PROT_EXEC) != 0 && pmap_is_current(pm)) 1280 cpu_icache_sync_range(va, PAGE_SIZE); 1281 1282 return (0); 1283 } 1284 1285 /* 1286 * pmap_remove() 1287 * 1288 * pmap_remove is responsible for nuking a number of mappings for a range 1289 * of virtual address space in the current pmap. 1290 */ 1291 1292 void 1293 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 1294 { 1295 struct l2_bucket *l2b; 1296 vaddr_t next_bucket; 1297 pt_entry_t *ptep; 1298 u_int mappings; 1299 1300 NPDEBUG(PDB_REMOVE, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 1301 pm, sva, eva)); 1302 1303 while (sva < eva) { 1304 /* 1305 * Do one L2 bucket's worth at a time. 1306 */ 1307 next_bucket = L2_NEXT_BUCKET(sva); 1308 if (next_bucket > eva) 1309 next_bucket = eva; 1310 1311 l2b = pmap_get_l2_bucket(pm, sva); 1312 if (l2b == NULL) { 1313 sva = next_bucket; 1314 continue; 1315 } 1316 1317 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 1318 mappings = 0; 1319 1320 while (sva < next_bucket) { 1321 struct vm_page *pg; 1322 pt_entry_t pte; 1323 paddr_t pa; 1324 1325 pte = *ptep; 1326 1327 if (pte == L2_TYPE_INV) { 1328 /* 1329 * Nothing here, move along 1330 */ 1331 sva += PAGE_SIZE; 1332 ptep++; 1333 continue; 1334 } 1335 1336 pm->pm_stats.resident_count--; 1337 pa = l2pte_pa(pte); 1338 1339 /* 1340 * Update flags. In a number of circumstances, 1341 * we could cluster a lot of these and do a 1342 * number of sequential pages in one go. 1343 */ 1344 pg = PHYS_TO_VM_PAGE(pa); 1345 if (pg != NULL) { 1346 struct pv_entry *pve; 1347 pve = pmap_remove_pv(pg, pm, sva); 1348 if (pve != NULL) 1349 pool_put(&pmap_pv_pool, pve); 1350 } 1351 1352 /* 1353 * If the cache is physically indexed, we need 1354 * to flush any changes to the page before it 1355 * gets invalidated. 1356 */ 1357 if (pg != NULL) 1358 pmap_clean_page(pg); 1359 1360 *ptep = L2_TYPE_INV; 1361 PTE_SYNC(ptep); 1362 if (pte & L2_V7_AF) 1363 pmap_tlb_flushID_SE(pm, sva); 1364 1365 sva += PAGE_SIZE; 1366 ptep++; 1367 mappings++; 1368 } 1369 1370 /* 1371 * Deal with any left overs 1372 */ 1373 if (!pmap_is_current(pm)) 1374 cpu_idcache_wbinv_all(); 1375 1376 pmap_free_l2_bucket(pm, l2b, mappings); 1377 } 1378 } 1379 1380 /* 1381 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 1382 * 1383 * We assume there is already sufficient KVM space available 1384 * to do this, as we can't allocate L2 descriptor tables/metadata 1385 * from here. 1386 */ 1387 void 1388 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 1389 { 1390 struct l2_bucket *l2b; 1391 pt_entry_t *ptep, opte, npte; 1392 pt_entry_t cache_mode = PTE_L2_S_CACHE_MODE; 1393 1394 NPDEBUG(PDB_KENTER, 1395 printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n", 1396 va, pa, prot)); 1397 1398 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1399 KDASSERT(l2b != NULL); 1400 1401 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1402 opte = *ptep; 1403 1404 if (opte == L2_TYPE_INV) 1405 l2b->l2b_occupancy++; 1406 1407 if (pa & PMAP_DEVICE) 1408 cache_mode = L2_B | L2_V7_S_XN; 1409 else if (pa & PMAP_NOCACHE) 1410 cache_mode = L2_V7_S_TEX(1); 1411 1412 npte = L2_S_PROTO | (pa & PMAP_PA_MASK) | L2_V7_AF | 1413 L2_S_PROT(PTE_KERNEL, prot) | cache_mode; 1414 *ptep = npte; 1415 PTE_SYNC(ptep); 1416 if (opte & L2_V7_AF) 1417 cpu_tlb_flushD_SE(va); 1418 1419 if (pa & PMAP_NOCACHE) { 1420 cpu_dcache_wbinv_range(va, PAGE_SIZE); 1421 cpu_sdcache_wbinv_range(va, (pa & PMAP_PA_MASK), PAGE_SIZE); 1422 } 1423 } 1424 1425 void 1426 pmap_kenter_cache(vaddr_t va, paddr_t pa, vm_prot_t prot, int cacheable) 1427 { 1428 if (cacheable == 0) 1429 pa |= PMAP_NOCACHE; 1430 pmap_kenter_pa(va, pa, prot); 1431 } 1432 1433 void 1434 pmap_kremove(vaddr_t va, vsize_t len) 1435 { 1436 struct l2_bucket *l2b; 1437 pt_entry_t *ptep, *sptep, opte; 1438 vaddr_t next_bucket, eva; 1439 u_int mappings; 1440 1441 NPDEBUG(PDB_KREMOVE, printf("pmap_kremove: va 0x%08lx, len 0x%08lx\n", 1442 va, len)); 1443 1444 eva = va + len; 1445 1446 while (va < eva) { 1447 next_bucket = L2_NEXT_BUCKET(va); 1448 if (next_bucket > eva) 1449 next_bucket = eva; 1450 1451 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1452 KDASSERT(l2b != NULL); 1453 1454 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 1455 mappings = 0; 1456 1457 while (va < next_bucket) { 1458 opte = *ptep; 1459 if (opte != L2_TYPE_INV) { 1460 *ptep = L2_TYPE_INV; 1461 PTE_SYNC(ptep); 1462 mappings++; 1463 } 1464 if (opte & L2_V7_AF) 1465 cpu_tlb_flushD_SE(va); 1466 va += PAGE_SIZE; 1467 ptep++; 1468 } 1469 KDASSERT(mappings <= l2b->l2b_occupancy); 1470 l2b->l2b_occupancy -= mappings; 1471 } 1472 } 1473 1474 int 1475 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 1476 { 1477 struct l2_dtable *l2; 1478 pd_entry_t *pl1pd, l1pd; 1479 pt_entry_t *ptep, pte; 1480 paddr_t pa; 1481 u_int l1idx; 1482 1483 1484 l1idx = L1_IDX(va); 1485 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1486 l1pd = *pl1pd; 1487 1488 if (l1pte_section_p(l1pd)) { 1489 /* 1490 * These should only happen for pmap_kernel() 1491 */ 1492 KDASSERT(pm == pmap_kernel()); 1493 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 1494 } else { 1495 /* 1496 * Note that we can't rely on the validity of the L1 1497 * descriptor as an indication that a mapping exists. 1498 * We have to look it up in the L2 dtable. 1499 */ 1500 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1501 1502 if (l2 == NULL || 1503 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 1504 return 0; 1505 } 1506 1507 ptep = &ptep[l2pte_index(va)]; 1508 pte = *ptep; 1509 1510 if (pte == L2_TYPE_INV) 1511 return 0; 1512 1513 switch (pte & L2_TYPE_MASK) { 1514 case L2_TYPE_L: 1515 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 1516 break; 1517 /* 1518 * Can't check for L2_TYPE_S on V7 because of the XN 1519 * bit being part of L2_TYPE_MASK for S mappings. 1520 */ 1521 default: 1522 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 1523 break; 1524 } 1525 } 1526 1527 if (pap != NULL) 1528 *pap = pa; 1529 1530 return 1; 1531 } 1532 1533 void 1534 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1535 { 1536 struct l2_bucket *l2b; 1537 pt_entry_t *ptep, opte, npte; 1538 vaddr_t next_bucket; 1539 int flush; 1540 1541 NPDEBUG(PDB_PROTECT, 1542 printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x", 1543 pm, sva, eva, prot)); 1544 1545 if ((prot & (PROT_WRITE | PROT_EXEC)) == (PROT_WRITE | PROT_EXEC)) 1546 return; 1547 1548 if (prot == PROT_NONE) { 1549 pmap_remove(pm, sva, eva); 1550 return; 1551 } 1552 1553 /* XXX is that threshold of 4 the best choice for v7? */ 1554 if (pmap_is_current(pm)) 1555 flush = ((eva - sva) > (PAGE_SIZE * 4)) ? -1 : 0; 1556 else 1557 flush = -1; 1558 1559 while (sva < eva) { 1560 next_bucket = L2_NEXT_BUCKET(sva); 1561 if (next_bucket > eva) 1562 next_bucket = eva; 1563 1564 l2b = pmap_get_l2_bucket(pm, sva); 1565 if (l2b == NULL) { 1566 sva = next_bucket; 1567 continue; 1568 } 1569 1570 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 1571 1572 while (sva < next_bucket) { 1573 npte = opte = *ptep; 1574 if (opte != L2_TYPE_INV) { 1575 struct vm_page *pg; 1576 1577 if ((prot & PROT_WRITE) == 0) 1578 npte |= L2_V7_AP(0x4); 1579 if ((prot & PROT_EXEC) == 0) 1580 npte |= L2_V7_S_XN; 1581 *ptep = npte; 1582 PTE_SYNC(ptep); 1583 1584 pg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 1585 if (pg != NULL && (prot & PROT_WRITE) == 0) 1586 pmap_modify_pv(pg, pm, sva, 1587 PVF_WRITE, 0); 1588 1589 if (flush >= 0) { 1590 flush++; 1591 if (opte & L2_V7_AF) 1592 cpu_tlb_flushID_SE(sva); 1593 } 1594 } 1595 1596 sva += PAGE_SIZE; 1597 ptep++; 1598 } 1599 } 1600 1601 if (flush < 0) 1602 pmap_tlb_flushID(pm); 1603 1604 NPDEBUG(PDB_PROTECT, printf("\n")); 1605 } 1606 1607 void 1608 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1609 { 1610 1611 NPDEBUG(PDB_PROTECT, 1612 printf("pmap_page_protect: pg %p (0x%08lx), prot 0x%x\n", 1613 pg, pg->phys_addr, prot)); 1614 1615 switch(prot) { 1616 case PROT_READ | PROT_WRITE | PROT_EXEC: 1617 case PROT_READ | PROT_WRITE: 1618 return; 1619 1620 case PROT_READ: 1621 case PROT_READ | PROT_EXEC: 1622 pmap_clearbit(pg, PVF_WRITE); 1623 break; 1624 1625 default: 1626 pmap_page_remove(pg); 1627 break; 1628 } 1629 } 1630 1631 /* 1632 * pmap_clear_modify: 1633 * 1634 * Clear the "modified" attribute for a page. 1635 */ 1636 int 1637 pmap_clear_modify(struct vm_page *pg) 1638 { 1639 int rv; 1640 1641 if (pg->mdpage.pvh_attrs & PVF_MOD) { 1642 rv = 1; 1643 pmap_clearbit(pg, PVF_MOD); 1644 } else 1645 rv = 0; 1646 1647 return (rv); 1648 } 1649 1650 /* 1651 * pmap_clear_reference: 1652 * 1653 * Clear the "referenced" attribute for a page. 1654 */ 1655 int 1656 pmap_clear_reference(struct vm_page *pg) 1657 { 1658 int rv; 1659 1660 if (pg->mdpage.pvh_attrs & PVF_REF) { 1661 rv = 1; 1662 pmap_clearbit(pg, PVF_REF); 1663 } else 1664 rv = 0; 1665 1666 return (rv); 1667 } 1668 1669 /* 1670 * pmap_is_modified: 1671 * 1672 * Test if a page has the "modified" attribute. 1673 */ 1674 /* See <arm/pmap.h> */ 1675 1676 /* 1677 * pmap_is_referenced: 1678 * 1679 * Test if a page has the "referenced" attribute. 1680 */ 1681 /* See <arm/pmap.h> */ 1682 1683 /* 1684 * dab_access() handles the following data aborts: 1685 * 1686 * FAULT_ACCESS_2 - Access flag fault -- Level 2 1687 * 1688 * Set the Access Flag and mark the page as referenced. 1689 */ 1690 int 1691 dab_access(trapframe_t *tf, u_int fsr, u_int far, struct proc *p) 1692 { 1693 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1694 vaddr_t va = trunc_page(far); 1695 struct l2_dtable *l2; 1696 struct l2_bucket *l2b; 1697 pt_entry_t *ptep, pte; 1698 struct pv_entry *pv; 1699 struct vm_page *pg; 1700 paddr_t pa; 1701 u_int l1idx; 1702 1703 if (!TRAP_USERMODE(tf) && far >= VM_MIN_KERNEL_ADDRESS) 1704 pm = pmap_kernel(); 1705 1706 l1idx = L1_IDX(va); 1707 1708 /* 1709 * If there is no l2_dtable for this address, then the process 1710 * has no business accessing it. 1711 */ 1712 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1713 KASSERT(l2 != NULL); 1714 1715 /* 1716 * Likewise if there is no L2 descriptor table 1717 */ 1718 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1719 KASSERT(l2b->l2b_kva != NULL); 1720 1721 /* 1722 * Check the PTE itself. 1723 */ 1724 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1725 pte = *ptep; 1726 KASSERT(pte != L2_TYPE_INV); 1727 1728 pa = l2pte_pa(pte); 1729 1730 /* 1731 * Perform page referenced emulation. 1732 */ 1733 KASSERT((pte & L2_V7_AF) == 0); 1734 1735 /* Extract the physical address of the page */ 1736 pg = PHYS_TO_VM_PAGE(pa); 1737 KASSERT(pg != NULL); 1738 1739 /* Get the current flags for this page. */ 1740 pv = pmap_find_pv(pg, pm, va); 1741 KASSERT(pv != NULL); 1742 1743 pg->mdpage.pvh_attrs |= PVF_REF; 1744 pv->pv_flags |= PVF_REF; 1745 pte |= L2_V7_AF; 1746 1747 *ptep = pte; 1748 PTE_SYNC(ptep); 1749 return 0; 1750 } 1751 1752 /* 1753 * Routine: pmap_proc_iflush 1754 * 1755 * Function: 1756 * Synchronize caches corresponding to [addr, addr+len) in p. 1757 * 1758 */ 1759 void 1760 pmap_proc_iflush(struct process *pr, vaddr_t va, vsize_t len) 1761 { 1762 /* We only need to do anything if it is the current process. */ 1763 if (pr == curproc->p_p) 1764 cpu_icache_sync_range(va, len); 1765 } 1766 1767 /* 1768 * Routine: pmap_unwire 1769 * Function: Clear the wired attribute for a map/virtual-address pair. 1770 * 1771 * In/out conditions: 1772 * The mapping must already exist in the pmap. 1773 */ 1774 void 1775 pmap_unwire(pmap_t pm, vaddr_t va) 1776 { 1777 struct l2_bucket *l2b; 1778 pt_entry_t *ptep, pte; 1779 struct vm_page *pg; 1780 paddr_t pa; 1781 1782 NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va)); 1783 1784 l2b = pmap_get_l2_bucket(pm, va); 1785 KDASSERT(l2b != NULL); 1786 1787 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1788 pte = *ptep; 1789 1790 /* Extract the physical address of the page */ 1791 pa = l2pte_pa(pte); 1792 1793 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 1794 /* Update the wired bit in the pv entry for this page. */ 1795 (void) pmap_modify_pv(pg, pm, va, PVF_WIRED, 0); 1796 } 1797 } 1798 1799 void 1800 pmap_activate(struct proc *p) 1801 { 1802 pmap_t pm; 1803 struct pcb *pcb; 1804 1805 pm = p->p_vmspace->vm_map.pmap; 1806 pcb = &p->p_addr->u_pcb; 1807 1808 pmap_set_pcb_pagedir(pm, pcb); 1809 1810 if (p == curproc) { 1811 u_int cur_ttb; 1812 1813 __asm volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 1814 1815 cur_ttb &= ~(L1_TABLE_SIZE - 1); 1816 1817 if (cur_ttb == (u_int)pcb->pcb_pagedir) { 1818 /* 1819 * No need to switch address spaces. 1820 */ 1821 return; 1822 } 1823 1824 __asm volatile("cpsid if"); 1825 cpu_setttb(pcb->pcb_pagedir); 1826 __asm volatile("cpsie if"); 1827 } 1828 } 1829 1830 void 1831 pmap_update(pmap_t pm) 1832 { 1833 /* 1834 * make sure TLB/cache operations have completed. 1835 */ 1836 } 1837 1838 /* 1839 * Retire the given physical map from service. 1840 * Should only be called if the map contains no valid mappings. 1841 */ 1842 void 1843 pmap_destroy(pmap_t pm) 1844 { 1845 u_int count; 1846 1847 /* 1848 * Drop reference count 1849 */ 1850 count = --pm->pm_refs; 1851 if (count > 0) 1852 return; 1853 1854 /* 1855 * reference count is zero, free pmap resources and then free pmap. 1856 */ 1857 1858 pmap_free_l1(pm); 1859 1860 /* return the pmap to the pool */ 1861 pool_put(&pmap_pmap_pool, pm); 1862 } 1863 1864 1865 /* 1866 * void pmap_reference(pmap_t pm) 1867 * 1868 * Add a reference to the specified pmap. 1869 */ 1870 void 1871 pmap_reference(pmap_t pm) 1872 { 1873 if (pm == NULL) 1874 return; 1875 1876 pm->pm_refs++; 1877 } 1878 1879 /* 1880 * pmap_zero_page() 1881 * 1882 * Zero a given physical page by mapping it at a page hook point. 1883 * In doing the zero page op, the page we zero is mapped cacheable, as with 1884 * StrongARM accesses to non-cached pages are non-burst making writing 1885 * _any_ bulk data very slow. 1886 */ 1887 void 1888 pmap_zero_page(struct vm_page *pg) 1889 { 1890 paddr_t phys = VM_PAGE_TO_PHYS(pg); 1891 #ifdef DEBUG 1892 if (pg->mdpage.pvh_list != NULL) 1893 panic("pmap_zero_page: page has mappings"); 1894 #endif 1895 1896 /* 1897 * Hook in the page, zero it, and purge the cache for that 1898 * zeroed page. Invalidate the TLB as needed. 1899 */ 1900 *cdst_pte = L2_S_PROTO | phys | L2_V7_AF | 1901 L2_S_PROT(PTE_KERNEL, PROT_WRITE) | PTE_L2_S_CACHE_MODE; 1902 PTE_SYNC(cdst_pte); 1903 cpu_tlb_flushD_SE(cdstp); 1904 bzero_page(cdstp); 1905 } 1906 1907 /* 1908 * pmap_copy_page() 1909 * 1910 * Copy one physical page into another, by mapping the pages into 1911 * hook points. The same comment regarding cachability as in 1912 * pmap_zero_page also applies here. 1913 */ 1914 void 1915 pmap_copy_page(struct vm_page *src_pg, struct vm_page *dst_pg) 1916 { 1917 paddr_t src = VM_PAGE_TO_PHYS(src_pg); 1918 paddr_t dst = VM_PAGE_TO_PHYS(dst_pg); 1919 #ifdef DEBUG 1920 if (dst_pg->mdpage.pvh_list != NULL) 1921 panic("pmap_copy_page: dst page has mappings"); 1922 #endif 1923 1924 /* 1925 * Map the pages into the page hook points, copy them, and purge 1926 * the cache for the appropriate page. Invalidate the TLB 1927 * as required. 1928 */ 1929 *csrc_pte = L2_S_PROTO | src | L2_V7_AF | 1930 L2_S_PROT(PTE_KERNEL, PROT_READ) | PTE_L2_S_CACHE_MODE; 1931 PTE_SYNC(csrc_pte); 1932 *cdst_pte = L2_S_PROTO | dst | L2_V7_AF | 1933 L2_S_PROT(PTE_KERNEL, PROT_WRITE) | PTE_L2_S_CACHE_MODE; 1934 PTE_SYNC(cdst_pte); 1935 cpu_tlb_flushD_SE(csrcp); 1936 cpu_tlb_flushD_SE(cdstp); 1937 bcopy_page(csrcp, cdstp); 1938 } 1939 1940 /* 1941 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1942 * 1943 * Return the start and end addresses of the kernel's virtual space. 1944 * These values are setup in pmap_bootstrap and are updated as pages 1945 * are allocated. 1946 */ 1947 void 1948 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1949 { 1950 *start = virtual_avail; 1951 *end = virtual_end; 1952 } 1953 1954 /* 1955 * Helper function for pmap_grow_l2_bucket() 1956 */ 1957 static __inline int 1958 pmap_grow_map(vaddr_t va, pt_entry_t cache_mode, paddr_t *pap) 1959 { 1960 struct l2_bucket *l2b; 1961 pt_entry_t *ptep; 1962 paddr_t pa; 1963 1964 KASSERT((va & PAGE_MASK) == 0); 1965 1966 if (uvm.page_init_done == 0) { 1967 if (uvm_page_physget(&pa) == 0) 1968 return (1); 1969 } else { 1970 struct vm_page *pg; 1971 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 1972 if (pg == NULL) 1973 return (1); 1974 pa = VM_PAGE_TO_PHYS(pg); 1975 } 1976 1977 if (pap) 1978 *pap = pa; 1979 1980 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1981 KDASSERT(l2b != NULL); 1982 1983 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1984 *ptep = L2_S_PROTO | pa | L2_V7_AF | cache_mode | 1985 L2_S_PROT(PTE_KERNEL, PROT_READ | PROT_WRITE); 1986 PTE_SYNC(ptep); 1987 cpu_tlb_flushD_SE(va); 1988 1989 memset((void *)va, 0, PAGE_SIZE); 1990 return (0); 1991 } 1992 1993 /* 1994 * This is the same as pmap_alloc_l2_bucket(), except that it is only 1995 * used by pmap_growkernel(). 1996 */ 1997 static __inline struct l2_bucket * 1998 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 1999 { 2000 struct l2_dtable *l2; 2001 struct l2_bucket *l2b; 2002 u_short l1idx; 2003 vaddr_t nva; 2004 2005 l1idx = L1_IDX(va); 2006 2007 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 2008 /* 2009 * No mapping at this address, as there is 2010 * no entry in the L1 table. 2011 * Need to allocate a new l2_dtable. 2012 */ 2013 nva = pmap_kernel_l2dtable_kva; 2014 if ((nva & PGOFSET) == 0) { 2015 /* 2016 * Need to allocate a backing page 2017 */ 2018 if (pmap_grow_map(nva, PTE_L2_S_CACHE_MODE, NULL)) 2019 return (NULL); 2020 } 2021 2022 l2 = (struct l2_dtable *)nva; 2023 nva += sizeof(struct l2_dtable); 2024 2025 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 2026 /* 2027 * The new l2_dtable straddles a page boundary. 2028 * Map in another page to cover it. 2029 */ 2030 if (pmap_grow_map(trunc_page(nva), 2031 PTE_L2_S_CACHE_MODE, NULL)) 2032 return (NULL); 2033 } 2034 2035 pmap_kernel_l2dtable_kva = nva; 2036 2037 /* 2038 * Link it into the parent pmap 2039 */ 2040 pm->pm_l2[L2_IDX(l1idx)] = l2; 2041 } 2042 2043 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2044 2045 /* 2046 * Fetch pointer to the L2 page table associated with the address. 2047 */ 2048 if (l2b->l2b_kva == NULL) { 2049 pt_entry_t *ptep; 2050 2051 /* 2052 * No L2 page table has been allocated. Chances are, this 2053 * is because we just allocated the l2_dtable, above. 2054 */ 2055 nva = pmap_kernel_l2ptp_kva; 2056 ptep = (pt_entry_t *)nva; 2057 if ((nva & PGOFSET) == 0) { 2058 /* 2059 * Need to allocate a backing page 2060 */ 2061 if (pmap_grow_map(nva, PTE_L2_S_CACHE_MODE_PT, 2062 &pmap_kernel_l2ptp_phys)) 2063 return (NULL); 2064 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 2065 } 2066 2067 l2->l2_occupancy++; 2068 l2b->l2b_kva = ptep; 2069 l2b->l2b_l1idx = l1idx; 2070 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 2071 2072 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 2073 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 2074 } 2075 2076 return (l2b); 2077 } 2078 2079 vaddr_t 2080 pmap_growkernel(vaddr_t maxkvaddr) 2081 { 2082 pmap_t kpm = pmap_kernel(); 2083 struct l1_ttable *l1; 2084 struct l2_bucket *l2b; 2085 pd_entry_t *pl1pd; 2086 int s; 2087 2088 if (maxkvaddr <= pmap_curmaxkvaddr) 2089 goto out; /* we are OK */ 2090 2091 NPDEBUG(PDB_GROWKERN, 2092 printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n", 2093 pmap_curmaxkvaddr, maxkvaddr)); 2094 2095 KDASSERT(maxkvaddr <= virtual_end); 2096 2097 /* 2098 * whoops! we need to add kernel PTPs 2099 */ 2100 2101 s = splhigh(); /* to be safe */ 2102 2103 /* Map 1MB at a time */ 2104 for (; pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE) { 2105 2106 l2b = pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 2107 KDASSERT(l2b != NULL); 2108 2109 /* Distribute new L1 entry to all other L1s */ 2110 TAILQ_FOREACH(l1, &l1_list, l1_link) { 2111 pl1pd = &l1->l1_kva[L1_IDX(pmap_curmaxkvaddr)]; 2112 *pl1pd = L1_C_PROTO | l2b->l2b_phys; 2113 PTE_SYNC(pl1pd); 2114 } 2115 } 2116 2117 /* 2118 * flush out the cache, expensive but growkernel will happen so 2119 * rarely 2120 */ 2121 cpu_dcache_wbinv_all(); 2122 cpu_sdcache_wbinv_all(); 2123 cpu_tlb_flushD(); 2124 2125 splx(s); 2126 2127 out: 2128 return (pmap_curmaxkvaddr); 2129 } 2130 2131 /************************ Utility routines ****************************/ 2132 2133 /* 2134 * vector_page_setprot: 2135 * 2136 * Manipulate the protection of the vector page. 2137 */ 2138 void 2139 vector_page_setprot(int prot) 2140 { 2141 struct l2_bucket *l2b; 2142 pt_entry_t *ptep; 2143 2144 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 2145 KDASSERT(l2b != NULL); 2146 2147 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 2148 2149 *ptep = (*ptep & ~L2_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 2150 PTE_SYNC(ptep); 2151 cpu_tlb_flushD_SE(vector_page); 2152 } 2153 2154 /* 2155 * This is used to stuff certain critical values into the PCB where they 2156 * can be accessed quickly from cpu_switch() et al. 2157 */ 2158 void 2159 pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 2160 { 2161 KDASSERT(pm->pm_l1); 2162 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 2163 } 2164 2165 /* 2166 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 2167 * Returns 1 if the mapping exists, else 0. 2168 * 2169 * NOTE: This function is only used by a couple of arm-specific modules. 2170 * It is not safe to take any pmap locks here, since we could be right 2171 * in the middle of debugging the pmap anyway... 2172 * 2173 * It is possible for this routine to return 0 even though a valid 2174 * mapping does exist. This is because we don't lock, so the metadata 2175 * state may be inconsistent. 2176 * 2177 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 2178 * a "section" mapping. 2179 */ 2180 int 2181 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 2182 { 2183 struct l2_dtable *l2; 2184 pd_entry_t *pl1pd, l1pd; 2185 pt_entry_t *ptep; 2186 u_short l1idx; 2187 2188 if (pm->pm_l1 == NULL) 2189 return 0; 2190 2191 l1idx = L1_IDX(va); 2192 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2193 l1pd = *pl1pd; 2194 2195 if (l1pte_section_p(l1pd)) { 2196 *ptp = NULL; 2197 return 1; 2198 } 2199 2200 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2201 if (l2 == NULL || 2202 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2203 return 0; 2204 } 2205 2206 *ptp = &ptep[l2pte_index(va)]; 2207 return 1; 2208 } 2209 2210 /************************ Bootstrapping routines ****************************/ 2211 2212 void 2213 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 2214 { 2215 l1->l1_kva = l1pt; 2216 2217 /* 2218 * Copy the kernel's L1 entries to each new L1. 2219 */ 2220 if (pmap_initialized) 2221 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 2222 2223 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, &l1->l1_physaddr) == 0) 2224 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 2225 2226 TAILQ_INSERT_TAIL(&l1_list, l1, l1_link); 2227 } 2228 2229 /* 2230 * pmap_bootstrap() is called from the board-specific initarm() routine 2231 * once the kernel L1/L2 descriptors tables have been set up. 2232 * 2233 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 2234 * spread over a number of disparate files/functions. 2235 * 2236 * We are passed the following parameters 2237 * - kernel_l1pt 2238 * This is a pointer to the base of the kernel's L1 translation table. 2239 * - vstart 2240 * 1MB-aligned start of managed kernel virtual memory. 2241 * - vend 2242 * 1MB-aligned end of managed kernel virtual memory. 2243 * 2244 * We use the first parameter to build the metadata (struct l1_ttable and 2245 * struct l2_dtable) necessary to track kernel mappings. 2246 */ 2247 #define PMAP_STATIC_L2_SIZE 16 2248 void 2249 pmap_bootstrap(pd_entry_t *kernel_l1pt, vaddr_t vstart, vaddr_t vend) 2250 { 2251 static struct l1_ttable static_l1; 2252 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 2253 struct l1_ttable *l1 = &static_l1; 2254 struct l2_dtable *l2; 2255 struct l2_bucket *l2b; 2256 pmap_t pm = pmap_kernel(); 2257 pd_entry_t pde; 2258 pt_entry_t *ptep; 2259 paddr_t pa; 2260 vsize_t size; 2261 int l1idx, l2idx, l2next = 0; 2262 2263 /* 2264 * Initialise the kernel pmap object 2265 */ 2266 pm->pm_l1 = l1; 2267 pm->pm_refs = 1; 2268 2269 /* 2270 * Scan the L1 translation table created by initarm() and create 2271 * the required metadata for all valid mappings found in it. 2272 */ 2273 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 2274 pde = kernel_l1pt[l1idx]; 2275 2276 /* 2277 * We're only interested in Coarse mappings. 2278 * pmap_extract() can deal with section mappings without 2279 * recourse to checking L2 metadata. 2280 */ 2281 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 2282 continue; 2283 2284 /* 2285 * Lookup the KVA of this L2 descriptor table 2286 */ 2287 pa = (paddr_t)(pde & L1_C_ADDR_MASK); 2288 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2289 if (ptep == NULL) { 2290 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 2291 (u_int)l1idx << L1_S_SHIFT, pa); 2292 } 2293 2294 /* 2295 * Fetch the associated L2 metadata structure. 2296 * Allocate a new one if necessary. 2297 */ 2298 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 2299 if (l2next == PMAP_STATIC_L2_SIZE) 2300 panic("pmap_bootstrap: out of static L2s"); 2301 pm->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++]; 2302 } 2303 2304 /* 2305 * One more L1 slot tracked... 2306 */ 2307 l2->l2_occupancy++; 2308 2309 /* 2310 * Fill in the details of the L2 descriptor in the 2311 * appropriate bucket. 2312 */ 2313 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2314 l2b->l2b_kva = ptep; 2315 l2b->l2b_phys = pa; 2316 l2b->l2b_l1idx = l1idx; 2317 2318 /* 2319 * Establish an initial occupancy count for this descriptor 2320 */ 2321 for (l2idx = 0; 2322 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 2323 l2idx++) { 2324 if (ptep[l2idx] != L2_TYPE_INV) 2325 l2b->l2b_occupancy++; 2326 } 2327 } 2328 2329 cpu_idcache_wbinv_all(); 2330 cpu_sdcache_wbinv_all(); 2331 cpu_tlb_flushID(); 2332 2333 /* 2334 * now we allocate the "special" VAs which are used for tmp mappings 2335 * by the pmap (and other modules). we allocate the VAs by advancing 2336 * virtual_avail (note that there are no pages mapped at these VAs). 2337 * 2338 * Managed KVM space start from wherever initarm() tells us. 2339 */ 2340 virtual_avail = vstart; 2341 virtual_end = vend; 2342 2343 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 2344 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 2345 pmap_alloc_specials(&virtual_avail, 1, (void *)&memhook, NULL); 2346 pmap_alloc_specials(&virtual_avail, round_page(MSGBUFSIZE) / PAGE_SIZE, 2347 (void *)&msgbufaddr, NULL); 2348 2349 /* 2350 * Allocate a range of kernel virtual address space to be used 2351 * for L2 descriptor tables and metadata allocation in 2352 * pmap_growkernel(). 2353 */ 2354 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 2355 pmap_alloc_specials(&virtual_avail, 2356 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 2357 &pmap_kernel_l2ptp_kva, NULL); 2358 2359 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 2360 pmap_alloc_specials(&virtual_avail, 2361 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 2362 &pmap_kernel_l2dtable_kva, NULL); 2363 2364 /* 2365 * We can now initialise the first L1's metadata. 2366 */ 2367 TAILQ_INIT(&l1_list); 2368 pmap_init_l1(l1, kernel_l1pt); 2369 2370 /* 2371 * Initialize the pmap pool. 2372 */ 2373 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, IPL_NONE, 0, 2374 "pmappl", &pool_allocator_single); 2375 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, IPL_VM, 0, 2376 "pvepl", &pmap_pv_allocator); 2377 pool_init(&pmap_l2dtable_pool, sizeof(struct l2_dtable), 0, IPL_VM, 0, 2378 "l2dtblpl", NULL); 2379 pool_init(&pmap_l2ptp_pool, L2_TABLE_SIZE_REAL, L2_TABLE_SIZE_REAL, 2380 IPL_VM, 0, "l2ptppl", &pool_allocator_single); 2381 2382 cpu_dcache_wbinv_all(); 2383 cpu_sdcache_wbinv_all(); 2384 } 2385 2386 void 2387 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 2388 { 2389 vaddr_t va = *availp; 2390 struct l2_bucket *l2b; 2391 2392 if (ptep) { 2393 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2394 if (l2b == NULL) 2395 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 2396 2397 if (ptep) 2398 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 2399 } 2400 2401 *vap = va; 2402 *availp = va + (PAGE_SIZE * pages); 2403 } 2404 2405 void 2406 pmap_init(void) 2407 { 2408 pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 2409 2410 pmap_initialized = 1; 2411 } 2412 2413 void * 2414 pmap_pv_page_alloc(struct pool *pp, int flags, int *slowdown) 2415 { 2416 struct kmem_dyn_mode kd = KMEM_DYN_INITIALIZER; 2417 2418 kd.kd_waitok = ISSET(flags, PR_WAITOK); 2419 kd.kd_slowdown = slowdown; 2420 2421 return (km_alloc(pp->pr_pgsize, 2422 pmap_initialized ? &kv_page : &kv_any, pp->pr_crange, &kd)); 2423 } 2424 2425 void 2426 pmap_pv_page_free(struct pool *pp, void *v) 2427 { 2428 km_free(v, pp->pr_pgsize, &kv_page, pp->pr_crange); 2429 } 2430 2431 /* 2432 * pmap_postinit() 2433 * 2434 * This routine is called after the vm and kmem subsystems have been 2435 * initialised. This allows the pmap code to perform any initialisation 2436 * that can only be done once the memory allocation is in place. 2437 */ 2438 void 2439 pmap_postinit(void) 2440 { 2441 pool_setlowat(&pmap_l2ptp_pool, 2442 (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 2443 pool_setlowat(&pmap_l2dtable_pool, 2444 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 2445 } 2446 2447 /* 2448 * Note that the following routines are used by board-specific initialisation 2449 * code to configure the initial kernel page tables. 2450 * 2451 * If ARM32_NEW_VM_LAYOUT is *not* defined, they operate on the assumption that 2452 * L2 page-table pages are 4KB in size and use 4 L1 slots. This mimics the 2453 * behaviour of the old pmap, and provides an easy migration path for 2454 * initial bring-up of the new pmap on existing ports. Fortunately, 2455 * pmap_bootstrap() compensates for this hackery. This is only a stop-gap and 2456 * will be deprecated. 2457 * 2458 * If ARM32_NEW_VM_LAYOUT *is* defined, these functions deal with 1KB L2 page 2459 * tables. 2460 */ 2461 2462 /* 2463 * This list exists for the benefit of pmap_map_chunk(). It keeps track 2464 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 2465 * find them as necessary. 2466 * 2467 * Note that the data on this list MUST remain valid after initarm() returns, 2468 * as pmap_bootstrap() uses it to construct L2 table metadata. 2469 */ 2470 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 2471 2472 vaddr_t 2473 kernel_pt_lookup(paddr_t pa) 2474 { 2475 pv_addr_t *pv; 2476 2477 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 2478 #ifndef ARM32_NEW_VM_LAYOUT 2479 if (pv->pv_pa == (pa & ~PGOFSET)) 2480 return (pv->pv_va | (pa & PGOFSET)); 2481 #else 2482 if (pv->pv_pa == pa) 2483 return (pv->pv_va); 2484 #endif 2485 } 2486 return (0); 2487 } 2488 2489 /* 2490 * pmap_map_section: 2491 * 2492 * Create a single section mapping. 2493 */ 2494 void 2495 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 2496 { 2497 pd_entry_t *pde = (pd_entry_t *) l1pt; 2498 pd_entry_t fl; 2499 2500 switch (cache) { 2501 case PTE_NOCACHE: 2502 default: 2503 fl = 0; 2504 break; 2505 2506 case PTE_CACHE: 2507 fl = PTE_L1_S_CACHE_MODE; 2508 break; 2509 2510 case PTE_PAGETABLE: 2511 fl = PTE_L1_S_CACHE_MODE_PT; 2512 break; 2513 } 2514 2515 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | L1_S_V7_AF | 2516 L1_S_PROT(PTE_KERNEL, prot) | fl; 2517 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 2518 } 2519 2520 /* 2521 * pmap_map_entry: 2522 * 2523 * Create a single page mapping. 2524 */ 2525 void 2526 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 2527 { 2528 pd_entry_t *pde = (pd_entry_t *) l1pt; 2529 pt_entry_t fl; 2530 pt_entry_t *pte; 2531 2532 switch (cache) { 2533 case PTE_NOCACHE: 2534 default: 2535 fl = 0; 2536 break; 2537 2538 case PTE_CACHE: 2539 fl = PTE_L2_S_CACHE_MODE; 2540 break; 2541 2542 case PTE_PAGETABLE: 2543 fl = PTE_L2_S_CACHE_MODE_PT; 2544 break; 2545 } 2546 2547 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 2548 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 2549 2550 #ifndef ARM32_NEW_VM_LAYOUT 2551 pte = (pt_entry_t *) 2552 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 2553 #else 2554 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 2555 #endif 2556 if (pte == NULL) 2557 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 2558 2559 #ifndef ARM32_NEW_VM_LAYOUT 2560 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | L2_V7_AF | 2561 L2_S_PROT(PTE_KERNEL, prot) | fl; 2562 PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]); 2563 #else 2564 pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_V7_AF | 2565 L2_S_PROT(PTE_KERNEL, prot) | fl; 2566 PTE_SYNC(&pte[l2pte_index(va)]); 2567 #endif 2568 } 2569 2570 /* 2571 * pmap_link_l2pt: 2572 * 2573 * Link the L2 page table specified by "l2pv" into the L1 2574 * page table at the slot for "va". 2575 */ 2576 void 2577 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 2578 { 2579 pd_entry_t *pde = (pd_entry_t *) l1pt; 2580 u_int slot = va >> L1_S_SHIFT; 2581 2582 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 2583 #ifdef ARM32_NEW_VM_LAYOUT 2584 PTE_SYNC(&pde[slot]); 2585 #else 2586 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 2587 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 2588 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 2589 PTE_SYNC_RANGE(&pde[slot + 0], 4); 2590 #endif 2591 2592 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 2593 } 2594 2595 /* 2596 * pmap_map_chunk: 2597 * 2598 * Map a chunk of memory using the most efficient mappings 2599 * possible (section, large page, small page) into the 2600 * provided L1 and L2 tables at the specified virtual address. 2601 */ 2602 vsize_t 2603 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 2604 int prot, int cache) 2605 { 2606 pd_entry_t *pde = (pd_entry_t *) l1pt; 2607 pt_entry_t *pte, f1, f2s, f2l; 2608 vsize_t resid; 2609 int i; 2610 2611 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 2612 2613 if (l1pt == 0) 2614 panic("pmap_map_chunk: no L1 table provided"); 2615 2616 #ifdef VERBOSE_INIT_ARM 2617 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 2618 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 2619 #endif 2620 2621 switch (cache) { 2622 case PTE_NOCACHE: 2623 default: 2624 f1 = 0; 2625 f2l = 0; 2626 f2s = 0; 2627 break; 2628 2629 case PTE_CACHE: 2630 f1 = PTE_L1_S_CACHE_MODE; 2631 f2l = PTE_L2_L_CACHE_MODE; 2632 f2s = PTE_L2_S_CACHE_MODE; 2633 break; 2634 2635 case PTE_PAGETABLE: 2636 f1 = PTE_L1_S_CACHE_MODE_PT; 2637 f2l = PTE_L2_L_CACHE_MODE_PT; 2638 f2s = PTE_L2_S_CACHE_MODE_PT; 2639 break; 2640 } 2641 2642 size = resid; 2643 2644 while (resid > 0) { 2645 /* See if we can use a section mapping. */ 2646 if (L1_S_MAPPABLE_P(va, pa, resid)) { 2647 #ifdef VERBOSE_INIT_ARM 2648 printf("S"); 2649 #endif 2650 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 2651 L1_S_V7_AF | L1_S_PROT(PTE_KERNEL, prot) | f1; 2652 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 2653 va += L1_S_SIZE; 2654 pa += L1_S_SIZE; 2655 resid -= L1_S_SIZE; 2656 continue; 2657 } 2658 2659 /* 2660 * Ok, we're going to use an L2 table. Make sure 2661 * one is actually in the corresponding L1 slot 2662 * for the current VA. 2663 */ 2664 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 2665 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 2666 2667 #ifndef ARM32_NEW_VM_LAYOUT 2668 pte = (pt_entry_t *) 2669 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 2670 #else 2671 pte = (pt_entry_t *) kernel_pt_lookup( 2672 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 2673 #endif 2674 if (pte == NULL) 2675 panic("pmap_map_chunk: can't find L2 table for VA" 2676 "0x%08lx", va); 2677 2678 /* See if we can use a L2 large page mapping. */ 2679 if (L2_L_MAPPABLE_P(va, pa, resid)) { 2680 #ifdef VERBOSE_INIT_ARM 2681 printf("L"); 2682 #endif 2683 for (i = 0; i < 16; i++) { 2684 #ifndef ARM32_NEW_VM_LAYOUT 2685 pte[((va >> PGSHIFT) & 0x3f0) + i] = 2686 L2_L_PROTO | pa | L2_V7_AF | 2687 L2_L_PROT(PTE_KERNEL, prot) | f2l; 2688 PTE_SYNC(&pte[((va >> PGSHIFT) & 0x3f0) + i]); 2689 #else 2690 pte[l2pte_index(va) + i] = 2691 L2_L_PROTO | pa | L2_V7_AF | 2692 L2_L_PROT(PTE_KERNEL, prot) | f2l; 2693 PTE_SYNC(&pte[l2pte_index(va) + i]); 2694 #endif 2695 } 2696 va += L2_L_SIZE; 2697 pa += L2_L_SIZE; 2698 resid -= L2_L_SIZE; 2699 continue; 2700 } 2701 2702 /* Use a small page mapping. */ 2703 #ifdef VERBOSE_INIT_ARM 2704 printf("P"); 2705 #endif 2706 #ifndef ARM32_NEW_VM_LAYOUT 2707 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | L2_V7_AF | 2708 L2_S_PROT(PTE_KERNEL, prot) | f2s; 2709 PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]); 2710 #else 2711 pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_V7_AF | 2712 L2_S_PROT(PTE_KERNEL, prot) | f2s; 2713 PTE_SYNC(&pte[l2pte_index(va)]); 2714 #endif 2715 va += PAGE_SIZE; 2716 pa += PAGE_SIZE; 2717 resid -= PAGE_SIZE; 2718 } 2719 #ifdef VERBOSE_INIT_ARM 2720 printf("\n"); 2721 #endif 2722 return (size); 2723 } 2724 2725 /********************** PTE initialization routines **************************/ 2726 2727 /* 2728 * This routine is called to set up cache modes, etc. 2729 */ 2730 2731 void 2732 pmap_pte_init_armv7(void) 2733 { 2734 uint32_t id_mmfr0, id_mmfr3; 2735 2736 pmap_needs_pte_sync = 1; 2737 2738 /* Check if the PXN bit is supported. */ 2739 __asm volatile("mrc p15, 0, %0, c0, c1, 4" : "=r"(id_mmfr0)); 2740 if ((id_mmfr0 & ID_MMFR0_VMSA_MASK) >= VMSA_V7_PXN) 2741 l1_c_pxn = L1_C_V7_PXN; 2742 2743 /* Check for coherent walk. */ 2744 __asm volatile("mrc p15, 0, %0, c0, c1, 7" : "=r"(id_mmfr3)); 2745 if ((id_mmfr3 & 0x00f00000) == 0x00100000) 2746 pmap_needs_pte_sync = 0; 2747 } 2748