1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2020 Justin Hibbits
5 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
6 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * Some hw specific parts of this pmap were derived or influenced
30 * by NetBSD's ibm4xx pmap module. More generic code is shared with
31 * a few other pmap modules from the FreeBSD tree.
32 */
33
34 /*
35 * VM layout notes:
36 *
37 * Kernel and user threads run within one common virtual address space
38 * defined by AS=0.
39 *
40 * 32-bit pmap:
41 * Virtual address space layout:
42 * -----------------------------
43 * 0x0000_0000 - 0x7fff_ffff : user process
44 * 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.)
45 * 0xc000_0000 - 0xffff_efff : KVA
46 */
47
48 #include <sys/cdefs.h>
49 #include "opt_ddb.h"
50 #include "opt_kstack_pages.h"
51
52 #include <sys/param.h>
53 #include <sys/conf.h>
54 #include <sys/malloc.h>
55 #include <sys/ktr.h>
56 #include <sys/proc.h>
57 #include <sys/user.h>
58 #include <sys/queue.h>
59 #include <sys/systm.h>
60 #include <sys/kernel.h>
61 #include <sys/kerneldump.h>
62 #include <sys/linker.h>
63 #include <sys/msgbuf.h>
64 #include <sys/lock.h>
65 #include <sys/mutex.h>
66 #include <sys/rwlock.h>
67 #include <sys/sched.h>
68 #include <sys/smp.h>
69 #include <sys/vmmeter.h>
70
71 #include <vm/vm.h>
72 #include <vm/vm_page.h>
73 #include <vm/vm_kern.h>
74 #include <vm/vm_pageout.h>
75 #include <vm/vm_extern.h>
76 #include <vm/vm_object.h>
77 #include <vm/vm_param.h>
78 #include <vm/vm_map.h>
79 #include <vm/vm_pager.h>
80 #include <vm/vm_phys.h>
81 #include <vm/vm_pagequeue.h>
82 #include <vm/uma.h>
83
84 #include <machine/_inttypes.h>
85 #include <machine/cpu.h>
86 #include <machine/pcb.h>
87 #include <machine/platform.h>
88
89 #include <machine/tlb.h>
90 #include <machine/spr.h>
91 #include <machine/md_var.h>
92 #include <machine/mmuvar.h>
93 #include <machine/pmap.h>
94 #include <machine/pte.h>
95
96 #include <ddb/ddb.h>
97
98 #define PRI0ptrX "08x"
99
100 /* Reserved KVA space and mutex for mmu_booke_zero_page. */
101 static vm_offset_t zero_page_va;
102 static struct mtx zero_page_mutex;
103
104 /* Reserved KVA space and mutex for mmu_booke_copy_page. */
105 static vm_offset_t copy_page_src_va;
106 static vm_offset_t copy_page_dst_va;
107 static struct mtx copy_page_mutex;
108
109 static vm_offset_t kernel_ptbl_root;
110 static unsigned int kernel_ptbls; /* Number of KVA ptbls. */
111
112 /**************************************************************************/
113 /* PMAP */
114 /**************************************************************************/
115
116 #define VM_MAPDEV_BASE ((vm_offset_t)VM_MAXUSER_ADDRESS + PAGE_SIZE)
117
118 static void tid_flush(tlbtid_t tid);
119
120 /**************************************************************************/
121 /* Page table management */
122 /**************************************************************************/
123
124 #define PMAP_ROOT_SIZE (sizeof(pte_t**) * PDIR_NENTRIES)
125 static void ptbl_init(void);
126 static struct ptbl_buf *ptbl_buf_alloc(void);
127 static void ptbl_buf_free(struct ptbl_buf *);
128 static void ptbl_free_pmap_ptbl(pmap_t, pte_t *);
129
130 static pte_t *ptbl_alloc(pmap_t, unsigned int, bool);
131 static void ptbl_free(pmap_t, unsigned int);
132 static void ptbl_hold(pmap_t, unsigned int);
133 static int ptbl_unhold(pmap_t, unsigned int);
134
135 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
136 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, bool);
137 static int pte_remove(pmap_t, vm_offset_t, uint8_t);
138 static pte_t *pte_find(pmap_t, vm_offset_t);
139
140 struct ptbl_buf {
141 TAILQ_ENTRY(ptbl_buf) link; /* list link */
142 vm_offset_t kva; /* va of mapping */
143 };
144
145 /* Number of kva ptbl buffers, each covering one ptbl (PTBL_PAGES). */
146 #define PTBL_BUFS (128 * 16)
147
148 /* ptbl free list and a lock used for access synchronization. */
149 static TAILQ_HEAD(, ptbl_buf) ptbl_buf_freelist;
150 static struct mtx ptbl_buf_freelist_lock;
151
152 /* Base address of kva space allocated fot ptbl bufs. */
153 static vm_offset_t ptbl_buf_pool_vabase;
154
155 /* Pointer to ptbl_buf structures. */
156 static struct ptbl_buf *ptbl_bufs;
157
158 /**************************************************************************/
159 /* Page table related */
160 /**************************************************************************/
161
162 /* Initialize pool of kva ptbl buffers. */
163 static void
ptbl_init(void)164 ptbl_init(void)
165 {
166 int i;
167
168 CTR3(KTR_PMAP, "%s: s (ptbl_bufs = 0x%08x size 0x%08x)", __func__,
169 (uint32_t)ptbl_bufs, sizeof(struct ptbl_buf) * PTBL_BUFS);
170 CTR3(KTR_PMAP, "%s: s (ptbl_buf_pool_vabase = 0x%08x size = 0x%08x)",
171 __func__, ptbl_buf_pool_vabase, PTBL_BUFS * PTBL_PAGES * PAGE_SIZE);
172
173 mtx_init(&ptbl_buf_freelist_lock, "ptbl bufs lock", NULL, MTX_DEF);
174 TAILQ_INIT(&ptbl_buf_freelist);
175
176 for (i = 0; i < PTBL_BUFS; i++) {
177 ptbl_bufs[i].kva =
178 ptbl_buf_pool_vabase + i * PTBL_PAGES * PAGE_SIZE;
179 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, &ptbl_bufs[i], link);
180 }
181 }
182
183 /* Get a ptbl_buf from the freelist. */
184 static struct ptbl_buf *
ptbl_buf_alloc(void)185 ptbl_buf_alloc(void)
186 {
187 struct ptbl_buf *buf;
188
189 mtx_lock(&ptbl_buf_freelist_lock);
190 buf = TAILQ_FIRST(&ptbl_buf_freelist);
191 if (buf != NULL)
192 TAILQ_REMOVE(&ptbl_buf_freelist, buf, link);
193 mtx_unlock(&ptbl_buf_freelist_lock);
194
195 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
196
197 return (buf);
198 }
199
200 /* Return ptbl buff to free pool. */
201 static void
ptbl_buf_free(struct ptbl_buf * buf)202 ptbl_buf_free(struct ptbl_buf *buf)
203 {
204
205 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
206
207 mtx_lock(&ptbl_buf_freelist_lock);
208 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, buf, link);
209 mtx_unlock(&ptbl_buf_freelist_lock);
210 }
211
212 /*
213 * Search the list of allocated ptbl bufs and find on list of allocated ptbls
214 */
215 static void
ptbl_free_pmap_ptbl(pmap_t pmap,pte_t * ptbl)216 ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl)
217 {
218 struct ptbl_buf *pbuf;
219
220 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
221
222 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
223
224 TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link)
225 if (pbuf->kva == (vm_offset_t)ptbl) {
226 /* Remove from pmap ptbl buf list. */
227 TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link);
228
229 /* Free corresponding ptbl buf. */
230 ptbl_buf_free(pbuf);
231 break;
232 }
233 }
234
235 /* Allocate page table. */
236 static pte_t *
ptbl_alloc(pmap_t pmap,unsigned int pdir_idx,bool nosleep)237 ptbl_alloc(pmap_t pmap, unsigned int pdir_idx, bool nosleep)
238 {
239 vm_page_t mtbl[PTBL_PAGES];
240 vm_page_t m;
241 struct ptbl_buf *pbuf;
242 unsigned int pidx;
243 pte_t *ptbl;
244 int i, j;
245
246 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
247 (pmap == kernel_pmap), pdir_idx);
248
249 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
250 ("ptbl_alloc: invalid pdir_idx"));
251 KASSERT((pmap->pm_pdir[pdir_idx] == NULL),
252 ("pte_alloc: valid ptbl entry exists!"));
253
254 pbuf = ptbl_buf_alloc();
255 if (pbuf == NULL)
256 panic("pte_alloc: couldn't alloc kernel virtual memory");
257
258 ptbl = (pte_t *)pbuf->kva;
259
260 CTR2(KTR_PMAP, "%s: ptbl kva = %p", __func__, ptbl);
261
262 for (i = 0; i < PTBL_PAGES; i++) {
263 pidx = (PTBL_PAGES * pdir_idx) + i;
264 while ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED)) == NULL) {
265 if (nosleep) {
266 ptbl_free_pmap_ptbl(pmap, ptbl);
267 for (j = 0; j < i; j++)
268 vm_page_free(mtbl[j]);
269 vm_wire_sub(i);
270 return (NULL);
271 }
272 PMAP_UNLOCK(pmap);
273 rw_wunlock(&pvh_global_lock);
274 vm_wait(NULL);
275 rw_wlock(&pvh_global_lock);
276 PMAP_LOCK(pmap);
277 }
278 m->pindex = pidx;
279 mtbl[i] = m;
280 }
281
282 /* Map allocated pages into kernel_pmap. */
283 mmu_booke_qenter((vm_offset_t)ptbl, mtbl, PTBL_PAGES);
284
285 /* Zero whole ptbl. */
286 bzero((caddr_t)ptbl, PTBL_PAGES * PAGE_SIZE);
287
288 /* Add pbuf to the pmap ptbl bufs list. */
289 TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link);
290
291 return (ptbl);
292 }
293
294 /* Free ptbl pages and invalidate pdir entry. */
295 static void
ptbl_free(pmap_t pmap,unsigned int pdir_idx)296 ptbl_free(pmap_t pmap, unsigned int pdir_idx)
297 {
298 pte_t *ptbl;
299 vm_paddr_t pa;
300 vm_offset_t va;
301 vm_page_t m;
302 int i;
303
304 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
305 (pmap == kernel_pmap), pdir_idx);
306
307 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
308 ("ptbl_free: invalid pdir_idx"));
309
310 ptbl = pmap->pm_pdir[pdir_idx];
311
312 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
313
314 KASSERT((ptbl != NULL), ("ptbl_free: null ptbl"));
315
316 /*
317 * Invalidate the pdir entry as soon as possible, so that other CPUs
318 * don't attempt to look up the page tables we are releasing.
319 */
320 mtx_lock_spin(&tlbivax_mutex);
321 tlb_miss_lock();
322
323 pmap->pm_pdir[pdir_idx] = NULL;
324
325 tlb_miss_unlock();
326 mtx_unlock_spin(&tlbivax_mutex);
327
328 for (i = 0; i < PTBL_PAGES; i++) {
329 va = ((vm_offset_t)ptbl + (i * PAGE_SIZE));
330 pa = pte_vatopa(kernel_pmap, va);
331 m = PHYS_TO_VM_PAGE(pa);
332 vm_page_free_zero(m);
333 vm_wire_sub(1);
334 mmu_booke_kremove(va);
335 }
336
337 ptbl_free_pmap_ptbl(pmap, ptbl);
338 }
339
340 /*
341 * Decrement ptbl pages hold count and attempt to free ptbl pages.
342 * Called when removing pte entry from ptbl.
343 *
344 * Return 1 if ptbl pages were freed.
345 */
346 static int
ptbl_unhold(pmap_t pmap,unsigned int pdir_idx)347 ptbl_unhold(pmap_t pmap, unsigned int pdir_idx)
348 {
349 pte_t *ptbl;
350 vm_paddr_t pa;
351 vm_page_t m;
352 int i;
353
354 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
355 (pmap == kernel_pmap), pdir_idx);
356
357 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
358 ("ptbl_unhold: invalid pdir_idx"));
359 KASSERT((pmap != kernel_pmap),
360 ("ptbl_unhold: unholding kernel ptbl!"));
361
362 ptbl = pmap->pm_pdir[pdir_idx];
363
364 //debugf("ptbl_unhold: ptbl = 0x%08x\n", (u_int32_t)ptbl);
365 KASSERT(((vm_offset_t)ptbl >= VM_MIN_KERNEL_ADDRESS),
366 ("ptbl_unhold: non kva ptbl"));
367
368 /* decrement hold count */
369 for (i = 0; i < PTBL_PAGES; i++) {
370 pa = pte_vatopa(kernel_pmap,
371 (vm_offset_t)ptbl + (i * PAGE_SIZE));
372 m = PHYS_TO_VM_PAGE(pa);
373 m->ref_count--;
374 }
375
376 /*
377 * Free ptbl pages if there are no pte etries in this ptbl.
378 * ref_count has the same value for all ptbl pages, so check the last
379 * page.
380 */
381 if (m->ref_count == 0) {
382 ptbl_free(pmap, pdir_idx);
383
384 //debugf("ptbl_unhold: e (freed ptbl)\n");
385 return (1);
386 }
387
388 return (0);
389 }
390
391 /*
392 * Increment hold count for ptbl pages. This routine is used when a new pte
393 * entry is being inserted into the ptbl.
394 */
395 static void
ptbl_hold(pmap_t pmap,unsigned int pdir_idx)396 ptbl_hold(pmap_t pmap, unsigned int pdir_idx)
397 {
398 vm_paddr_t pa;
399 pte_t *ptbl;
400 vm_page_t m;
401 int i;
402
403 CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap,
404 pdir_idx);
405
406 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
407 ("ptbl_hold: invalid pdir_idx"));
408 KASSERT((pmap != kernel_pmap),
409 ("ptbl_hold: holding kernel ptbl!"));
410
411 ptbl = pmap->pm_pdir[pdir_idx];
412
413 KASSERT((ptbl != NULL), ("ptbl_hold: null ptbl"));
414
415 for (i = 0; i < PTBL_PAGES; i++) {
416 pa = pte_vatopa(kernel_pmap,
417 (vm_offset_t)ptbl + (i * PAGE_SIZE));
418 m = PHYS_TO_VM_PAGE(pa);
419 m->ref_count++;
420 }
421 }
422
423 /*
424 * Clean pte entry, try to free page table page if requested.
425 *
426 * Return 1 if ptbl pages were freed, otherwise return 0.
427 */
428 static int
pte_remove(pmap_t pmap,vm_offset_t va,uint8_t flags)429 pte_remove(pmap_t pmap, vm_offset_t va, uint8_t flags)
430 {
431 unsigned int pdir_idx = PDIR_IDX(va);
432 unsigned int ptbl_idx = PTBL_IDX(va);
433 vm_page_t m;
434 pte_t *ptbl;
435 pte_t *pte;
436
437 //int su = (pmap == kernel_pmap);
438 //debugf("pte_remove: s (su = %d pmap = 0x%08x va = 0x%08x flags = %d)\n",
439 // su, (u_int32_t)pmap, va, flags);
440
441 ptbl = pmap->pm_pdir[pdir_idx];
442 KASSERT(ptbl, ("pte_remove: null ptbl"));
443
444 pte = &ptbl[ptbl_idx];
445
446 if (pte == NULL || !PTE_ISVALID(pte))
447 return (0);
448
449 if (PTE_ISWIRED(pte))
450 pmap->pm_stats.wired_count--;
451
452 /* Get vm_page_t for mapped pte. */
453 m = PHYS_TO_VM_PAGE(PTE_PA(pte));
454
455 /* Handle managed entry. */
456 if (PTE_ISMANAGED(pte)) {
457 if (PTE_ISMODIFIED(pte))
458 vm_page_dirty(m);
459
460 if (PTE_ISREFERENCED(pte))
461 vm_page_aflag_set(m, PGA_REFERENCED);
462
463 pv_remove(pmap, va, m);
464 } else if (pmap == kernel_pmap && m && m->md.pv_tracked) {
465 /*
466 * Always pv_insert()/pv_remove() on MPC85XX, in case DPAA is
467 * used. This is needed by the NCSW support code for fast
468 * VA<->PA translation.
469 */
470 pv_remove(pmap, va, m);
471 if (TAILQ_EMPTY(&m->md.pv_list))
472 m->md.pv_tracked = false;
473 }
474
475 mtx_lock_spin(&tlbivax_mutex);
476 tlb_miss_lock();
477
478 tlb0_flush_entry(va);
479 *pte = 0;
480
481 tlb_miss_unlock();
482 mtx_unlock_spin(&tlbivax_mutex);
483
484 pmap->pm_stats.resident_count--;
485
486 if (flags & PTBL_UNHOLD) {
487 //debugf("pte_remove: e (unhold)\n");
488 return (ptbl_unhold(pmap, pdir_idx));
489 }
490
491 //debugf("pte_remove: e\n");
492 return (0);
493 }
494
495 /*
496 * Insert PTE for a given page and virtual address.
497 */
498 static int
pte_enter(pmap_t pmap,vm_page_t m,vm_offset_t va,uint32_t flags,bool nosleep)499 pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
500 bool nosleep)
501 {
502 unsigned int pdir_idx = PDIR_IDX(va);
503 unsigned int ptbl_idx = PTBL_IDX(va);
504 pte_t *ptbl, *pte, pte_tmp;
505
506 CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__,
507 pmap == kernel_pmap, pmap, va);
508
509 /* Get the page table pointer. */
510 ptbl = pmap->pm_pdir[pdir_idx];
511
512 if (ptbl == NULL) {
513 /* Allocate page table pages. */
514 ptbl = ptbl_alloc(pmap, pdir_idx, nosleep);
515 if (ptbl == NULL) {
516 KASSERT(nosleep, ("nosleep and NULL ptbl"));
517 return (ENOMEM);
518 }
519 pmap->pm_pdir[pdir_idx] = ptbl;
520 pte = &ptbl[ptbl_idx];
521 } else {
522 /*
523 * Check if there is valid mapping for requested
524 * va, if there is, remove it.
525 */
526 pte = &pmap->pm_pdir[pdir_idx][ptbl_idx];
527 if (PTE_ISVALID(pte)) {
528 pte_remove(pmap, va, PTBL_HOLD);
529 } else {
530 /*
531 * pte is not used, increment hold count
532 * for ptbl pages.
533 */
534 if (pmap != kernel_pmap)
535 ptbl_hold(pmap, pdir_idx);
536 }
537 }
538
539 /*
540 * Insert pv_entry into pv_list for mapped page if part of managed
541 * memory.
542 */
543 if ((m->oflags & VPO_UNMANAGED) == 0) {
544 flags |= PTE_MANAGED;
545
546 /* Create and insert pv entry. */
547 pv_insert(pmap, va, m);
548 }
549
550 pmap->pm_stats.resident_count++;
551
552 pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
553 pte_tmp |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */
554
555 mtx_lock_spin(&tlbivax_mutex);
556 tlb_miss_lock();
557
558 tlb0_flush_entry(va);
559 *pte = pte_tmp;
560
561 tlb_miss_unlock();
562 mtx_unlock_spin(&tlbivax_mutex);
563 return (0);
564 }
565
566 /* Return the pa for the given pmap/va. */
567 static vm_paddr_t
pte_vatopa(pmap_t pmap,vm_offset_t va)568 pte_vatopa(pmap_t pmap, vm_offset_t va)
569 {
570 vm_paddr_t pa = 0;
571 pte_t *pte;
572
573 pte = pte_find(pmap, va);
574 if ((pte != NULL) && PTE_ISVALID(pte))
575 pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
576 return (pa);
577 }
578
579 /* Get a pointer to a PTE in a page table. */
580 static pte_t *
pte_find(pmap_t pmap,vm_offset_t va)581 pte_find(pmap_t pmap, vm_offset_t va)
582 {
583 unsigned int pdir_idx = PDIR_IDX(va);
584 unsigned int ptbl_idx = PTBL_IDX(va);
585
586 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
587
588 if (pmap->pm_pdir[pdir_idx])
589 return (&(pmap->pm_pdir[pdir_idx][ptbl_idx]));
590
591 return (NULL);
592 }
593
594 /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */
595 static __inline pte_t *
pte_find_next(pmap_t pmap,vm_offset_t * pva)596 pte_find_next(pmap_t pmap, vm_offset_t *pva)
597 {
598 vm_offset_t va;
599 pte_t **pdir;
600 pte_t *pte;
601 unsigned long i, j;
602
603 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
604
605 va = *pva;
606 i = PDIR_IDX(va);
607 j = PTBL_IDX(va);
608 pdir = pmap->pm_pdir;
609 for (; i < PDIR_NENTRIES; i++, j = 0) {
610 if (pdir[i] == NULL)
611 continue;
612 for (; j < PTBL_NENTRIES; j++) {
613 pte = &pdir[i][j];
614 if (!PTE_ISVALID(pte))
615 continue;
616 *pva = PDIR_SIZE * i + PAGE_SIZE * j;
617 return (pte);
618 }
619 }
620 return (NULL);
621 }
622
623 /* Set up kernel page tables. */
624 static void
kernel_pte_alloc(vm_offset_t data_end,vm_offset_t addr)625 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
626 {
627 pte_t *pte;
628 vm_offset_t va;
629 vm_offset_t pdir_start;
630 int i;
631
632 kptbl_min = VM_MIN_KERNEL_ADDRESS / PDIR_SIZE;
633 kernel_pmap->pm_pdir = (pte_t **)kernel_ptbl_root;
634
635 pdir_start = kernel_ptbl_root + PDIR_NENTRIES * sizeof(pte_t);
636
637 /* Initialize kernel pdir */
638 for (i = 0; i < kernel_ptbls; i++) {
639 kernel_pmap->pm_pdir[kptbl_min + i] =
640 (pte_t *)(pdir_start + (i * PAGE_SIZE * PTBL_PAGES));
641 }
642
643 /*
644 * Fill in PTEs covering kernel code and data. They are not required
645 * for address translation, as this area is covered by static TLB1
646 * entries, but for pte_vatopa() to work correctly with kernel area
647 * addresses.
648 */
649 for (va = addr; va < data_end; va += PAGE_SIZE) {
650 pte = &(kernel_pmap->pm_pdir[PDIR_IDX(va)][PTBL_IDX(va)]);
651 powerpc_sync();
652 *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart));
653 *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED |
654 PTE_VALID | PTE_PS_4KB;
655 }
656 }
657
658 static vm_offset_t
mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)659 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
660 {
661 /* Allocate space for ptbl_bufs. */
662 ptbl_bufs = (struct ptbl_buf *)data_end;
663 data_end += sizeof(struct ptbl_buf) * PTBL_BUFS;
664 debugf(" ptbl_bufs at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
665 (uintptr_t)ptbl_bufs, data_end);
666
667 data_end = round_page(data_end);
668
669 kernel_ptbl_root = data_end;
670 data_end += PDIR_NENTRIES * sizeof(pte_t*);
671
672 /* Allocate PTE tables for kernel KVA. */
673 kernel_ptbls = howmany(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
674 PDIR_SIZE);
675 data_end += kernel_ptbls * PTBL_PAGES * PAGE_SIZE;
676 debugf(" kernel ptbls: %d\n", kernel_ptbls);
677 debugf(" kernel pdir at %#jx end = %#jx\n",
678 (uintmax_t)kernel_ptbl_root, (uintmax_t)data_end);
679
680 return (data_end);
681 }
682
683 /*
684 * Initialize a preallocated and zeroed pmap structure,
685 * such as one in a vmspace structure.
686 */
687 static int
mmu_booke_pinit(pmap_t pmap)688 mmu_booke_pinit(pmap_t pmap)
689 {
690 int i;
691
692 CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
693 curthread->td_proc->p_pid, curthread->td_proc->p_comm);
694
695 KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
696
697 for (i = 0; i < MAXCPU; i++)
698 pmap->pm_tid[i] = TID_NONE;
699 CPU_ZERO(&kernel_pmap->pm_active);
700 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
701 pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK);
702 bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES);
703 TAILQ_INIT(&pmap->pm_ptbl_list);
704
705 return (1);
706 }
707
708 /*
709 * Release any resources held by the given physical map.
710 * Called when a pmap initialized by mmu_booke_pinit is being released.
711 * Should only be called if the map contains no valid mappings.
712 */
713 static void
mmu_booke_release(pmap_t pmap)714 mmu_booke_release(pmap_t pmap)
715 {
716
717 KASSERT(pmap->pm_stats.resident_count == 0,
718 ("pmap_release: pmap resident count %ld != 0",
719 pmap->pm_stats.resident_count));
720 uma_zfree(ptbl_root_zone, pmap->pm_pdir);
721 }
722
723 static void
mmu_booke_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)724 mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
725 {
726 pte_t *pte;
727 vm_paddr_t pa = 0;
728 int sync_sz, valid;
729 pmap_t pmap;
730 vm_page_t m;
731 vm_offset_t addr;
732 int active;
733
734 rw_wlock(&pvh_global_lock);
735 pmap = PCPU_GET(curpmap);
736 active = (pm == kernel_pmap || pm == pmap) ? 1 : 0;
737 while (sz > 0) {
738 PMAP_LOCK(pm);
739 pte = pte_find(pm, va);
740 valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
741 if (valid)
742 pa = PTE_PA(pte);
743 PMAP_UNLOCK(pm);
744 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
745 sync_sz = min(sync_sz, sz);
746 if (valid) {
747 if (!active) {
748 /*
749 * Create a mapping in the active pmap.
750 *
751 * XXX: We use the zero page here, because
752 * it isn't likely to be in use.
753 * If we ever decide to support
754 * security.bsd.map_at_zero on Book-E, change
755 * this to some other address that isn't
756 * normally mappable.
757 */
758 addr = 0;
759 m = PHYS_TO_VM_PAGE(pa);
760 PMAP_LOCK(pmap);
761 pte_enter(pmap, m, addr,
762 PTE_SR | PTE_VALID, false);
763 __syncicache((void *)(addr + (va & PAGE_MASK)),
764 sync_sz);
765 pte_remove(pmap, addr, PTBL_UNHOLD);
766 PMAP_UNLOCK(pmap);
767 } else
768 __syncicache((void *)va, sync_sz);
769 }
770 va += sync_sz;
771 sz -= sync_sz;
772 }
773 rw_wunlock(&pvh_global_lock);
774 }
775
776 /*
777 * mmu_booke_zero_page_area zeros the specified hardware page by
778 * mapping it into virtual memory and using bzero to clear
779 * its contents.
780 *
781 * off and size must reside within a single page.
782 */
783 static void
mmu_booke_zero_page_area(vm_page_t m,int off,int size)784 mmu_booke_zero_page_area(vm_page_t m, int off, int size)
785 {
786 vm_offset_t va;
787
788 /* XXX KASSERT off and size are within a single page? */
789
790 mtx_lock(&zero_page_mutex);
791 va = zero_page_va;
792
793 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m));
794 bzero((caddr_t)va + off, size);
795 mmu_booke_kremove(va);
796
797 mtx_unlock(&zero_page_mutex);
798 }
799
800 /*
801 * mmu_booke_zero_page zeros the specified hardware page.
802 */
803 static void
mmu_booke_zero_page(vm_page_t m)804 mmu_booke_zero_page(vm_page_t m)
805 {
806 vm_offset_t off, va;
807
808 va = zero_page_va;
809 mtx_lock(&zero_page_mutex);
810
811 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m));
812
813 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
814 __asm __volatile("dcbz 0,%0" :: "r"(va + off));
815
816 mmu_booke_kremove(va);
817
818 mtx_unlock(&zero_page_mutex);
819 }
820
821 /*
822 * mmu_booke_copy_page copies the specified (machine independent) page by
823 * mapping the page into virtual memory and using memcopy to copy the page,
824 * one machine dependent page at a time.
825 */
826 static void
mmu_booke_copy_page(vm_page_t sm,vm_page_t dm)827 mmu_booke_copy_page(vm_page_t sm, vm_page_t dm)
828 {
829 vm_offset_t sva, dva;
830
831 sva = copy_page_src_va;
832 dva = copy_page_dst_va;
833
834 mtx_lock(©_page_mutex);
835 mmu_booke_kenter(sva, VM_PAGE_TO_PHYS(sm));
836 mmu_booke_kenter(dva, VM_PAGE_TO_PHYS(dm));
837
838 memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
839
840 mmu_booke_kremove(dva);
841 mmu_booke_kremove(sva);
842 mtx_unlock(©_page_mutex);
843 }
844
845 static inline void
mmu_booke_copy_pages(vm_page_t * ma,vm_offset_t a_offset,vm_page_t * mb,vm_offset_t b_offset,int xfersize)846 mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
847 vm_page_t *mb, vm_offset_t b_offset, int xfersize)
848 {
849 void *a_cp, *b_cp;
850 vm_offset_t a_pg_offset, b_pg_offset;
851 int cnt;
852
853 mtx_lock(©_page_mutex);
854 while (xfersize > 0) {
855 a_pg_offset = a_offset & PAGE_MASK;
856 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
857 mmu_booke_kenter(copy_page_src_va,
858 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]));
859 a_cp = (char *)copy_page_src_va + a_pg_offset;
860 b_pg_offset = b_offset & PAGE_MASK;
861 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
862 mmu_booke_kenter(copy_page_dst_va,
863 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]));
864 b_cp = (char *)copy_page_dst_va + b_pg_offset;
865 bcopy(a_cp, b_cp, cnt);
866 mmu_booke_kremove(copy_page_dst_va);
867 mmu_booke_kremove(copy_page_src_va);
868 a_offset += cnt;
869 b_offset += cnt;
870 xfersize -= cnt;
871 }
872 mtx_unlock(©_page_mutex);
873 }
874
875 static vm_offset_t
mmu_booke_quick_enter_page(vm_page_t m)876 mmu_booke_quick_enter_page(vm_page_t m)
877 {
878 vm_paddr_t paddr;
879 vm_offset_t qaddr;
880 uint32_t flags;
881 pte_t *pte;
882
883 paddr = VM_PAGE_TO_PHYS(m);
884
885 flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID;
886 flags |= tlb_calc_wimg(paddr, pmap_page_get_memattr(m)) << PTE_MAS2_SHIFT;
887 flags |= PTE_PS_4KB;
888
889 critical_enter();
890 qaddr = PCPU_GET(qmap_addr);
891
892 pte = pte_find(kernel_pmap, qaddr);
893
894 KASSERT(*pte == 0, ("mmu_booke_quick_enter_page: PTE busy"));
895
896 /*
897 * XXX: tlbivax is broadcast to other cores, but qaddr should
898 * not be present in other TLBs. Is there a better instruction
899 * sequence to use? Or just forget it & use mmu_booke_kenter()...
900 */
901 __asm __volatile("tlbivax 0, %0" :: "r"(qaddr & MAS2_EPN_MASK));
902 __asm __volatile("isync; msync");
903
904 *pte = PTE_RPN_FROM_PA(paddr) | flags;
905
906 /* Flush the real memory from the instruction cache. */
907 if ((flags & (PTE_I | PTE_G)) == 0)
908 __syncicache((void *)qaddr, PAGE_SIZE);
909
910 return (qaddr);
911 }
912
913 static void
mmu_booke_quick_remove_page(vm_offset_t addr)914 mmu_booke_quick_remove_page(vm_offset_t addr)
915 {
916 pte_t *pte;
917
918 pte = pte_find(kernel_pmap, addr);
919
920 KASSERT(PCPU_GET(qmap_addr) == addr,
921 ("mmu_booke_quick_remove_page: invalid address"));
922 KASSERT(*pte != 0,
923 ("mmu_booke_quick_remove_page: PTE not in use"));
924
925 *pte = 0;
926 critical_exit();
927 }
928
929 /**************************************************************************/
930 /* TID handling */
931 /**************************************************************************/
932
933 /*
934 * Invalidate all TLB0 entries which match the given TID. Note this is
935 * dedicated for cases when invalidations should NOT be propagated to other
936 * CPUs.
937 */
938 static void
tid_flush(tlbtid_t tid)939 tid_flush(tlbtid_t tid)
940 {
941 register_t msr;
942 uint32_t mas0, mas1, mas2;
943 int entry, way;
944
945 /* Don't evict kernel translations */
946 if (tid == TID_KERNEL)
947 return;
948
949 msr = mfmsr();
950 __asm __volatile("wrteei 0");
951
952 /*
953 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use
954 * it for PID invalidation.
955 */
956 switch ((mfpvr() >> 16) & 0xffff) {
957 case FSL_E500mc:
958 case FSL_E5500:
959 case FSL_E6500:
960 mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT);
961 /* tlbilxpid */
962 __asm __volatile("isync; .long 0x7c200024; isync; msync");
963 __asm __volatile("wrtee %0" :: "r"(msr));
964 return;
965 }
966
967 for (way = 0; way < TLB0_WAYS; way++)
968 for (entry = 0; entry < TLB0_ENTRIES_PER_WAY; entry++) {
969 mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way);
970 mtspr(SPR_MAS0, mas0);
971
972 mas2 = entry << MAS2_TLB0_ENTRY_IDX_SHIFT;
973 mtspr(SPR_MAS2, mas2);
974
975 __asm __volatile("isync; tlbre");
976
977 mas1 = mfspr(SPR_MAS1);
978
979 if (!(mas1 & MAS1_VALID))
980 continue;
981 if (((mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT) != tid)
982 continue;
983 mas1 &= ~MAS1_VALID;
984 mtspr(SPR_MAS1, mas1);
985 __asm __volatile("isync; tlbwe; isync; msync");
986 }
987 __asm __volatile("wrtee %0" :: "r"(msr));
988 }
989