1 /* $NetBSD: pmap_segtab.c,v 1.2 2015/06/11 08:04:44 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap_segtab.c,v 1.2 2015/06/11 08:04:44 matt Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #define __PMAP_PRIVATE 99 100 #include "opt_multiprocessor.h" 101 102 #include <sys/param.h> 103 #include <sys/systm.h> 104 #include <sys/proc.h> 105 #include <sys/mutex.h> 106 #include <sys/atomic.h> 107 108 #include <uvm/uvm.h> 109 110 CTASSERT(NBPG >= sizeof(pmap_segtab_t)); 111 112 struct pmap_segtab_info { 113 pmap_segtab_t *free_segtab; /* free list kept locally */ 114 #ifdef DEBUG 115 uint32_t nget_segtab; 116 uint32_t nput_segtab; 117 uint32_t npage_segtab; 118 #define SEGTAB_ADD(n, v) (pmap_segtab_info.n ## _segtab += (v)) 119 #else 120 #define SEGTAB_ADD(n, v) ((void) 0) 121 #endif 122 #ifdef PMAP_PTP_CACHE 123 struct pgflist ptp_pgflist; /* Keep a list of idle page tables. */ 124 #endif 125 } pmap_segtab_info = { 126 #ifdef PMAP_PTP_CACHE 127 .ptp_pgflist = LIST_HEAD_INITIALIZER(pmap_segtab_info.ptp_pgflist), 128 #endif 129 }; 130 131 kmutex_t pmap_segtab_lock __cacheline_aligned; 132 133 static inline struct vm_page * 134 pmap_pte_pagealloc(void) 135 { 136 struct vm_page *pg; 137 138 pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_ZERO|UVM_PGA_USERESERVE); 139 if (pg) { 140 #ifdef UVM_PAGE_TRKOWN 141 pg->owner_tag = NULL; 142 #endif 143 UVM_PAGE_OWN(pg, "pmap-ptp"); 144 } 145 146 return pg; 147 } 148 149 static inline pt_entry_t * 150 pmap_segmap(struct pmap *pmap, vaddr_t va) 151 { 152 pmap_segtab_t *stp = pmap->pm_segtab; 153 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 154 #ifdef _LP64 155 stp = stp->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)]; 156 if (stp == NULL) 157 return NULL; 158 #endif 159 160 return stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)]; 161 } 162 163 pt_entry_t * 164 pmap_pte_lookup(pmap_t pmap, vaddr_t va) 165 { 166 pt_entry_t *pte = pmap_segmap(pmap, va); 167 if (pte == NULL) 168 return NULL; 169 170 return pte + ((va >> PGSHIFT) & (NPTEPG - 1)); 171 } 172 173 static void 174 pmap_segtab_free(pmap_segtab_t *stp) 175 { 176 /* 177 * Insert the the segtab into the segtab freelist. 178 */ 179 mutex_spin_enter(&pmap_segtab_lock); 180 stp->seg_seg[0] = pmap_segtab_info.free_segtab; 181 pmap_segtab_info.free_segtab = stp; 182 SEGTAB_ADD(nput, 1); 183 mutex_spin_exit(&pmap_segtab_lock); 184 } 185 186 static void 187 pmap_segtab_release(pmap_t pmap, pmap_segtab_t **stp_p, bool free_stp, 188 pte_callback_t callback, uintptr_t flags, 189 vaddr_t va, vsize_t vinc) 190 { 191 pmap_segtab_t *stp = *stp_p; 192 193 for (size_t i = va / vinc; i < PMAP_SEGTABSIZE; i++, va += vinc) { 194 #ifdef _LP64 195 if (vinc > NBSEG) { 196 if (stp->seg_seg[i] != NULL) { 197 pmap_segtab_release(pmap, &stp->seg_seg[i], 198 true, callback, flags, va, vinc / NSEGPG); 199 KASSERT(stp->seg_seg[i] == NULL); 200 } 201 continue; 202 } 203 #endif 204 KASSERT(vinc == NBSEG); 205 206 /* get pointer to segment map */ 207 pt_entry_t *pte = stp->seg_tab[i]; 208 if (pte == NULL) 209 continue; 210 211 /* 212 * If our caller want a callback, do so. 213 */ 214 if (callback != NULL) { 215 (*callback)(pmap, va, va + vinc, pte, flags); 216 } 217 #ifdef DEBUG 218 for (size_t j = 0; j < NPTEPG; j++) { 219 if (!pte_zero_p(pte[j])) 220 panic("%s: pte entry %p not 0 (%#"PRIxPTE")", 221 __func__, &pte[j], pte_value(pte[j])); 222 } 223 #endif 224 // PMAP_UNMAP_POOLPAGE should handle any VCA issues itself 225 paddr_t pa = PMAP_UNMAP_POOLPAGE((vaddr_t)pte); 226 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 227 #ifdef PMAP_PTP_CACHE 228 mutex_spin_enter(&pmap_segtab_lock); 229 LIST_INSERT_HEAD(&pmap_segtab_info.ptp_pgflist, pg, listq.list); 230 mutex_spin_exit(&pmap_segtab_lock); 231 #else 232 uvm_pagefree(pg); 233 #endif 234 235 stp->seg_tab[i] = NULL; 236 } 237 238 if (free_stp) { 239 pmap_segtab_free(stp); 240 *stp_p = NULL; 241 } 242 } 243 244 /* 245 * Create and return a physical map. 246 * 247 * If the size specified for the map 248 * is zero, the map is an actual physical 249 * map, and may be referenced by the 250 * hardware. 251 * 252 * If the size specified is non-zero, 253 * the map will be used in software only, and 254 * is bounded by that size. 255 */ 256 static pmap_segtab_t * 257 pmap_segtab_alloc(void) 258 { 259 pmap_segtab_t *stp; 260 261 again: 262 mutex_spin_enter(&pmap_segtab_lock); 263 if (__predict_true((stp = pmap_segtab_info.free_segtab) != NULL)) { 264 pmap_segtab_info.free_segtab = stp->seg_seg[0]; 265 stp->seg_seg[0] = NULL; 266 SEGTAB_ADD(nget, 1); 267 } 268 mutex_spin_exit(&pmap_segtab_lock); 269 270 if (__predict_false(stp == NULL)) { 271 struct vm_page * const stp_pg = pmap_pte_pagealloc(); 272 273 if (__predict_false(stp_pg == NULL)) { 274 /* 275 * XXX What else can we do? Could we deadlock here? 276 */ 277 uvm_wait("pmap_create"); 278 goto again; 279 } 280 SEGTAB_ADD(npage, 1); 281 const paddr_t stp_pa = VM_PAGE_TO_PHYS(stp_pg); 282 283 stp = (pmap_segtab_t *)PMAP_MAP_POOLPAGE(stp_pa); 284 const size_t n = NBPG / sizeof(*stp); 285 if (n > 1) { 286 /* 287 * link all the segtabs in this page together 288 */ 289 for (size_t i = 1; i < n - 1; i++) { 290 stp[i].seg_seg[0] = &stp[i+1]; 291 } 292 /* 293 * Now link the new segtabs into the free segtab list. 294 */ 295 mutex_spin_enter(&pmap_segtab_lock); 296 stp[n-1].seg_seg[0] = pmap_segtab_info.free_segtab; 297 pmap_segtab_info.free_segtab = stp + 1; 298 SEGTAB_ADD(nput, n - 1); 299 mutex_spin_exit(&pmap_segtab_lock); 300 } 301 } 302 303 #ifdef DEBUG 304 for (size_t i = 0; i < PMAP_SEGTABSIZE; i++) { 305 if (stp->seg_tab[i] != 0) 306 panic("%s: pm_segtab.seg_tab[%zu] != 0", __func__, i); 307 } 308 #endif 309 return stp; 310 } 311 312 /* 313 * Allocate the top segment table for the pmap. 314 */ 315 void 316 pmap_segtab_init(pmap_t pmap) 317 { 318 319 pmap->pm_segtab = pmap_segtab_alloc(); 320 } 321 322 /* 323 * Retire the given physical map from service. 324 * Should only be called if the map contains 325 * no valid mappings. 326 */ 327 void 328 pmap_segtab_destroy(pmap_t pmap, pte_callback_t func, uintptr_t flags) 329 { 330 if (pmap->pm_segtab == NULL) 331 return; 332 333 #ifdef _LP64 334 const vsize_t vinc = NBXSEG; 335 #else 336 const vsize_t vinc = NBSEG; 337 #endif 338 pmap_segtab_release(pmap, &pmap->pm_segtab, 339 func == NULL, func, flags, pmap->pm_minaddr, vinc); 340 } 341 342 /* 343 * Make a new pmap (vmspace) active for the given process. 344 */ 345 void 346 pmap_segtab_activate(struct pmap *pm, struct lwp *l) 347 { 348 if (l == curlwp) { 349 KASSERT(pm == l->l_proc->p_vmspace->vm_map.pmap); 350 if (pm == pmap_kernel()) { 351 l->l_cpu->ci_pmap_user_segtab = (void*)0xdeadbabe; 352 #ifdef _LP64 353 l->l_cpu->ci_pmap_user_seg0tab = (void*)0xdeadbabe; 354 #endif 355 } else { 356 l->l_cpu->ci_pmap_user_segtab = pm->pm_segtab; 357 #ifdef _LP64 358 l->l_cpu->ci_pmap_user_seg0tab = pm->pm_segtab->seg_seg[0]; 359 #endif 360 } 361 } 362 } 363 364 /* 365 * Act on the given range of addresses from the specified map. 366 * 367 * It is assumed that the start and end are properly rounded to 368 * the page size. 369 */ 370 void 371 pmap_pte_process(pmap_t pmap, vaddr_t sva, vaddr_t eva, 372 pte_callback_t callback, uintptr_t flags) 373 { 374 #if 0 375 printf("%s: %p, %"PRIxVADDR", %"PRIxVADDR", %p, %"PRIxPTR"\n", 376 __func__, pmap, sva, eva, callback, flags); 377 #endif 378 while (sva < eva) { 379 vaddr_t lastseg_va = pmap_trunc_seg(sva) + NBSEG; 380 if (lastseg_va == 0 || lastseg_va > eva) 381 lastseg_va = eva; 382 383 /* 384 * If VA belongs to an unallocated segment, 385 * skip to the next segment boundary. 386 */ 387 pt_entry_t * const pte = pmap_pte_lookup(pmap, sva); 388 if (pte != NULL) { 389 /* 390 * Callback to deal with the ptes for this segment. 391 */ 392 (*callback)(pmap, sva, lastseg_va, pte, flags); 393 } 394 /* 395 * In theory we could release pages with no entries, 396 * but that takes more effort than we want here. 397 */ 398 sva = lastseg_va; 399 } 400 } 401 402 /* 403 * Return a pointer for the pte that corresponds to the specified virtual 404 * address (va) in the target physical map, allocating if needed. 405 */ 406 pt_entry_t * 407 pmap_pte_reserve(pmap_t pmap, vaddr_t va, int flags) 408 { 409 pmap_segtab_t *stp = pmap->pm_segtab; 410 pt_entry_t *pte; 411 412 pte = pmap_pte_lookup(pmap, va); 413 if (__predict_false(pte == NULL)) { 414 #ifdef _LP64 415 pmap_segtab_t ** const stp_p = 416 &stp->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)]; 417 if (__predict_false((stp = *stp_p) == NULL)) { 418 pmap_segtab_t *nstp = pmap_segtab_alloc(); 419 #ifdef MULTIPROCESSOR 420 pmap_segtab_t *ostp = atomic_cas_ptr(stp_p, NULL, nstp); 421 if (__predict_false(ostp != NULL)) { 422 pmap_segtab_free(nstp); 423 nstp = ostp; 424 } 425 #else 426 *stp_p = nstp; 427 #endif /* MULTIPROCESSOR */ 428 stp = nstp; 429 } 430 KASSERT(stp == pmap->pm_segtab->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)]); 431 #endif /* _LP64 */ 432 struct vm_page *pg = NULL; 433 #ifdef PMAP_PTP_CACHE 434 mutex_spin_enter(&pmap_segtab_lock); 435 if ((pg = LIST_FIRST(&pmap_segtab_info.ptp_pgflist)) != NULL) { 436 LIST_REMOVE(pg, listq.list); 437 KASSERT(LIST_FIRST(&pmap_segtab_info.ptp_pgflist) != pg); 438 } 439 mutex_spin_exit(&pmap_segtab_lock); 440 #endif 441 if (pg == NULL) 442 pg = pmap_pte_pagealloc(); 443 if (pg == NULL) { 444 if (flags & PMAP_CANFAIL) 445 return NULL; 446 panic("%s: cannot allocate page table page " 447 "for va %" PRIxVADDR, __func__, va); 448 } 449 450 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 451 pte = (pt_entry_t *)PMAP_MAP_POOLPAGE(pa); 452 pt_entry_t ** const pte_p = 453 &stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)]; 454 #ifdef MULTIPROCESSOR 455 pt_entry_t *opte = atomic_cas_ptr(pte_p, NULL, pte); 456 /* 457 * If another thread allocated the segtab needed for this va 458 * free the page we just allocated. 459 */ 460 if (__predict_false(opte != NULL)) { 461 #ifdef PMAP_PTP_CACHE 462 mutex_spin_enter(&pmap_segtab_lock); 463 LIST_INSERT_HEAD(&pmap_segtab_info.ptp_pgflist, 464 pg, listq.list); 465 mutex_spin_exit(&pmap_segtab_lock); 466 #else 467 PMAP_UNMAP_POOLPAGE((vaddr_t)pte); 468 uvm_pagefree(pg); 469 #endif 470 pte = opte; 471 } 472 #else 473 *pte_p = pte; 474 #endif 475 KASSERT(pte == stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)]); 476 477 #ifdef DEBUG 478 for (size_t i = 0; i < NPTEPG; i++) { 479 if (!pte_zero_p(pte[i])) 480 panic("%s: new segmap %p not empty @ %zu", 481 __func__, pte, i); 482 } 483 #endif 484 pte += (va >> PGSHIFT) & (NPTEPG - 1); 485 } 486 487 return pte; 488 } 489