1 /* $NetBSD: subr_physmap.c,v 1.2 2013/01/19 01:04:51 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2013 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(1, "$NetBSD: subr_physmap.c,v 1.2 2013/01/19 01:04:51 rmind Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/physmap.h> 37 #include <sys/kmem.h> 38 39 #include <dev/mm.h> 40 41 /* 42 * This file contain support routines used to create and destroy lists of 43 * physical pages from lists of pages or ranges of virtual address. By using 44 * these physical maps, the kernel can avoid mapping physical I/O in the 45 * kernel's address space in most cases. 46 */ 47 48 typedef struct { 49 physmap_t *pc_physmap; 50 physmap_segment_t *pc_segs; 51 vsize_t pc_offset; 52 vsize_t pc_klen; 53 vaddr_t pc_kva; 54 u_int pc_nsegs; 55 vm_prot_t pc_prot; 56 bool pc_direct_mapped; 57 } physmap_cookie_t; 58 59 /* 60 * Allocate a physmap structure that requires "maxsegs" segments. 61 */ 62 static physmap_t * 63 physmap_alloc(size_t maxsegs) 64 { 65 const size_t mapsize = offsetof(physmap_t, pm_segs[maxsegs]); 66 67 KASSERT(maxsegs > 0); 68 69 physmap_t * const map = kmem_zalloc(mapsize, KM_SLEEP); 70 map->pm_maxsegs = maxsegs; 71 72 return map; 73 } 74 75 static int 76 physmap_fill(physmap_t *map, pmap_t pmap, vaddr_t va, vsize_t len) 77 { 78 size_t nsegs = map->pm_nsegs; 79 physmap_segment_t *ps = &map->pm_segs[nsegs]; 80 vsize_t offset = va - trunc_page(va); 81 82 if (nsegs == 0) { 83 if (!pmap_extract(pmap, va, &ps->ps_addr)) { 84 return EFAULT; 85 } 86 ps->ps_len = MIN(len, PAGE_SIZE - offset); 87 if (ps->ps_len == len) { 88 map->pm_nsegs = 1; 89 return 0; 90 } 91 offset = 0; 92 } else { 93 /* 94 * Backup to the last segment since we have to see if we can 95 * merge virtual addresses that are physically contiguous into 96 * as few segments as possible. 97 */ 98 ps--; 99 nsegs--; 100 } 101 102 paddr_t lastaddr = ps->ps_addr + ps->ps_len; 103 for (;;) { 104 paddr_t curaddr; 105 if (!pmap_extract(pmap, va, &curaddr)) { 106 return EFAULT; 107 } 108 if (curaddr != lastaddr) { 109 ps++; 110 nsegs++; 111 KASSERT(nsegs < map->pm_maxsegs); 112 ps->ps_addr = curaddr; 113 lastaddr = curaddr; 114 } 115 if (offset + len > PAGE_SIZE) { 116 ps->ps_len += PAGE_SIZE - offset; 117 lastaddr = ps->ps_addr + ps->ps_len; 118 len -= PAGE_SIZE - offset; 119 lastaddr += PAGE_SIZE - offset; 120 offset = 0; 121 } else { 122 ps->ps_len += len; 123 map->pm_nsegs = nsegs + 1; 124 return 0; 125 } 126 } 127 } 128 129 /* 130 * Create a physmap and populate it with the pages that are used to mapped 131 * linear range of virtual addresses. It is assumed that uvm_vslock has been 132 * called to lock these pages into memory. 133 */ 134 int 135 physmap_create_linear(physmap_t **map_p, const struct vmspace *vs, vaddr_t va, 136 vsize_t len) 137 { 138 const size_t maxsegs = atop(round_page(va + len) - trunc_page(va)); 139 physmap_t * const map = physmap_alloc(maxsegs); 140 int error = physmap_fill(map, vs->vm_map.pmap, va, len); 141 if (error) { 142 physmap_destroy(map); 143 *map_p = NULL; 144 return error; 145 } 146 *map_p = map; 147 return 0; 148 } 149 150 /* 151 * Create a physmap and populate it with the pages that are contained in an 152 * iovec array. It is assumed that uvm_vslock has been called to lock these 153 * pages into memory. 154 */ 155 int 156 physmap_create_iov(physmap_t **map_p, const struct vmspace *vs, 157 struct iovec *iov, size_t iovlen) 158 { 159 size_t maxsegs = 0; 160 for (size_t i = 0; i < iovlen; i++) { 161 const vaddr_t start = (vaddr_t) iov[i].iov_base; 162 const vaddr_t end = start + iov[i].iov_len; 163 maxsegs += atop(round_page(end) - trunc_page(start)); 164 } 165 physmap_t * const map = physmap_alloc(maxsegs); 166 167 for (size_t i = 0; i < iovlen; i++) { 168 int error = physmap_fill(map, vs->vm_map.pmap, 169 (vaddr_t) iov[i].iov_base, iov[i].iov_len); 170 if (error) { 171 physmap_destroy(map); 172 *map_p = NULL; 173 return error; 174 } 175 } 176 *map_p = map; 177 return 0; 178 } 179 180 /* 181 * This uses a list of vm_page structure to create a physmap. 182 */ 183 physmap_t * 184 physmap_create_pagelist(struct vm_page **pgs, size_t npgs) 185 { 186 physmap_t * const map = physmap_alloc(npgs); 187 188 physmap_segment_t *ps = map->pm_segs; 189 190 /* 191 * Initialize the first segment. 192 */ 193 paddr_t lastaddr = VM_PAGE_TO_PHYS(pgs[0]); 194 ps->ps_addr = lastaddr; 195 ps->ps_len = PAGE_SIZE; 196 197 for (pgs++; npgs-- > 1; pgs++) { 198 /* 199 * lastaddr needs to be increased by a page. 200 */ 201 lastaddr += PAGE_SIZE; 202 paddr_t curaddr = VM_PAGE_TO_PHYS(*pgs); 203 if (curaddr != lastaddr) { 204 /* 205 * If the addresses are not the same, we need to use 206 * a new segemnt. Set its address and update lastaddr. 207 */ 208 ps++; 209 ps->ps_addr = curaddr; 210 lastaddr = curaddr; 211 } 212 /* 213 * Increase this segment's length by a page 214 */ 215 ps->ps_len += PAGE_SIZE; 216 } 217 218 map->pm_nsegs = ps + 1 - map->pm_segs; 219 return map; 220 } 221 222 void 223 physmap_destroy(physmap_t *map) 224 { 225 const size_t mapsize = offsetof(physmap_t, pm_segs[map->pm_maxsegs]); 226 227 kmem_free(map, mapsize); 228 } 229 230 void * 231 physmap_map_init(physmap_t *map, size_t offset, vm_prot_t prot) 232 { 233 physmap_cookie_t * const pc = kmem_zalloc(sizeof(*pc), KM_SLEEP); 234 235 KASSERT(prot == VM_PROT_READ || prot == (VM_PROT_READ|VM_PROT_WRITE)); 236 237 pc->pc_physmap = map; 238 pc->pc_segs = map->pm_segs; 239 pc->pc_nsegs = map->pm_nsegs; 240 pc->pc_prot = prot; 241 pc->pc_klen = 0; 242 pc->pc_kva = 0; 243 pc->pc_direct_mapped = false; 244 245 /* 246 * Skip to the first segment we are interested in. 247 */ 248 while (offset >= pc->pc_segs->ps_len) { 249 offset -= pc->pc_segs->ps_len; 250 pc->pc_segs++; 251 pc->pc_nsegs--; 252 } 253 254 pc->pc_offset = offset; 255 256 return pc; 257 } 258 259 size_t 260 physmap_map(void *cookie, vaddr_t *kvap) 261 { 262 physmap_cookie_t * const pc = cookie; 263 264 /* 265 * If there is currently a non-direct mapped KVA region allocated, 266 * free it now. 267 */ 268 if (pc->pc_kva != 0 && !pc->pc_direct_mapped) { 269 pmap_kremove(pc->pc_kva, pc->pc_klen); 270 pmap_update(pmap_kernel()); 271 uvm_km_free(kernel_map, pc->pc_kva, pc->pc_klen, 272 UVM_KMF_VAONLY); 273 } 274 275 /* 276 * If there are no more segments to process, return 0 indicating 277 * we are done. 278 */ 279 if (pc->pc_nsegs == 0) { 280 return 0; 281 } 282 283 /* 284 * Get starting physical address of this segment and its length. 285 */ 286 paddr_t pa = pc->pc_segs->ps_addr + pc->pc_offset; 287 const size_t koff = pa & PAGE_MASK; 288 const size_t len = pc->pc_segs->ps_len - pc->pc_offset; 289 290 /* 291 * Now that we have the starting offset in the page, reset to the 292 * beginning of the page. 293 */ 294 pa = trunc_page(pa); 295 296 /* 297 * We are now done with this segment; advance to the next one. 298 */ 299 pc->pc_segs++; 300 pc->pc_nsegs--; 301 pc->pc_offset = 0; 302 303 /* 304 * Find out how many pages we are mapping. 305 */ 306 pc->pc_klen = round_page(len); 307 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 308 /* 309 * Always try to direct map it since that's nearly zero cost. 310 */ 311 pc->pc_direct_mapped = mm_md_direct_mapped_phys(pa, &pc->pc_kva); 312 #endif 313 if (!pc->pc_direct_mapped) { 314 /* 315 * If we can't direct map it, we have to allocate some KVA 316 * so we map it via the kernel_map. 317 */ 318 pc->pc_kva = uvm_km_alloc(kernel_map, pc->pc_klen, 319 atop(pa) & uvmexp.ncolors, 320 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 321 KASSERT(pc->pc_kva != 0); 322 323 /* 324 * Setup mappings for this segment. 325 */ 326 for (size_t poff = 0; poff < pc->pc_klen; poff += PAGE_SIZE) { 327 pmap_kenter_pa(pc->pc_kva + poff, pa + poff, 328 pc->pc_prot, 0); 329 } 330 /* 331 * Make them real. 332 */ 333 pmap_update(pmap_kernel()); 334 } 335 /* 336 * Return the starting KVA (including offset into the page) and 337 * the length of this segment. 338 */ 339 *kvap = pc->pc_kva + koff; 340 return len; 341 } 342 343 void 344 physmap_map_fini(void *cookie) 345 { 346 physmap_cookie_t * const pc = cookie; 347 348 /* 349 * If there is currently a non-direct mapped KVA region allocated, 350 * free it now. 351 */ 352 if (pc->pc_kva != 0 && !pc->pc_direct_mapped) { 353 pmap_kremove(pc->pc_kva, pc->pc_klen); 354 pmap_update(pmap_kernel()); 355 uvm_km_free(kernel_map, pc->pc_kva, pc->pc_klen, 356 UVM_KMF_VAONLY); 357 } 358 359 /* 360 * Free the cookie. 361 */ 362 kmem_free(pc, sizeof(*pc)); 363 } 364 365 /* 366 * genio needs to zero pages past the EOF or without backing storage (think 367 * sparse files). But since we are using physmaps, there is no kva to use with 368 * memset so we need a helper to obtain a kva and memset the desired memory. 369 */ 370 void 371 physmap_zero(physmap_t *map, size_t offset, size_t len) 372 { 373 void * const cookie = physmap_map_init(map, offset, 374 VM_PROT_READ|VM_PROT_WRITE); 375 376 for (;;) { 377 vaddr_t kva; 378 size_t seglen = physmap_map(cookie, &kva); 379 KASSERT(seglen != 0); 380 if (seglen > len) 381 seglen = len; 382 memset((void *)kva, 0, seglen); 383 if (seglen == len) 384 break; 385 } 386 387 physmap_map_fini(cookie); 388 } 389