1 /* $NetBSD: mm.c,v 1.16 2012/02/21 21:57:06 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2008, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Christos Zoulas, Joerg Sonnenberger and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Special /dev/{mem,kmem,zero,null} memory devices. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: mm.c,v 1.16 2012/02/21 21:57:06 rmind Exp $"); 38 39 #include "opt_compat_netbsd.h" 40 41 #include <sys/param.h> 42 #include <sys/conf.h> 43 #include <sys/ioctl.h> 44 #include <sys/mman.h> 45 #include <sys/uio.h> 46 #include <sys/termios.h> 47 48 #include <dev/mm.h> 49 50 #include <uvm/uvm_extern.h> 51 52 static void * dev_zero_page __read_mostly; 53 static kmutex_t dev_mem_lock __cacheline_aligned; 54 static vaddr_t dev_mem_addr __read_mostly; 55 56 static dev_type_read(mm_readwrite); 57 static dev_type_ioctl(mm_ioctl); 58 static dev_type_mmap(mm_mmap); 59 static dev_type_ioctl(mm_ioctl); 60 61 const struct cdevsw mem_cdevsw = { 62 #ifdef __HAVE_MM_MD_OPEN 63 mm_md_open, 64 #else 65 nullopen, 66 #endif 67 nullclose, mm_readwrite, mm_readwrite, 68 mm_ioctl, nostop, notty, nopoll, mm_mmap, nokqfilter, 69 D_MPSAFE 70 }; 71 72 #ifdef pmax /* XXX */ 73 const struct cdevsw mem_ultrix_cdevsw = { 74 nullopen, nullclose, mm_readwrite, mm_readwrite, mm_ioctl, 75 nostop, notty, nopoll, mm_mmap, nokqfilter, D_MPSAFE 76 }; 77 #endif 78 79 /* 80 * mm_init: initialize memory device driver. 81 */ 82 void 83 mm_init(void) 84 { 85 vaddr_t pg; 86 87 mutex_init(&dev_mem_lock, MUTEX_DEFAULT, IPL_NONE); 88 89 /* Read-only zero-page. */ 90 pg = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 91 KASSERT(pg != 0); 92 pmap_protect(pmap_kernel(), pg, pg + PAGE_SIZE, VM_PROT_READ); 93 pmap_update(pmap_kernel()); 94 dev_zero_page = (void *)pg; 95 96 #ifndef __HAVE_MM_MD_CACHE_ALIASING 97 /* KVA for mappings during I/O. */ 98 dev_mem_addr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 99 UVM_KMF_VAONLY|UVM_KMF_WAITVA); 100 KASSERT(dev_mem_addr != 0); 101 #else 102 dev_mem_addr = 0; 103 #endif 104 } 105 106 107 /* 108 * dev_mem_getva: get a special virtual address. If architecture requires, 109 * allocate VA according to PA, which avoids cache-aliasing issues. Use a 110 * constant, general mapping address otherwise. 111 */ 112 static inline vaddr_t 113 dev_mem_getva(paddr_t pa) 114 { 115 #ifdef __HAVE_MM_MD_CACHE_ALIASING 116 const vsize_t coloroff = trunc_page(pa) & ptoa(uvmexp.colormask); 117 const vaddr_t kva = uvm_km_alloc(kernel_map, PAGE_SIZE + coloroff, 118 ptoa(uvmexp.ncolors), UVM_KMF_VAONLY | UVM_KMF_WAITVA); 119 120 return kva + coloroff; 121 #else 122 return dev_mem_addr; 123 #endif 124 } 125 126 static inline void 127 dev_mem_relva(paddr_t pa, vaddr_t va) 128 { 129 #ifdef __HAVE_MM_MD_CACHE_ALIASING 130 const vsize_t coloroff = trunc_page(pa) & ptoa(uvmexp.colormask); 131 const vaddr_t origva = va - coloroff; 132 133 uvm_km_free(kernel_map, origva, PAGE_SIZE + coloroff, UVM_KMF_VAONLY); 134 #else 135 KASSERT(dev_mem_addr == va); 136 #endif 137 } 138 139 /* 140 * dev_kmem_readwrite: helper for DEV_MEM (/dev/mem) case of R/W. 141 */ 142 static int 143 dev_mem_readwrite(struct uio *uio, struct iovec *iov) 144 { 145 paddr_t paddr; 146 vaddr_t vaddr; 147 vm_prot_t prot; 148 size_t len, offset; 149 bool have_direct; 150 int error; 151 152 /* Check for wrap around. */ 153 if ((intptr_t)uio->uio_offset != uio->uio_offset) { 154 return EFAULT; 155 } 156 paddr = uio->uio_offset & ~PAGE_MASK; 157 prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ; 158 error = mm_md_physacc(paddr, prot); 159 if (error) { 160 return error; 161 } 162 offset = uio->uio_offset & PAGE_MASK; 163 len = MIN(uio->uio_resid, PAGE_SIZE - offset); 164 165 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 166 /* Is physical address directly mapped? Return VA. */ 167 have_direct = mm_md_direct_mapped_phys(paddr, &vaddr); 168 #else 169 vaddr = 0; 170 have_direct = false; 171 #endif 172 if (!have_direct) { 173 /* Get a special virtual address. */ 174 const vaddr_t va = dev_mem_getva(paddr); 175 176 /* Map selected KVA to physical address. */ 177 mutex_enter(&dev_mem_lock); 178 pmap_kenter_pa(va, paddr, prot, 0); 179 pmap_update(pmap_kernel()); 180 181 /* Perform I/O. */ 182 vaddr = va + offset; 183 error = uiomove((void *)vaddr, len, uio); 184 185 /* Unmap, flush before unlock. */ 186 pmap_kremove(va, PAGE_SIZE); 187 pmap_update(pmap_kernel()); 188 mutex_exit(&dev_mem_lock); 189 190 /* "Release" the virtual address. */ 191 dev_mem_relva(paddr, va); 192 } else { 193 /* Direct map, just perform I/O. */ 194 vaddr += offset; 195 error = uiomove((void *)vaddr, len, uio); 196 } 197 return error; 198 } 199 200 /* 201 * dev_kmem_readwrite: helper for DEV_KMEM (/dev/kmem) case of R/W. 202 */ 203 static int 204 dev_kmem_readwrite(struct uio *uio, struct iovec *iov) 205 { 206 void *addr; 207 size_t len, offset; 208 vm_prot_t prot; 209 int error; 210 bool md_kva; 211 212 /* Check for wrap around. */ 213 addr = (void *)(intptr_t)uio->uio_offset; 214 if ((uintptr_t)addr != uio->uio_offset) { 215 return EFAULT; 216 } 217 /* 218 * Handle non-page aligned offset. 219 * Otherwise, we operate in page-by-page basis. 220 */ 221 offset = uio->uio_offset & PAGE_MASK; 222 len = MIN(uio->uio_resid, PAGE_SIZE - offset); 223 prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ; 224 225 md_kva = false; 226 227 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_IO 228 paddr_t paddr; 229 /* MD case: is this is a directly mapped address? */ 230 if (mm_md_direct_mapped_io(addr, &paddr)) { 231 /* If so, validate physical address. */ 232 error = mm_md_physacc(paddr, prot); 233 if (error) { 234 return error; 235 } 236 md_kva = true; 237 } 238 #endif 239 if (!md_kva) { 240 bool checked = false; 241 242 #ifdef __HAVE_MM_MD_KERNACC 243 /* MD check for the address. */ 244 error = mm_md_kernacc(addr, prot, &checked); 245 if (error) { 246 return error; 247 } 248 #endif 249 /* UVM check for the address (unless MD indicated to not). */ 250 if (!checked && !uvm_kernacc(addr, len, prot)) { 251 return EFAULT; 252 } 253 } 254 error = uiomove(addr, len, uio); 255 return error; 256 } 257 258 /* 259 * dev_zero_readwrite: helper for DEV_ZERO (/dev/null) case of R/W. 260 */ 261 static inline int 262 dev_zero_readwrite(struct uio *uio, struct iovec *iov) 263 { 264 size_t len; 265 266 /* Nothing to do for the write case. */ 267 if (uio->uio_rw == UIO_WRITE) { 268 uio->uio_resid = 0; 269 return 0; 270 } 271 /* 272 * Read in page-by-page basis, caller will continue. 273 * Cut appropriately for a single/last-iteration cases. 274 */ 275 len = MIN(iov->iov_len, PAGE_SIZE); 276 return uiomove(dev_zero_page, len, uio); 277 } 278 279 /* 280 * mm_readwrite: general memory R/W function. 281 */ 282 static int 283 mm_readwrite(dev_t dev, struct uio *uio, int flags) 284 { 285 struct iovec *iov; 286 int error; 287 288 #ifdef __HAVE_MM_MD_READWRITE 289 /* If defined - there are extra MD cases. */ 290 switch (minor(dev)) { 291 case DEV_MEM: 292 case DEV_KMEM: 293 case DEV_NULL: 294 case DEV_ZERO: 295 #if defined(COMPAT_16) && defined(__arm) 296 case _DEV_ZERO_oARM: 297 #endif 298 break; 299 default: 300 return mm_md_readwrite(dev, uio); 301 } 302 #endif 303 error = 0; 304 while (uio->uio_resid > 0 && error == 0) { 305 iov = uio->uio_iov; 306 if (iov->iov_len == 0) { 307 /* Processed; next I/O vector. */ 308 uio->uio_iov++; 309 uio->uio_iovcnt--; 310 KASSERT(uio->uio_iovcnt >= 0); 311 continue; 312 } 313 /* Helper functions will process in page-by-page basis. */ 314 switch (minor(dev)) { 315 case DEV_MEM: 316 error = dev_mem_readwrite(uio, iov); 317 break; 318 case DEV_KMEM: 319 error = dev_kmem_readwrite(uio, iov); 320 break; 321 case DEV_NULL: 322 if (uio->uio_rw == UIO_WRITE) { 323 uio->uio_resid = 0; 324 } 325 /* Break directly out of the loop. */ 326 return 0; 327 #if defined(COMPAT_16) && defined(__arm) 328 case _DEV_ZERO_oARM: 329 #endif 330 case DEV_ZERO: 331 error = dev_zero_readwrite(uio, iov); 332 break; 333 default: 334 error = ENXIO; 335 break; 336 } 337 } 338 return error; 339 } 340 341 /* 342 * mm_mmap: general mmap() handler. 343 */ 344 static paddr_t 345 mm_mmap(dev_t dev, off_t off, int acc) 346 { 347 vm_prot_t prot; 348 349 #ifdef __HAVE_MM_MD_MMAP 350 /* If defined - there are extra mmap() MD cases. */ 351 switch (minor(dev)) { 352 case DEV_MEM: 353 case DEV_KMEM: 354 case DEV_NULL: 355 #if defined(COMPAT_16) && defined(__arm) 356 case _DEV_ZERO_oARM: 357 #endif 358 case DEV_ZERO: 359 break; 360 default: 361 return mm_md_mmap(dev, off, acc); 362 } 363 #endif 364 /* 365 * /dev/null does not make sense, /dev/kmem is volatile and 366 * /dev/zero is handled in mmap already. 367 */ 368 if (minor(dev) != DEV_MEM) { 369 return -1; 370 } 371 372 prot = 0; 373 if (acc & PROT_EXEC) 374 prot |= VM_PROT_EXECUTE; 375 if (acc & PROT_READ) 376 prot |= VM_PROT_READ; 377 if (acc & PROT_WRITE) 378 prot |= VM_PROT_WRITE; 379 380 /* Validate the physical address. */ 381 if (mm_md_physacc(off, prot) != 0) { 382 return -1; 383 } 384 return off >> PGSHIFT; 385 } 386 387 static int 388 mm_ioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 389 { 390 391 switch (cmd) { 392 case FIONBIO: 393 /* We never block anyway. */ 394 return 0; 395 396 case FIOSETOWN: 397 case FIOGETOWN: 398 case TIOCGPGRP: 399 case TIOCSPGRP: 400 case TIOCGETA: 401 return ENOTTY; 402 403 case FIOASYNC: 404 if ((*(int *)data) == 0) { 405 return 0; 406 } 407 /* FALLTHROUGH */ 408 default: 409 return EOPNOTSUPP; 410 } 411 } 412