1 /* 2 * (MPSAFE) 3 * 4 * Copyright (c) 1988 University of Utah. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 37 * 38 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 39 * $FreeBSD: src/sys/vm/vm_mmap.c,v 1.108.2.6 2002/07/02 20:06:19 dillon Exp $ 40 */ 41 42 /* 43 * Mapped file (mmap) interface to VM 44 */ 45 46 #include <sys/param.h> 47 #include <sys/kernel.h> 48 #include <sys/systm.h> 49 #include <sys/sysproto.h> 50 #include <sys/filedesc.h> 51 #include <sys/kern_syscall.h> 52 #include <sys/proc.h> 53 #include <sys/priv.h> 54 #include <sys/resource.h> 55 #include <sys/resourcevar.h> 56 #include <sys/vnode.h> 57 #include <sys/fcntl.h> 58 #include <sys/file.h> 59 #include <sys/mman.h> 60 #include <sys/conf.h> 61 #include <sys/stat.h> 62 #include <sys/vmmeter.h> 63 #include <sys/sysctl.h> 64 65 #include <vm/vm.h> 66 #include <vm/vm_param.h> 67 #include <sys/lock.h> 68 #include <vm/pmap.h> 69 #include <vm/vm_map.h> 70 #include <vm/vm_object.h> 71 #include <vm/vm_page.h> 72 #include <vm/vm_pager.h> 73 #include <vm/vm_pageout.h> 74 #include <vm/vm_extern.h> 75 #include <vm/vm_kern.h> 76 77 #include <sys/file2.h> 78 #include <sys/thread.h> 79 #include <sys/thread2.h> 80 #include <vm/vm_page2.h> 81 82 static int max_proc_mmap = 1000000; 83 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, ""); 84 int vkernel_enable; 85 SYSCTL_INT(_vm, OID_AUTO, vkernel_enable, CTLFLAG_RW, &vkernel_enable, 0, ""); 86 87 /* 88 * sstk_args(int incr) 89 * 90 * MPSAFE 91 */ 92 int 93 sys_sstk(struct sstk_args *uap) 94 { 95 /* Not yet implemented */ 96 return (EOPNOTSUPP); 97 } 98 99 /* 100 * mmap_args(void *addr, size_t len, int prot, int flags, int fd, 101 * long pad, off_t pos) 102 * 103 * Memory Map (mmap) system call. Note that the file offset 104 * and address are allowed to be NOT page aligned, though if 105 * the MAP_FIXED flag it set, both must have the same remainder 106 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 107 * page-aligned, the actual mapping starts at trunc_page(addr) 108 * and the return value is adjusted up by the page offset. 109 * 110 * Generally speaking, only character devices which are themselves 111 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 112 * there would be no cache coherency between a descriptor and a VM mapping 113 * both to the same character device. 114 * 115 * Block devices can be mmap'd no matter what they represent. Cache coherency 116 * is maintained as long as you do not write directly to the underlying 117 * character device. 118 * 119 * No requirements 120 */ 121 int 122 kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen, 123 int uprot, int uflags, int fd, off_t upos, void **res) 124 { 125 struct thread *td = curthread; 126 struct proc *p = td->td_proc; 127 struct file *fp = NULL; 128 struct vnode *vp; 129 vm_offset_t addr; 130 vm_offset_t tmpaddr; 131 vm_size_t size, pageoff; 132 vm_prot_t prot, maxprot; 133 void *handle; 134 int flags, error; 135 off_t pos; 136 vm_object_t obj; 137 138 KKASSERT(p); 139 140 addr = (vm_offset_t) uaddr; 141 size = ulen; 142 prot = uprot & VM_PROT_ALL; 143 flags = uflags; 144 pos = upos; 145 146 /* 147 * Make sure mapping fits into numeric range etc. 148 * 149 * NOTE: We support the full unsigned range for size now. 150 */ 151 if (((flags & MAP_ANON) && (fd != -1 || pos != 0))) 152 return (EINVAL); 153 154 if (size == 0) 155 return (EINVAL); 156 157 if (flags & MAP_STACK) { 158 if (fd != -1) 159 return (EINVAL); 160 if ((prot & (PROT_READ|PROT_WRITE)) != (PROT_READ|PROT_WRITE)) 161 return (EINVAL); 162 flags |= MAP_ANON; 163 pos = 0; 164 } 165 166 /* 167 * Virtual page tables cannot be used with MAP_STACK. Apart from 168 * it not making any sense, the aux union is used by both 169 * types. 170 * 171 * Because the virtual page table is stored in the backing object 172 * and might be updated by the kernel, the mapping must be R+W. 173 */ 174 if (flags & MAP_VPAGETABLE) { 175 if (vkernel_enable == 0) 176 return (EOPNOTSUPP); 177 if (flags & MAP_STACK) 178 return (EINVAL); 179 if ((prot & (PROT_READ|PROT_WRITE)) != (PROT_READ|PROT_WRITE)) 180 return (EINVAL); 181 } 182 183 /* 184 * Align the file position to a page boundary, 185 * and save its page offset component. 186 */ 187 pageoff = (pos & PAGE_MASK); 188 pos -= pageoff; 189 190 /* Adjust size for rounding (on both ends). */ 191 size += pageoff; /* low end... */ 192 size = (vm_size_t) round_page(size); /* hi end */ 193 if (size < ulen) /* wrap */ 194 return(EINVAL); 195 196 /* 197 * Check for illegal addresses. Watch out for address wrap... Note 198 * that VM_*_ADDRESS are not constants due to casts (argh). 199 */ 200 if (flags & (MAP_FIXED | MAP_TRYFIXED)) { 201 /* 202 * The specified address must have the same remainder 203 * as the file offset taken modulo PAGE_SIZE, so it 204 * should be aligned after adjustment by pageoff. 205 */ 206 addr -= pageoff; 207 if (addr & PAGE_MASK) 208 return (EINVAL); 209 210 /* 211 * Address range must be all in user VM space and not wrap. 212 */ 213 tmpaddr = addr + size; 214 if (tmpaddr < addr) 215 return (EINVAL); 216 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 217 return (EINVAL); 218 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) 219 return (EINVAL); 220 } else { 221 /* 222 * Get a hint of where to map. It also provides mmap offset 223 * randomization if enabled. 224 */ 225 addr = vm_map_hint(p, addr, prot); 226 } 227 228 if (flags & MAP_ANON) { 229 /* 230 * Mapping blank space is trivial. 231 */ 232 handle = NULL; 233 maxprot = VM_PROT_ALL; 234 } else { 235 /* 236 * Mapping file, get fp for validation. Obtain vnode and make 237 * sure it is of appropriate type. 238 */ 239 fp = holdfp(td, fd, -1); 240 if (fp == NULL) 241 return (EBADF); 242 if (fp->f_type != DTYPE_VNODE) { 243 error = EINVAL; 244 goto done; 245 } 246 /* 247 * POSIX shared-memory objects are defined to have 248 * kernel persistence, and are not defined to support 249 * read(2)/write(2) -- or even open(2). Thus, we can 250 * use MAP_ASYNC to trade on-disk coherence for speed. 251 * The shm_open(3) library routine turns on the FPOSIXSHM 252 * flag to request this behavior. 253 */ 254 if (fp->f_flag & FPOSIXSHM) 255 flags |= MAP_NOSYNC; 256 vp = (struct vnode *) fp->f_data; 257 258 /* 259 * Validate the vnode for the operation. 260 */ 261 switch(vp->v_type) { 262 case VREG: 263 /* 264 * Get the proper underlying object 265 */ 266 if ((obj = vp->v_object) == NULL) { 267 error = EINVAL; 268 goto done; 269 } 270 KKASSERT((struct vnode *)obj->handle == vp); 271 break; 272 case VCHR: 273 /* 274 * Make sure a device has not been revoked. 275 * Mappability is handled by the device layer. 276 */ 277 if (vp->v_rdev == NULL) { 278 error = EBADF; 279 goto done; 280 } 281 break; 282 default: 283 /* 284 * Nothing else is mappable. 285 */ 286 error = EINVAL; 287 goto done; 288 } 289 290 /* 291 * XXX hack to handle use of /dev/zero to map anon memory (ala 292 * SunOS). 293 */ 294 if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) { 295 handle = NULL; 296 maxprot = VM_PROT_ALL; 297 flags |= MAP_ANON; 298 pos = 0; 299 } else { 300 /* 301 * cdevs does not provide private mappings of any kind. 302 */ 303 if (vp->v_type == VCHR && 304 (flags & (MAP_PRIVATE|MAP_COPY))) { 305 error = EINVAL; 306 goto done; 307 } 308 /* 309 * Ensure that file and memory protections are 310 * compatible. Note that we only worry about 311 * writability if mapping is shared; in this case, 312 * current and max prot are dictated by the open file. 313 * XXX use the vnode instead? Problem is: what 314 * credentials do we use for determination? What if 315 * proc does a setuid? 316 */ 317 maxprot = VM_PROT_EXECUTE; 318 if (fp->f_flag & FREAD) { 319 maxprot |= VM_PROT_READ; 320 } else if (prot & PROT_READ) { 321 error = EACCES; 322 goto done; 323 } 324 /* 325 * If we are sharing potential changes (either via 326 * MAP_SHARED or via the implicit sharing of character 327 * device mappings), and we are trying to get write 328 * permission although we opened it without asking 329 * for it, bail out. Check for superuser, only if 330 * we're at securelevel < 1, to allow the XIG X server 331 * to continue to work. 332 * 333 * PROT_WRITE + MAP_SHARED 334 */ 335 if ((flags & MAP_SHARED) != 0 || vp->v_type == VCHR) { 336 if ((fp->f_flag & FWRITE) != 0) { 337 struct vattr va; 338 if ((error = VOP_GETATTR(vp, &va))) { 339 goto done; 340 } 341 if ((va.va_flags & 342 (IMMUTABLE|APPEND)) == 0) { 343 maxprot |= VM_PROT_WRITE; 344 345 /* 346 * SHARED+RW file mmap() 347 * updates v_lastwrite_ts. 348 */ 349 if ((prot & PROT_WRITE) && 350 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY) == 0) { 351 vfs_timestamp(&vp->v_lastwrite_ts); 352 vsetflags(vp, VLASTWRITETS); 353 vn_unlock(vp); 354 } 355 } else if (prot & PROT_WRITE) { 356 error = EPERM; 357 goto done; 358 } 359 } else if ((prot & PROT_WRITE) != 0) { 360 error = EACCES; 361 goto done; 362 } 363 } else { 364 maxprot |= VM_PROT_WRITE; 365 } 366 handle = (void *)vp; 367 } 368 } 369 370 lwkt_gettoken(&vms->vm_map.token); 371 372 /* 373 * Do not allow more then a certain number of vm_map_entry structures 374 * per process. 0 to disable. 375 */ 376 if (max_proc_mmap && vms->vm_map.nentries >= max_proc_mmap) { 377 error = ENOMEM; 378 lwkt_reltoken(&vms->vm_map.token); 379 goto done; 380 } 381 382 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 383 flags, handle, pos); 384 if (error == 0) 385 *res = (void *)(addr + pageoff); 386 387 lwkt_reltoken(&vms->vm_map.token); 388 done: 389 if (fp) 390 dropfp(td, fd, fp); 391 392 return (error); 393 } 394 395 /* 396 * mmap system call handler 397 * 398 * No requirements. 399 */ 400 int 401 sys_mmap(struct mmap_args *uap) 402 { 403 int error; 404 405 error = kern_mmap(curproc->p_vmspace, uap->addr, uap->len, 406 uap->prot, uap->flags, 407 uap->fd, uap->pos, &uap->sysmsg_resultp); 408 409 return (error); 410 } 411 412 /* 413 * msync system call handler 414 * 415 * msync_args(void *addr, size_t len, int flags) 416 * 417 * No requirements 418 */ 419 int 420 sys_msync(struct msync_args *uap) 421 { 422 struct proc *p = curproc; 423 vm_offset_t addr; 424 vm_offset_t tmpaddr; 425 vm_size_t size, pageoff; 426 int flags; 427 vm_map_t map; 428 int rv; 429 430 addr = (vm_offset_t) uap->addr; 431 size = uap->len; 432 flags = uap->flags; 433 434 pageoff = (addr & PAGE_MASK); 435 addr -= pageoff; 436 size += pageoff; 437 size = (vm_size_t) round_page(size); 438 if (size < uap->len) /* wrap */ 439 return(EINVAL); 440 tmpaddr = addr + size; /* workaround gcc4 opt */ 441 if (tmpaddr < addr) /* wrap */ 442 return(EINVAL); 443 444 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 445 return (EINVAL); 446 447 map = &p->p_vmspace->vm_map; 448 449 /* 450 * map->token serializes extracting the address range for size == 0 451 * msyncs with the vm_map_clean call; if the token were not held 452 * across the two calls, an intervening munmap/mmap pair, for example, 453 * could cause msync to occur on a wrong region. 454 */ 455 lwkt_gettoken(&map->token); 456 457 /* 458 * XXX Gak! If size is zero we are supposed to sync "all modified 459 * pages with the region containing addr". Unfortunately, we don't 460 * really keep track of individual mmaps so we approximate by flushing 461 * the range of the map entry containing addr. This can be incorrect 462 * if the region splits or is coalesced with a neighbor. 463 */ 464 if (size == 0) { 465 vm_map_entry_t entry; 466 467 vm_map_lock_read(map); 468 rv = vm_map_lookup_entry(map, addr, &entry); 469 if (rv == FALSE) { 470 vm_map_unlock_read(map); 471 rv = KERN_INVALID_ADDRESS; 472 goto done; 473 } 474 addr = entry->start; 475 size = entry->end - entry->start; 476 vm_map_unlock_read(map); 477 } 478 479 /* 480 * Clean the pages and interpret the return value. 481 */ 482 rv = vm_map_clean(map, addr, addr + size, (flags & MS_ASYNC) == 0, 483 (flags & MS_INVALIDATE) != 0); 484 done: 485 lwkt_reltoken(&map->token); 486 487 switch (rv) { 488 case KERN_SUCCESS: 489 break; 490 case KERN_INVALID_ADDRESS: 491 return (EINVAL); /* Sun returns ENOMEM? */ 492 case KERN_FAILURE: 493 return (EIO); 494 default: 495 return (EINVAL); 496 } 497 498 return (0); 499 } 500 501 /* 502 * munmap system call handler 503 * 504 * munmap_args(void *addr, size_t len) 505 * 506 * No requirements 507 */ 508 int 509 sys_munmap(struct munmap_args *uap) 510 { 511 struct proc *p = curproc; 512 vm_offset_t addr; 513 vm_offset_t tmpaddr; 514 vm_size_t size, pageoff; 515 vm_map_t map; 516 517 addr = (vm_offset_t) uap->addr; 518 size = uap->len; 519 520 pageoff = (addr & PAGE_MASK); 521 addr -= pageoff; 522 size += pageoff; 523 size = (vm_size_t) round_page(size); 524 if (size < uap->len) /* wrap */ 525 return(EINVAL); 526 tmpaddr = addr + size; /* workaround gcc4 opt */ 527 if (tmpaddr < addr) /* wrap */ 528 return(EINVAL); 529 530 if (size == 0) 531 return (0); 532 533 /* 534 * Check for illegal addresses. Watch out for address wrap... Note 535 * that VM_*_ADDRESS are not constants due to casts (argh). 536 */ 537 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 538 return (EINVAL); 539 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) 540 return (EINVAL); 541 542 map = &p->p_vmspace->vm_map; 543 544 /* map->token serializes between the map check and the actual unmap */ 545 lwkt_gettoken(&map->token); 546 547 /* 548 * Make sure entire range is allocated. 549 */ 550 if (!vm_map_check_protection(map, addr, addr + size, 551 VM_PROT_NONE, FALSE)) { 552 lwkt_reltoken(&map->token); 553 return (EINVAL); 554 } 555 /* returns nothing but KERN_SUCCESS anyway */ 556 vm_map_remove(map, addr, addr + size); 557 lwkt_reltoken(&map->token); 558 return (0); 559 } 560 561 /* 562 * mprotect_args(const void *addr, size_t len, int prot) 563 * 564 * No requirements. 565 */ 566 int 567 sys_mprotect(struct mprotect_args *uap) 568 { 569 struct proc *p = curproc; 570 vm_offset_t addr; 571 vm_offset_t tmpaddr; 572 vm_size_t size, pageoff; 573 vm_prot_t prot; 574 int error; 575 576 addr = (vm_offset_t) uap->addr; 577 size = uap->len; 578 prot = uap->prot & VM_PROT_ALL; 579 580 pageoff = (addr & PAGE_MASK); 581 addr -= pageoff; 582 size += pageoff; 583 size = (vm_size_t) round_page(size); 584 if (size < uap->len) /* wrap */ 585 return(EINVAL); 586 tmpaddr = addr + size; /* workaround gcc4 opt */ 587 if (tmpaddr < addr) /* wrap */ 588 return(EINVAL); 589 590 switch (vm_map_protect(&p->p_vmspace->vm_map, addr, addr + size, 591 prot, FALSE)) { 592 case KERN_SUCCESS: 593 error = 0; 594 break; 595 case KERN_PROTECTION_FAILURE: 596 error = EACCES; 597 break; 598 default: 599 error = EINVAL; 600 break; 601 } 602 return (error); 603 } 604 605 /* 606 * minherit system call handler 607 * 608 * minherit_args(void *addr, size_t len, int inherit) 609 * 610 * No requirements. 611 */ 612 int 613 sys_minherit(struct minherit_args *uap) 614 { 615 struct proc *p = curproc; 616 vm_offset_t addr; 617 vm_offset_t tmpaddr; 618 vm_size_t size, pageoff; 619 vm_inherit_t inherit; 620 int error; 621 622 addr = (vm_offset_t)uap->addr; 623 size = uap->len; 624 inherit = uap->inherit; 625 626 pageoff = (addr & PAGE_MASK); 627 addr -= pageoff; 628 size += pageoff; 629 size = (vm_size_t) round_page(size); 630 if (size < uap->len) /* wrap */ 631 return(EINVAL); 632 tmpaddr = addr + size; /* workaround gcc4 opt */ 633 if (tmpaddr < addr) /* wrap */ 634 return(EINVAL); 635 636 switch (vm_map_inherit(&p->p_vmspace->vm_map, addr, 637 addr + size, inherit)) { 638 case KERN_SUCCESS: 639 error = 0; 640 break; 641 case KERN_PROTECTION_FAILURE: 642 error = EACCES; 643 break; 644 default: 645 error = EINVAL; 646 break; 647 } 648 return (error); 649 } 650 651 /* 652 * madvise system call handler 653 * 654 * madvise_args(void *addr, size_t len, int behav) 655 * 656 * No requirements. 657 */ 658 int 659 sys_madvise(struct madvise_args *uap) 660 { 661 struct proc *p = curproc; 662 vm_offset_t start, end; 663 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len; 664 int error; 665 666 /* 667 * Check for illegal behavior 668 */ 669 if (uap->behav < 0 || uap->behav >= MADV_CONTROL_END) 670 return (EINVAL); 671 /* 672 * Check for illegal addresses. Watch out for address wrap... Note 673 * that VM_*_ADDRESS are not constants due to casts (argh). 674 */ 675 if (tmpaddr < (vm_offset_t)uap->addr) 676 return (EINVAL); 677 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 678 return (EINVAL); 679 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) 680 return (EINVAL); 681 682 /* 683 * Since this routine is only advisory, we default to conservative 684 * behavior. 685 */ 686 start = trunc_page((vm_offset_t)uap->addr); 687 end = round_page(tmpaddr); 688 689 error = vm_map_madvise(&p->p_vmspace->vm_map, start, end, 690 uap->behav, 0); 691 return (error); 692 } 693 694 /* 695 * mcontrol system call handler 696 * 697 * mcontrol_args(void *addr, size_t len, int behav, off_t value) 698 * 699 * No requirements 700 */ 701 int 702 sys_mcontrol(struct mcontrol_args *uap) 703 { 704 struct proc *p = curproc; 705 vm_offset_t start, end; 706 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len; 707 int error; 708 709 /* 710 * Check for illegal behavior 711 */ 712 if (uap->behav < 0 || uap->behav > MADV_CONTROL_END) 713 return (EINVAL); 714 /* 715 * Check for illegal addresses. Watch out for address wrap... Note 716 * that VM_*_ADDRESS are not constants due to casts (argh). 717 */ 718 if (tmpaddr < (vm_offset_t) uap->addr) 719 return (EINVAL); 720 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 721 return (EINVAL); 722 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) 723 return (EINVAL); 724 725 /* 726 * Since this routine is only advisory, we default to conservative 727 * behavior. 728 */ 729 start = trunc_page((vm_offset_t)uap->addr); 730 end = round_page(tmpaddr); 731 732 error = vm_map_madvise(&p->p_vmspace->vm_map, start, end, 733 uap->behav, uap->value); 734 return (error); 735 } 736 737 738 /* 739 * mincore system call handler 740 * 741 * mincore_args(const void *addr, size_t len, char *vec) 742 * 743 * No requirements 744 */ 745 int 746 sys_mincore(struct mincore_args *uap) 747 { 748 struct proc *p = curproc; 749 vm_offset_t addr, first_addr; 750 vm_offset_t end, cend; 751 pmap_t pmap; 752 vm_map_t map; 753 char *vec; 754 int error; 755 int vecindex, lastvecindex; 756 vm_map_entry_t current; 757 vm_map_entry_t entry; 758 int mincoreinfo; 759 unsigned int timestamp; 760 761 /* 762 * Make sure that the addresses presented are valid for user 763 * mode. 764 */ 765 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 766 end = addr + (vm_size_t)round_page(uap->len); 767 if (end < addr) 768 return (EINVAL); 769 if (VM_MAX_USER_ADDRESS > 0 && end > VM_MAX_USER_ADDRESS) 770 return (EINVAL); 771 772 /* 773 * Address of byte vector 774 */ 775 vec = uap->vec; 776 777 map = &p->p_vmspace->vm_map; 778 pmap = vmspace_pmap(p->p_vmspace); 779 780 lwkt_gettoken(&map->token); 781 vm_map_lock_read(map); 782 RestartScan: 783 timestamp = map->timestamp; 784 785 if (!vm_map_lookup_entry(map, addr, &entry)) 786 entry = entry->next; 787 788 /* 789 * Do this on a map entry basis so that if the pages are not 790 * in the current processes address space, we can easily look 791 * up the pages elsewhere. 792 */ 793 lastvecindex = -1; 794 for(current = entry; 795 (current != &map->header) && (current->start < end); 796 current = current->next) { 797 798 /* 799 * ignore submaps (for now) or null objects 800 */ 801 if (current->maptype != VM_MAPTYPE_NORMAL && 802 current->maptype != VM_MAPTYPE_VPAGETABLE) { 803 continue; 804 } 805 if (current->object.vm_object == NULL) 806 continue; 807 808 /* 809 * limit this scan to the current map entry and the 810 * limits for the mincore call 811 */ 812 if (addr < current->start) 813 addr = current->start; 814 cend = current->end; 815 if (cend > end) 816 cend = end; 817 818 /* 819 * scan this entry one page at a time 820 */ 821 while (addr < cend) { 822 /* 823 * Check pmap first, it is likely faster, also 824 * it can provide info as to whether we are the 825 * one referencing or modifying the page. 826 * 827 * If we have to check the VM object, only mess 828 * around with normal maps. Do not mess around 829 * with virtual page tables (XXX). 830 */ 831 mincoreinfo = pmap_mincore(pmap, addr); 832 if (mincoreinfo == 0 && 833 current->maptype == VM_MAPTYPE_NORMAL) { 834 vm_pindex_t pindex; 835 vm_ooffset_t offset; 836 vm_page_t m; 837 838 /* 839 * calculate the page index into the object 840 */ 841 offset = current->offset + (addr - current->start); 842 pindex = OFF_TO_IDX(offset); 843 844 /* 845 * if the page is resident, then gather 846 * information about it. spl protection is 847 * required to maintain the object 848 * association. And XXX what if the page is 849 * busy? What's the deal with that? 850 * 851 * XXX vm_token - legacy for pmap_ts_referenced 852 * in x86 and vkernel pmap code. 853 */ 854 lwkt_gettoken(&vm_token); 855 vm_object_hold(current->object.vm_object); 856 m = vm_page_lookup(current->object.vm_object, 857 pindex); 858 if (m && m->valid) { 859 mincoreinfo = MINCORE_INCORE; 860 if (m->dirty || pmap_is_modified(m)) 861 mincoreinfo |= MINCORE_MODIFIED_OTHER; 862 if ((m->flags & PG_REFERENCED) || 863 pmap_ts_referenced(m)) { 864 vm_page_flag_set(m, PG_REFERENCED); 865 mincoreinfo |= MINCORE_REFERENCED_OTHER; 866 } 867 } 868 vm_object_drop(current->object.vm_object); 869 lwkt_reltoken(&vm_token); 870 } 871 872 /* 873 * subyte may page fault. In case it needs to modify 874 * the map, we release the lock. 875 */ 876 vm_map_unlock_read(map); 877 878 /* 879 * calculate index into user supplied byte vector 880 */ 881 vecindex = OFF_TO_IDX(addr - first_addr); 882 883 /* 884 * If we have skipped map entries, we need to make sure that 885 * the byte vector is zeroed for those skipped entries. 886 */ 887 while((lastvecindex + 1) < vecindex) { 888 error = subyte( vec + lastvecindex, 0); 889 if (error) { 890 error = EFAULT; 891 goto done; 892 } 893 ++lastvecindex; 894 } 895 896 /* 897 * Pass the page information to the user 898 */ 899 error = subyte(vec + vecindex, mincoreinfo); 900 if (error) { 901 error = EFAULT; 902 goto done; 903 } 904 905 /* 906 * If the map has changed, due to the subyte, 907 * the previous output may be invalid. 908 */ 909 vm_map_lock_read(map); 910 if (timestamp != map->timestamp) 911 goto RestartScan; 912 913 lastvecindex = vecindex; 914 addr += PAGE_SIZE; 915 } 916 } 917 918 /* 919 * subyte may page fault. In case it needs to modify 920 * the map, we release the lock. 921 */ 922 vm_map_unlock_read(map); 923 924 /* 925 * Zero the last entries in the byte vector. 926 */ 927 vecindex = OFF_TO_IDX(end - first_addr); 928 while((lastvecindex + 1) < vecindex) { 929 error = subyte( vec + lastvecindex, 0); 930 if (error) { 931 error = EFAULT; 932 goto done; 933 } 934 ++lastvecindex; 935 } 936 937 /* 938 * If the map has changed, due to the subyte, the previous 939 * output may be invalid. 940 */ 941 vm_map_lock_read(map); 942 if (timestamp != map->timestamp) 943 goto RestartScan; 944 vm_map_unlock_read(map); 945 946 error = 0; 947 done: 948 lwkt_reltoken(&map->token); 949 return (error); 950 } 951 952 /* 953 * mlock system call handler 954 * 955 * mlock_args(const void *addr, size_t len) 956 * 957 * No requirements 958 */ 959 int 960 sys_mlock(struct mlock_args *uap) 961 { 962 vm_offset_t addr; 963 vm_offset_t tmpaddr; 964 vm_size_t size, pageoff; 965 struct thread *td = curthread; 966 struct proc *p = td->td_proc; 967 int error; 968 969 addr = (vm_offset_t) uap->addr; 970 size = uap->len; 971 972 pageoff = (addr & PAGE_MASK); 973 addr -= pageoff; 974 size += pageoff; 975 size = (vm_size_t) round_page(size); 976 if (size < uap->len) /* wrap */ 977 return (EINVAL); 978 if (size == 0) /* silently allow 0 size */ 979 return (0); 980 tmpaddr = addr + size; /* workaround gcc4 opt */ 981 if (tmpaddr < addr) /* wrap */ 982 return (EINVAL); 983 984 if (atop(size) + vmstats.v_wire_count > vm_page_max_wired) 985 return (EAGAIN); 986 987 /* 988 * We do not need to synchronize against other threads updating ucred; 989 * they update p->ucred, which is synchronized into td_ucred ourselves. 990 */ 991 #ifdef pmap_wired_count 992 if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) > 993 p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur) { 994 return (ENOMEM); 995 } 996 #else 997 error = priv_check_cred(td->td_ucred, PRIV_ROOT, 0); 998 if (error) { 999 return (error); 1000 } 1001 #endif 1002 error = vm_map_unwire(&p->p_vmspace->vm_map, addr, addr + size, FALSE); 1003 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1004 } 1005 1006 /* 1007 * mlockall(int how) 1008 * 1009 * No requirements 1010 */ 1011 int 1012 sys_mlockall(struct mlockall_args *uap) 1013 { 1014 struct thread *td = curthread; 1015 struct proc *p = td->td_proc; 1016 vm_map_t map = &p->p_vmspace->vm_map; 1017 vm_map_entry_t entry; 1018 int how = uap->how; 1019 int rc = KERN_SUCCESS; 1020 1021 if (((how & MCL_CURRENT) == 0) && ((how & MCL_FUTURE) == 0)) 1022 return (EINVAL); 1023 1024 rc = priv_check_cred(td->td_ucred, PRIV_ROOT, 0); 1025 if (rc) 1026 return (rc); 1027 1028 vm_map_lock(map); 1029 do { 1030 if (how & MCL_CURRENT) { 1031 for(entry = map->header.next; 1032 entry != &map->header; 1033 entry = entry->next); 1034 1035 rc = ENOSYS; 1036 break; 1037 } 1038 1039 if (how & MCL_FUTURE) 1040 map->flags |= MAP_WIREFUTURE; 1041 } while(0); 1042 vm_map_unlock(map); 1043 1044 return (rc); 1045 } 1046 1047 /* 1048 * munlockall(void) 1049 * 1050 * Unwire all user-wired map entries, cancel MCL_FUTURE. 1051 * 1052 * No requirements 1053 */ 1054 int 1055 sys_munlockall(struct munlockall_args *uap) 1056 { 1057 struct thread *td = curthread; 1058 struct proc *p = td->td_proc; 1059 vm_map_t map = &p->p_vmspace->vm_map; 1060 vm_map_entry_t entry; 1061 int rc = KERN_SUCCESS; 1062 1063 vm_map_lock(map); 1064 1065 /* Clear MAP_WIREFUTURE to cancel mlockall(MCL_FUTURE) */ 1066 map->flags &= ~MAP_WIREFUTURE; 1067 1068 retry: 1069 for (entry = map->header.next; 1070 entry != &map->header; 1071 entry = entry->next) { 1072 if ((entry->eflags & MAP_ENTRY_USER_WIRED) == 0) 1073 continue; 1074 1075 /* 1076 * If we encounter an in-transition entry, we release the 1077 * map lock and retry the scan; we do not decrement any 1078 * wired_count more than once because we do not touch 1079 * any entries with MAP_ENTRY_USER_WIRED not set. 1080 * 1081 * There is a potential interleaving with concurrent 1082 * mlockall()s here -- if we abort a scan, an mlockall() 1083 * could start, wire a number of entries before our 1084 * current position in, and then stall itself on this 1085 * or any other in-transition entry. If that occurs, when 1086 * we resume, we will unwire those entries. 1087 */ 1088 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { 1089 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; 1090 ++mycpu->gd_cnt.v_intrans_coll; 1091 ++mycpu->gd_cnt.v_intrans_wait; 1092 vm_map_transition_wait(map, 1); 1093 goto retry; 1094 } 1095 1096 KASSERT(entry->wired_count > 0, 1097 ("wired_count was 0 with USER_WIRED set! %p", entry)); 1098 1099 /* Drop wired count, if it hits zero, unwire the entry */ 1100 entry->eflags &= ~MAP_ENTRY_USER_WIRED; 1101 entry->wired_count--; 1102 if (entry->wired_count == 0) 1103 vm_fault_unwire(map, entry); 1104 } 1105 1106 vm_map_unlock(map); 1107 1108 return (rc); 1109 } 1110 1111 /* 1112 * munlock system call handler 1113 * 1114 * munlock_args(const void *addr, size_t len) 1115 * 1116 * No requirements 1117 */ 1118 int 1119 sys_munlock(struct munlock_args *uap) 1120 { 1121 struct thread *td = curthread; 1122 struct proc *p = td->td_proc; 1123 vm_offset_t addr; 1124 vm_offset_t tmpaddr; 1125 vm_size_t size, pageoff; 1126 int error; 1127 1128 addr = (vm_offset_t) uap->addr; 1129 size = uap->len; 1130 1131 pageoff = (addr & PAGE_MASK); 1132 addr -= pageoff; 1133 size += pageoff; 1134 size = (vm_size_t) round_page(size); 1135 1136 tmpaddr = addr + size; 1137 if (tmpaddr < addr) /* wrap */ 1138 return (EINVAL); 1139 if (size == 0) /* silently allow 0 size */ 1140 return (0); 1141 1142 #ifndef pmap_wired_count 1143 error = priv_check(td, PRIV_ROOT); 1144 if (error) 1145 return (error); 1146 #endif 1147 1148 error = vm_map_unwire(&p->p_vmspace->vm_map, addr, addr + size, TRUE); 1149 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1150 } 1151 1152 /* 1153 * Internal version of mmap. 1154 * Currently used by mmap, exec, and sys5 shared memory. 1155 * Handle is either a vnode pointer or NULL for MAP_ANON. 1156 * 1157 * No requirements 1158 */ 1159 int 1160 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1161 vm_prot_t maxprot, int flags, void *handle, vm_ooffset_t foff) 1162 { 1163 boolean_t fitit; 1164 vm_object_t object; 1165 vm_offset_t eaddr; 1166 vm_size_t esize; 1167 vm_size_t align; 1168 int (*uksmap)(cdev_t dev, vm_page_t fake); 1169 struct vnode *vp; 1170 struct thread *td = curthread; 1171 struct proc *p; 1172 int rv = KERN_SUCCESS; 1173 off_t objsize; 1174 int docow; 1175 int error; 1176 1177 if (size == 0) 1178 return (0); 1179 1180 objsize = round_page(size); 1181 if (objsize < size) 1182 return (EINVAL); 1183 size = objsize; 1184 1185 lwkt_gettoken(&map->token); 1186 1187 /* 1188 * XXX messy code, fixme 1189 * 1190 * NOTE: Overflow checks require discrete statements or GCC4 1191 * will optimize it out. 1192 */ 1193 if ((p = curproc) != NULL && map == &p->p_vmspace->vm_map) { 1194 esize = map->size + size; /* workaround gcc4 opt */ 1195 if (esize < map->size || 1196 esize > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1197 lwkt_reltoken(&map->token); 1198 return(ENOMEM); 1199 } 1200 } 1201 1202 /* 1203 * We currently can only deal with page aligned file offsets. 1204 * The check is here rather than in the syscall because the 1205 * kernel calls this function internally for other mmaping 1206 * operations (such as in exec) and non-aligned offsets will 1207 * cause pmap inconsistencies...so we want to be sure to 1208 * disallow this in all cases. 1209 * 1210 * NOTE: Overflow checks require discrete statements or GCC4 1211 * will optimize it out. 1212 */ 1213 if (foff & PAGE_MASK) { 1214 lwkt_reltoken(&map->token); 1215 return (EINVAL); 1216 } 1217 1218 /* 1219 * Handle alignment. For large memory maps it is possible 1220 * that the MMU can optimize the page table so align anything 1221 * that is a multiple of SEG_SIZE to SEG_SIZE. 1222 * 1223 * Also align any large mapping (bigger than 16x SG_SIZE) to a 1224 * SEG_SIZE address boundary. 1225 */ 1226 if (flags & MAP_SIZEALIGN) { 1227 align = size; 1228 if ((align ^ (align - 1)) != (align << 1) - 1) { 1229 lwkt_reltoken(&map->token); 1230 return (EINVAL); 1231 } 1232 } else if ((flags & MAP_FIXED) == 0 && 1233 ((size & SEG_MASK) == 0 || size > SEG_SIZE * 16)) { 1234 align = SEG_SIZE; 1235 } else { 1236 align = PAGE_SIZE; 1237 } 1238 1239 if ((flags & (MAP_FIXED | MAP_TRYFIXED)) == 0) { 1240 fitit = TRUE; 1241 *addr = round_page(*addr); 1242 } else { 1243 if (*addr != trunc_page(*addr)) { 1244 lwkt_reltoken(&map->token); 1245 return (EINVAL); 1246 } 1247 eaddr = *addr + size; 1248 if (eaddr < *addr) { 1249 lwkt_reltoken(&map->token); 1250 return (EINVAL); 1251 } 1252 fitit = FALSE; 1253 if ((flags & MAP_TRYFIXED) == 0) 1254 vm_map_remove(map, *addr, *addr + size); 1255 } 1256 1257 uksmap = NULL; 1258 1259 /* 1260 * Lookup/allocate object. 1261 */ 1262 if (flags & MAP_ANON) { 1263 /* 1264 * Unnamed anonymous regions always start at 0. 1265 */ 1266 if (handle) { 1267 /* 1268 * Default memory object 1269 */ 1270 object = default_pager_alloc(handle, objsize, 1271 prot, foff); 1272 if (object == NULL) { 1273 lwkt_reltoken(&map->token); 1274 return(ENOMEM); 1275 } 1276 docow = MAP_PREFAULT_PARTIAL; 1277 } else { 1278 /* 1279 * Implicit single instance of a default memory 1280 * object, so we don't need a VM object yet. 1281 */ 1282 foff = 0; 1283 object = NULL; 1284 docow = 0; 1285 } 1286 vp = NULL; 1287 } else { 1288 vp = (struct vnode *)handle; 1289 1290 /* 1291 * Non-anonymous mappings of VCHR (aka not /dev/zero) 1292 * cannot specify MAP_STACK or MAP_VPAGETABLE. 1293 */ 1294 if (vp->v_type == VCHR) { 1295 if (flags & (MAP_STACK | MAP_VPAGETABLE)) { 1296 lwkt_reltoken(&map->token); 1297 return(EINVAL); 1298 } 1299 } 1300 1301 if (vp->v_type == VCHR && vp->v_rdev->si_ops->d_uksmap) { 1302 /* 1303 * Device mappings without a VM object, typically 1304 * sharing permanently allocated kernel memory or 1305 * process-context-specific (per-process) data. 1306 * 1307 * Force them to be shared. 1308 */ 1309 uksmap = vp->v_rdev->si_ops->d_uksmap; 1310 object = NULL; 1311 docow = MAP_PREFAULT_PARTIAL; 1312 flags &= ~(MAP_PRIVATE|MAP_COPY); 1313 flags |= MAP_SHARED; 1314 } else if (vp->v_type == VCHR) { 1315 /* 1316 * Device mappings (device size unknown?). 1317 * Force them to be shared. 1318 */ 1319 error = dev_dmmap_single(vp->v_rdev, &foff, objsize, 1320 &object, prot, NULL); 1321 1322 if (error == ENODEV) { 1323 handle = (void *)(intptr_t)vp->v_rdev; 1324 object = dev_pager_alloc(handle, objsize, prot, foff); 1325 if (object == NULL) { 1326 lwkt_reltoken(&map->token); 1327 return(EINVAL); 1328 } 1329 } else if (error) { 1330 lwkt_reltoken(&map->token); 1331 return(error); 1332 } 1333 1334 docow = MAP_PREFAULT_PARTIAL; 1335 flags &= ~(MAP_PRIVATE|MAP_COPY); 1336 flags |= MAP_SHARED; 1337 } else { 1338 /* 1339 * Regular file mapping (typically). The attribute 1340 * check is for the link count test only. mmapable 1341 * vnodes must already have a VM object assigned. 1342 */ 1343 struct vattr vat; 1344 int error; 1345 1346 error = VOP_GETATTR(vp, &vat); 1347 if (error) { 1348 lwkt_reltoken(&map->token); 1349 return (error); 1350 } 1351 docow = MAP_PREFAULT_PARTIAL; 1352 object = vnode_pager_reference(vp); 1353 if (object == NULL && vp->v_type == VREG) { 1354 lwkt_reltoken(&map->token); 1355 kprintf("Warning: cannot mmap vnode %p, no " 1356 "object\n", vp); 1357 return(EINVAL); 1358 } 1359 1360 /* 1361 * If it is a regular file without any references 1362 * we do not need to sync it. 1363 */ 1364 if (vp->v_type == VREG && vat.va_nlink == 0) { 1365 flags |= MAP_NOSYNC; 1366 } 1367 } 1368 } 1369 1370 /* 1371 * Deal with the adjusted flags 1372 */ 1373 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1374 docow |= MAP_COPY_ON_WRITE; 1375 if (flags & MAP_NOSYNC) 1376 docow |= MAP_DISABLE_SYNCER; 1377 if (flags & MAP_NOCORE) 1378 docow |= MAP_DISABLE_COREDUMP; 1379 1380 /* 1381 * This may place the area in its own page directory if (size) is 1382 * large enough, otherwise it typically returns its argument. 1383 * 1384 * (object can be NULL) 1385 */ 1386 if (fitit) { 1387 *addr = pmap_addr_hint(object, *addr, size); 1388 } 1389 1390 /* 1391 * Stack mappings need special attention. 1392 * 1393 * Mappings that use virtual page tables will default to storing 1394 * the page table at offset 0. 1395 */ 1396 if (uksmap) { 1397 rv = vm_map_find(map, uksmap, vp->v_rdev, 1398 foff, addr, size, 1399 align, fitit, 1400 VM_MAPTYPE_UKSMAP, VM_SUBSYS_MMAP, 1401 prot, maxprot, docow); 1402 } else if (flags & MAP_STACK) { 1403 rv = vm_map_stack(map, *addr, size, flags, 1404 prot, maxprot, docow); 1405 } else if (flags & MAP_VPAGETABLE) { 1406 rv = vm_map_find(map, object, NULL, 1407 foff, addr, size, 1408 align, fitit, 1409 VM_MAPTYPE_VPAGETABLE, VM_SUBSYS_MMAP, 1410 prot, maxprot, docow); 1411 } else { 1412 rv = vm_map_find(map, object, NULL, 1413 foff, addr, size, 1414 align, fitit, 1415 VM_MAPTYPE_NORMAL, VM_SUBSYS_MMAP, 1416 prot, maxprot, docow); 1417 } 1418 1419 if (rv != KERN_SUCCESS) { 1420 /* 1421 * Lose the object reference. Will destroy the 1422 * object if it's an unnamed anonymous mapping 1423 * or named anonymous without other references. 1424 * 1425 * (NOTE: object can be NULL) 1426 */ 1427 vm_object_deallocate(object); 1428 goto out; 1429 } 1430 1431 /* 1432 * Shared memory is also shared with children. 1433 */ 1434 if (flags & (MAP_SHARED|MAP_INHERIT)) { 1435 rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE); 1436 if (rv != KERN_SUCCESS) { 1437 vm_map_remove(map, *addr, *addr + size); 1438 goto out; 1439 } 1440 } 1441 1442 /* If a process has marked all future mappings for wiring, do so */ 1443 if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE)) 1444 vm_map_unwire(map, *addr, *addr + size, FALSE); 1445 1446 /* 1447 * Set the access time on the vnode 1448 */ 1449 if (vp != NULL) 1450 vn_mark_atime(vp, td); 1451 out: 1452 lwkt_reltoken(&map->token); 1453 1454 switch (rv) { 1455 case KERN_SUCCESS: 1456 return (0); 1457 case KERN_INVALID_ADDRESS: 1458 case KERN_NO_SPACE: 1459 return (ENOMEM); 1460 case KERN_PROTECTION_FAILURE: 1461 return (EACCES); 1462 default: 1463 return (EINVAL); 1464 } 1465 } 1466 1467 /* 1468 * Translate a Mach VM return code to zero on success or the appropriate errno 1469 * on failure. 1470 */ 1471 int 1472 vm_mmap_to_errno(int rv) 1473 { 1474 1475 switch (rv) { 1476 case KERN_SUCCESS: 1477 return (0); 1478 case KERN_INVALID_ADDRESS: 1479 case KERN_NO_SPACE: 1480 return (ENOMEM); 1481 case KERN_PROTECTION_FAILURE: 1482 return (EACCES); 1483 default: 1484 return (EINVAL); 1485 } 1486 } 1487