1 /* 2 * (MPSAFE) 3 * 4 * Copyright (c) 1988 University of Utah. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 37 * 38 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 39 * $FreeBSD: src/sys/vm/vm_mmap.c,v 1.108.2.6 2002/07/02 20:06:19 dillon Exp $ 40 */ 41 42 /* 43 * Mapped file (mmap) interface to VM 44 */ 45 46 #include <sys/param.h> 47 #include <sys/kernel.h> 48 #include <sys/systm.h> 49 #include <sys/sysproto.h> 50 #include <sys/filedesc.h> 51 #include <sys/kern_syscall.h> 52 #include <sys/proc.h> 53 #include <sys/priv.h> 54 #include <sys/resource.h> 55 #include <sys/resourcevar.h> 56 #include <sys/vnode.h> 57 #include <sys/fcntl.h> 58 #include <sys/file.h> 59 #include <sys/mman.h> 60 #include <sys/conf.h> 61 #include <sys/stat.h> 62 #include <sys/vmmeter.h> 63 #include <sys/sysctl.h> 64 65 #include <vm/vm.h> 66 #include <vm/vm_param.h> 67 #include <sys/lock.h> 68 #include <vm/pmap.h> 69 #include <vm/vm_map.h> 70 #include <vm/vm_object.h> 71 #include <vm/vm_page.h> 72 #include <vm/vm_pager.h> 73 #include <vm/vm_pageout.h> 74 #include <vm/vm_extern.h> 75 #include <vm/vm_kern.h> 76 77 #include <sys/file2.h> 78 #include <sys/thread.h> 79 #include <sys/thread2.h> 80 #include <vm/vm_page2.h> 81 82 static int max_proc_mmap = 1000000; 83 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, ""); 84 int vkernel_enable; 85 SYSCTL_INT(_vm, OID_AUTO, vkernel_enable, CTLFLAG_RW, &vkernel_enable, 0, ""); 86 87 /* 88 * MPSAFE 89 */ 90 int 91 sys_sbrk(struct sbrk_args *uap) 92 { 93 /* Not yet implemented */ 94 return (EOPNOTSUPP); 95 } 96 97 /* 98 * sstk_args(int incr) 99 * 100 * MPSAFE 101 */ 102 int 103 sys_sstk(struct sstk_args *uap) 104 { 105 /* Not yet implemented */ 106 return (EOPNOTSUPP); 107 } 108 109 /* 110 * mmap_args(void *addr, size_t len, int prot, int flags, int fd, 111 * long pad, off_t pos) 112 * 113 * Memory Map (mmap) system call. Note that the file offset 114 * and address are allowed to be NOT page aligned, though if 115 * the MAP_FIXED flag it set, both must have the same remainder 116 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 117 * page-aligned, the actual mapping starts at trunc_page(addr) 118 * and the return value is adjusted up by the page offset. 119 * 120 * Generally speaking, only character devices which are themselves 121 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 122 * there would be no cache coherency between a descriptor and a VM mapping 123 * both to the same character device. 124 * 125 * Block devices can be mmap'd no matter what they represent. Cache coherency 126 * is maintained as long as you do not write directly to the underlying 127 * character device. 128 * 129 * No requirements 130 */ 131 int 132 kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen, 133 int uprot, int uflags, int fd, off_t upos, void **res) 134 { 135 struct thread *td = curthread; 136 struct proc *p = td->td_proc; 137 struct file *fp = NULL; 138 struct vnode *vp; 139 vm_offset_t addr; 140 vm_offset_t tmpaddr; 141 vm_size_t size, pageoff; 142 vm_prot_t prot, maxprot; 143 void *handle; 144 int flags, error; 145 off_t pos; 146 vm_object_t obj; 147 148 KKASSERT(p); 149 150 addr = (vm_offset_t) uaddr; 151 size = ulen; 152 prot = uprot & VM_PROT_ALL; 153 flags = uflags; 154 pos = upos; 155 156 /* 157 * Make sure mapping fits into numeric range etc. 158 * 159 * NOTE: We support the full unsigned range for size now. 160 */ 161 if (((flags & MAP_ANON) && (fd != -1 || pos != 0))) 162 return (EINVAL); 163 164 if (size == 0) 165 return (EINVAL); 166 167 if (flags & MAP_STACK) { 168 if (fd != -1) 169 return (EINVAL); 170 if ((prot & (PROT_READ|PROT_WRITE)) != (PROT_READ|PROT_WRITE)) 171 return (EINVAL); 172 flags |= MAP_ANON; 173 pos = 0; 174 } 175 176 /* 177 * Virtual page tables cannot be used with MAP_STACK. Apart from 178 * it not making any sense, the aux union is used by both 179 * types. 180 * 181 * Because the virtual page table is stored in the backing object 182 * and might be updated by the kernel, the mapping must be R+W. 183 */ 184 if (flags & MAP_VPAGETABLE) { 185 if (vkernel_enable == 0) 186 return (EOPNOTSUPP); 187 if (flags & MAP_STACK) 188 return (EINVAL); 189 if ((prot & (PROT_READ|PROT_WRITE)) != (PROT_READ|PROT_WRITE)) 190 return (EINVAL); 191 } 192 193 /* 194 * Align the file position to a page boundary, 195 * and save its page offset component. 196 */ 197 pageoff = (pos & PAGE_MASK); 198 pos -= pageoff; 199 200 /* Adjust size for rounding (on both ends). */ 201 size += pageoff; /* low end... */ 202 size = (vm_size_t) round_page(size); /* hi end */ 203 if (size < ulen) /* wrap */ 204 return(EINVAL); 205 206 /* 207 * Check for illegal addresses. Watch out for address wrap... Note 208 * that VM_*_ADDRESS are not constants due to casts (argh). 209 */ 210 if (flags & (MAP_FIXED | MAP_TRYFIXED)) { 211 /* 212 * The specified address must have the same remainder 213 * as the file offset taken modulo PAGE_SIZE, so it 214 * should be aligned after adjustment by pageoff. 215 */ 216 addr -= pageoff; 217 if (addr & PAGE_MASK) 218 return (EINVAL); 219 220 /* 221 * Address range must be all in user VM space and not wrap. 222 */ 223 tmpaddr = addr + size; 224 if (tmpaddr < addr) 225 return (EINVAL); 226 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 227 return (EINVAL); 228 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) 229 return (EINVAL); 230 } else { 231 /* 232 * Get a hint of where to map. It also provides mmap offset 233 * randomization if enabled. 234 */ 235 addr = vm_map_hint(p, addr, prot); 236 } 237 238 if (flags & MAP_ANON) { 239 /* 240 * Mapping blank space is trivial. 241 */ 242 handle = NULL; 243 maxprot = VM_PROT_ALL; 244 } else { 245 /* 246 * Mapping file, get fp for validation. Obtain vnode and make 247 * sure it is of appropriate type. 248 */ 249 fp = holdfp(td, fd, -1); 250 if (fp == NULL) 251 return (EBADF); 252 if (fp->f_type != DTYPE_VNODE) { 253 error = EINVAL; 254 goto done; 255 } 256 /* 257 * POSIX shared-memory objects are defined to have 258 * kernel persistence, and are not defined to support 259 * read(2)/write(2) -- or even open(2). Thus, we can 260 * use MAP_ASYNC to trade on-disk coherence for speed. 261 * The shm_open(3) library routine turns on the FPOSIXSHM 262 * flag to request this behavior. 263 */ 264 if (fp->f_flag & FPOSIXSHM) 265 flags |= MAP_NOSYNC; 266 vp = (struct vnode *) fp->f_data; 267 268 /* 269 * Validate the vnode for the operation. 270 */ 271 switch(vp->v_type) { 272 case VREG: 273 /* 274 * Get the proper underlying object 275 */ 276 if ((obj = vp->v_object) == NULL) { 277 error = EINVAL; 278 goto done; 279 } 280 KKASSERT((struct vnode *)obj->handle == vp); 281 break; 282 case VCHR: 283 /* 284 * Make sure a device has not been revoked. 285 * Mappability is handled by the device layer. 286 */ 287 if (vp->v_rdev == NULL) { 288 error = EBADF; 289 goto done; 290 } 291 break; 292 default: 293 /* 294 * Nothing else is mappable. 295 */ 296 error = EINVAL; 297 goto done; 298 } 299 300 /* 301 * XXX hack to handle use of /dev/zero to map anon memory (ala 302 * SunOS). 303 */ 304 if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) { 305 handle = NULL; 306 maxprot = VM_PROT_ALL; 307 flags |= MAP_ANON; 308 pos = 0; 309 } else { 310 /* 311 * cdevs does not provide private mappings of any kind. 312 */ 313 if (vp->v_type == VCHR && 314 (flags & (MAP_PRIVATE|MAP_COPY))) { 315 error = EINVAL; 316 goto done; 317 } 318 /* 319 * Ensure that file and memory protections are 320 * compatible. Note that we only worry about 321 * writability if mapping is shared; in this case, 322 * current and max prot are dictated by the open file. 323 * XXX use the vnode instead? Problem is: what 324 * credentials do we use for determination? What if 325 * proc does a setuid? 326 */ 327 maxprot = VM_PROT_EXECUTE; 328 if (fp->f_flag & FREAD) { 329 maxprot |= VM_PROT_READ; 330 } else if (prot & PROT_READ) { 331 error = EACCES; 332 goto done; 333 } 334 /* 335 * If we are sharing potential changes (either via 336 * MAP_SHARED or via the implicit sharing of character 337 * device mappings), and we are trying to get write 338 * permission although we opened it without asking 339 * for it, bail out. Check for superuser, only if 340 * we're at securelevel < 1, to allow the XIG X server 341 * to continue to work. 342 * 343 * PROT_WRITE + MAP_SHARED 344 */ 345 if ((flags & MAP_SHARED) != 0 || vp->v_type == VCHR) { 346 if ((fp->f_flag & FWRITE) != 0) { 347 struct vattr va; 348 if ((error = VOP_GETATTR(vp, &va))) { 349 goto done; 350 } 351 if ((va.va_flags & 352 (IMMUTABLE|APPEND)) == 0) { 353 maxprot |= VM_PROT_WRITE; 354 355 /* 356 * SHARED+RW file mmap() 357 * updates v_lastwrite_ts. 358 */ 359 if ((prot & PROT_WRITE) && 360 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY) == 0) { 361 vfs_timestamp(&vp->v_lastwrite_ts); 362 vsetflags(vp, VLASTWRITETS); 363 vn_unlock(vp); 364 } 365 } else if (prot & PROT_WRITE) { 366 error = EPERM; 367 goto done; 368 } 369 } else if ((prot & PROT_WRITE) != 0) { 370 error = EACCES; 371 goto done; 372 } 373 } else { 374 maxprot |= VM_PROT_WRITE; 375 } 376 handle = (void *)vp; 377 } 378 } 379 380 lwkt_gettoken(&vms->vm_map.token); 381 382 /* 383 * Do not allow more then a certain number of vm_map_entry structures 384 * per process. 0 to disable. 385 */ 386 if (max_proc_mmap && vms->vm_map.nentries >= max_proc_mmap) { 387 error = ENOMEM; 388 lwkt_reltoken(&vms->vm_map.token); 389 goto done; 390 } 391 392 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 393 flags, handle, pos); 394 if (error == 0) 395 *res = (void *)(addr + pageoff); 396 397 lwkt_reltoken(&vms->vm_map.token); 398 done: 399 if (fp) 400 dropfp(td, fd, fp); 401 402 return (error); 403 } 404 405 /* 406 * mmap system call handler 407 * 408 * No requirements. 409 */ 410 int 411 sys_mmap(struct mmap_args *uap) 412 { 413 int error; 414 415 error = kern_mmap(curproc->p_vmspace, uap->addr, uap->len, 416 uap->prot, uap->flags, 417 uap->fd, uap->pos, &uap->sysmsg_resultp); 418 419 return (error); 420 } 421 422 /* 423 * msync system call handler 424 * 425 * msync_args(void *addr, size_t len, int flags) 426 * 427 * No requirements 428 */ 429 int 430 sys_msync(struct msync_args *uap) 431 { 432 struct proc *p = curproc; 433 vm_offset_t addr; 434 vm_offset_t tmpaddr; 435 vm_size_t size, pageoff; 436 int flags; 437 vm_map_t map; 438 int rv; 439 440 addr = (vm_offset_t) uap->addr; 441 size = uap->len; 442 flags = uap->flags; 443 444 pageoff = (addr & PAGE_MASK); 445 addr -= pageoff; 446 size += pageoff; 447 size = (vm_size_t) round_page(size); 448 if (size < uap->len) /* wrap */ 449 return(EINVAL); 450 tmpaddr = addr + size; /* workaround gcc4 opt */ 451 if (tmpaddr < addr) /* wrap */ 452 return(EINVAL); 453 454 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 455 return (EINVAL); 456 457 map = &p->p_vmspace->vm_map; 458 459 /* 460 * map->token serializes extracting the address range for size == 0 461 * msyncs with the vm_map_clean call; if the token were not held 462 * across the two calls, an intervening munmap/mmap pair, for example, 463 * could cause msync to occur on a wrong region. 464 */ 465 lwkt_gettoken(&map->token); 466 467 /* 468 * XXX Gak! If size is zero we are supposed to sync "all modified 469 * pages with the region containing addr". Unfortunately, we don't 470 * really keep track of individual mmaps so we approximate by flushing 471 * the range of the map entry containing addr. This can be incorrect 472 * if the region splits or is coalesced with a neighbor. 473 */ 474 if (size == 0) { 475 vm_map_entry_t entry; 476 477 vm_map_lock_read(map); 478 rv = vm_map_lookup_entry(map, addr, &entry); 479 if (rv == FALSE) { 480 vm_map_unlock_read(map); 481 rv = KERN_INVALID_ADDRESS; 482 goto done; 483 } 484 addr = entry->start; 485 size = entry->end - entry->start; 486 vm_map_unlock_read(map); 487 } 488 489 /* 490 * Clean the pages and interpret the return value. 491 */ 492 rv = vm_map_clean(map, addr, addr + size, (flags & MS_ASYNC) == 0, 493 (flags & MS_INVALIDATE) != 0); 494 done: 495 lwkt_reltoken(&map->token); 496 497 switch (rv) { 498 case KERN_SUCCESS: 499 break; 500 case KERN_INVALID_ADDRESS: 501 return (EINVAL); /* Sun returns ENOMEM? */ 502 case KERN_FAILURE: 503 return (EIO); 504 default: 505 return (EINVAL); 506 } 507 508 return (0); 509 } 510 511 /* 512 * munmap system call handler 513 * 514 * munmap_args(void *addr, size_t len) 515 * 516 * No requirements 517 */ 518 int 519 sys_munmap(struct munmap_args *uap) 520 { 521 struct proc *p = curproc; 522 vm_offset_t addr; 523 vm_offset_t tmpaddr; 524 vm_size_t size, pageoff; 525 vm_map_t map; 526 527 addr = (vm_offset_t) uap->addr; 528 size = uap->len; 529 530 pageoff = (addr & PAGE_MASK); 531 addr -= pageoff; 532 size += pageoff; 533 size = (vm_size_t) round_page(size); 534 if (size < uap->len) /* wrap */ 535 return(EINVAL); 536 tmpaddr = addr + size; /* workaround gcc4 opt */ 537 if (tmpaddr < addr) /* wrap */ 538 return(EINVAL); 539 540 if (size == 0) 541 return (0); 542 543 /* 544 * Check for illegal addresses. Watch out for address wrap... Note 545 * that VM_*_ADDRESS are not constants due to casts (argh). 546 */ 547 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 548 return (EINVAL); 549 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) 550 return (EINVAL); 551 552 map = &p->p_vmspace->vm_map; 553 554 /* map->token serializes between the map check and the actual unmap */ 555 lwkt_gettoken(&map->token); 556 557 /* 558 * Make sure entire range is allocated. 559 */ 560 if (!vm_map_check_protection(map, addr, addr + size, 561 VM_PROT_NONE, FALSE)) { 562 lwkt_reltoken(&map->token); 563 return (EINVAL); 564 } 565 /* returns nothing but KERN_SUCCESS anyway */ 566 vm_map_remove(map, addr, addr + size); 567 lwkt_reltoken(&map->token); 568 return (0); 569 } 570 571 /* 572 * mprotect_args(const void *addr, size_t len, int prot) 573 * 574 * No requirements. 575 */ 576 int 577 sys_mprotect(struct mprotect_args *uap) 578 { 579 struct proc *p = curproc; 580 vm_offset_t addr; 581 vm_offset_t tmpaddr; 582 vm_size_t size, pageoff; 583 vm_prot_t prot; 584 int error; 585 586 addr = (vm_offset_t) uap->addr; 587 size = uap->len; 588 prot = uap->prot & VM_PROT_ALL; 589 590 pageoff = (addr & PAGE_MASK); 591 addr -= pageoff; 592 size += pageoff; 593 size = (vm_size_t) round_page(size); 594 if (size < uap->len) /* wrap */ 595 return(EINVAL); 596 tmpaddr = addr + size; /* workaround gcc4 opt */ 597 if (tmpaddr < addr) /* wrap */ 598 return(EINVAL); 599 600 switch (vm_map_protect(&p->p_vmspace->vm_map, addr, addr + size, 601 prot, FALSE)) { 602 case KERN_SUCCESS: 603 error = 0; 604 break; 605 case KERN_PROTECTION_FAILURE: 606 error = EACCES; 607 break; 608 default: 609 error = EINVAL; 610 break; 611 } 612 return (error); 613 } 614 615 /* 616 * minherit system call handler 617 * 618 * minherit_args(void *addr, size_t len, int inherit) 619 * 620 * No requirements. 621 */ 622 int 623 sys_minherit(struct minherit_args *uap) 624 { 625 struct proc *p = curproc; 626 vm_offset_t addr; 627 vm_offset_t tmpaddr; 628 vm_size_t size, pageoff; 629 vm_inherit_t inherit; 630 int error; 631 632 addr = (vm_offset_t)uap->addr; 633 size = uap->len; 634 inherit = uap->inherit; 635 636 pageoff = (addr & PAGE_MASK); 637 addr -= pageoff; 638 size += pageoff; 639 size = (vm_size_t) round_page(size); 640 if (size < uap->len) /* wrap */ 641 return(EINVAL); 642 tmpaddr = addr + size; /* workaround gcc4 opt */ 643 if (tmpaddr < addr) /* wrap */ 644 return(EINVAL); 645 646 switch (vm_map_inherit(&p->p_vmspace->vm_map, addr, 647 addr + size, inherit)) { 648 case KERN_SUCCESS: 649 error = 0; 650 break; 651 case KERN_PROTECTION_FAILURE: 652 error = EACCES; 653 break; 654 default: 655 error = EINVAL; 656 break; 657 } 658 return (error); 659 } 660 661 /* 662 * madvise system call handler 663 * 664 * madvise_args(void *addr, size_t len, int behav) 665 * 666 * No requirements. 667 */ 668 int 669 sys_madvise(struct madvise_args *uap) 670 { 671 struct proc *p = curproc; 672 vm_offset_t start, end; 673 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len; 674 int error; 675 676 /* 677 * Check for illegal behavior 678 */ 679 if (uap->behav < 0 || uap->behav >= MADV_CONTROL_END) 680 return (EINVAL); 681 /* 682 * Check for illegal addresses. Watch out for address wrap... Note 683 * that VM_*_ADDRESS are not constants due to casts (argh). 684 */ 685 if (tmpaddr < (vm_offset_t)uap->addr) 686 return (EINVAL); 687 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 688 return (EINVAL); 689 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) 690 return (EINVAL); 691 692 /* 693 * Since this routine is only advisory, we default to conservative 694 * behavior. 695 */ 696 start = trunc_page((vm_offset_t)uap->addr); 697 end = round_page(tmpaddr); 698 699 error = vm_map_madvise(&p->p_vmspace->vm_map, start, end, 700 uap->behav, 0); 701 return (error); 702 } 703 704 /* 705 * mcontrol system call handler 706 * 707 * mcontrol_args(void *addr, size_t len, int behav, off_t value) 708 * 709 * No requirements 710 */ 711 int 712 sys_mcontrol(struct mcontrol_args *uap) 713 { 714 struct proc *p = curproc; 715 vm_offset_t start, end; 716 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len; 717 int error; 718 719 /* 720 * Check for illegal behavior 721 */ 722 if (uap->behav < 0 || uap->behav > MADV_CONTROL_END) 723 return (EINVAL); 724 /* 725 * Check for illegal addresses. Watch out for address wrap... Note 726 * that VM_*_ADDRESS are not constants due to casts (argh). 727 */ 728 if (tmpaddr < (vm_offset_t) uap->addr) 729 return (EINVAL); 730 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) 731 return (EINVAL); 732 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) 733 return (EINVAL); 734 735 /* 736 * Since this routine is only advisory, we default to conservative 737 * behavior. 738 */ 739 start = trunc_page((vm_offset_t)uap->addr); 740 end = round_page(tmpaddr); 741 742 error = vm_map_madvise(&p->p_vmspace->vm_map, start, end, 743 uap->behav, uap->value); 744 return (error); 745 } 746 747 748 /* 749 * mincore system call handler 750 * 751 * mincore_args(const void *addr, size_t len, char *vec) 752 * 753 * No requirements 754 */ 755 int 756 sys_mincore(struct mincore_args *uap) 757 { 758 struct proc *p = curproc; 759 vm_offset_t addr, first_addr; 760 vm_offset_t end, cend; 761 pmap_t pmap; 762 vm_map_t map; 763 char *vec; 764 int error; 765 int vecindex, lastvecindex; 766 vm_map_entry_t current; 767 vm_map_entry_t entry; 768 int mincoreinfo; 769 unsigned int timestamp; 770 771 /* 772 * Make sure that the addresses presented are valid for user 773 * mode. 774 */ 775 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 776 end = addr + (vm_size_t)round_page(uap->len); 777 if (end < addr) 778 return (EINVAL); 779 if (VM_MAX_USER_ADDRESS > 0 && end > VM_MAX_USER_ADDRESS) 780 return (EINVAL); 781 782 /* 783 * Address of byte vector 784 */ 785 vec = uap->vec; 786 787 map = &p->p_vmspace->vm_map; 788 pmap = vmspace_pmap(p->p_vmspace); 789 790 lwkt_gettoken(&map->token); 791 vm_map_lock_read(map); 792 RestartScan: 793 timestamp = map->timestamp; 794 795 if (!vm_map_lookup_entry(map, addr, &entry)) 796 entry = entry->next; 797 798 /* 799 * Do this on a map entry basis so that if the pages are not 800 * in the current processes address space, we can easily look 801 * up the pages elsewhere. 802 */ 803 lastvecindex = -1; 804 for(current = entry; 805 (current != &map->header) && (current->start < end); 806 current = current->next) { 807 808 /* 809 * ignore submaps (for now) or null objects 810 */ 811 if (current->maptype != VM_MAPTYPE_NORMAL && 812 current->maptype != VM_MAPTYPE_VPAGETABLE) { 813 continue; 814 } 815 if (current->object.vm_object == NULL) 816 continue; 817 818 /* 819 * limit this scan to the current map entry and the 820 * limits for the mincore call 821 */ 822 if (addr < current->start) 823 addr = current->start; 824 cend = current->end; 825 if (cend > end) 826 cend = end; 827 828 /* 829 * scan this entry one page at a time 830 */ 831 while (addr < cend) { 832 /* 833 * Check pmap first, it is likely faster, also 834 * it can provide info as to whether we are the 835 * one referencing or modifying the page. 836 * 837 * If we have to check the VM object, only mess 838 * around with normal maps. Do not mess around 839 * with virtual page tables (XXX). 840 */ 841 mincoreinfo = pmap_mincore(pmap, addr); 842 if (mincoreinfo == 0 && 843 current->maptype == VM_MAPTYPE_NORMAL) { 844 vm_pindex_t pindex; 845 vm_ooffset_t offset; 846 vm_page_t m; 847 848 /* 849 * calculate the page index into the object 850 */ 851 offset = current->offset + (addr - current->start); 852 pindex = OFF_TO_IDX(offset); 853 854 /* 855 * if the page is resident, then gather 856 * information about it. spl protection is 857 * required to maintain the object 858 * association. And XXX what if the page is 859 * busy? What's the deal with that? 860 * 861 * XXX vm_token - legacy for pmap_ts_referenced 862 * in x86 and vkernel pmap code. 863 */ 864 lwkt_gettoken(&vm_token); 865 vm_object_hold(current->object.vm_object); 866 m = vm_page_lookup(current->object.vm_object, 867 pindex); 868 if (m && m->valid) { 869 mincoreinfo = MINCORE_INCORE; 870 if (m->dirty || pmap_is_modified(m)) 871 mincoreinfo |= MINCORE_MODIFIED_OTHER; 872 if ((m->flags & PG_REFERENCED) || 873 pmap_ts_referenced(m)) { 874 vm_page_flag_set(m, PG_REFERENCED); 875 mincoreinfo |= MINCORE_REFERENCED_OTHER; 876 } 877 } 878 vm_object_drop(current->object.vm_object); 879 lwkt_reltoken(&vm_token); 880 } 881 882 /* 883 * subyte may page fault. In case it needs to modify 884 * the map, we release the lock. 885 */ 886 vm_map_unlock_read(map); 887 888 /* 889 * calculate index into user supplied byte vector 890 */ 891 vecindex = OFF_TO_IDX(addr - first_addr); 892 893 /* 894 * If we have skipped map entries, we need to make sure that 895 * the byte vector is zeroed for those skipped entries. 896 */ 897 while((lastvecindex + 1) < vecindex) { 898 error = subyte( vec + lastvecindex, 0); 899 if (error) { 900 error = EFAULT; 901 goto done; 902 } 903 ++lastvecindex; 904 } 905 906 /* 907 * Pass the page information to the user 908 */ 909 error = subyte(vec + vecindex, mincoreinfo); 910 if (error) { 911 error = EFAULT; 912 goto done; 913 } 914 915 /* 916 * If the map has changed, due to the subyte, 917 * the previous output may be invalid. 918 */ 919 vm_map_lock_read(map); 920 if (timestamp != map->timestamp) 921 goto RestartScan; 922 923 lastvecindex = vecindex; 924 addr += PAGE_SIZE; 925 } 926 } 927 928 /* 929 * subyte may page fault. In case it needs to modify 930 * the map, we release the lock. 931 */ 932 vm_map_unlock_read(map); 933 934 /* 935 * Zero the last entries in the byte vector. 936 */ 937 vecindex = OFF_TO_IDX(end - first_addr); 938 while((lastvecindex + 1) < vecindex) { 939 error = subyte( vec + lastvecindex, 0); 940 if (error) { 941 error = EFAULT; 942 goto done; 943 } 944 ++lastvecindex; 945 } 946 947 /* 948 * If the map has changed, due to the subyte, the previous 949 * output may be invalid. 950 */ 951 vm_map_lock_read(map); 952 if (timestamp != map->timestamp) 953 goto RestartScan; 954 vm_map_unlock_read(map); 955 956 error = 0; 957 done: 958 lwkt_reltoken(&map->token); 959 return (error); 960 } 961 962 /* 963 * mlock system call handler 964 * 965 * mlock_args(const void *addr, size_t len) 966 * 967 * No requirements 968 */ 969 int 970 sys_mlock(struct mlock_args *uap) 971 { 972 vm_offset_t addr; 973 vm_offset_t tmpaddr; 974 vm_size_t size, pageoff; 975 struct thread *td = curthread; 976 struct proc *p = td->td_proc; 977 int error; 978 979 addr = (vm_offset_t) uap->addr; 980 size = uap->len; 981 982 pageoff = (addr & PAGE_MASK); 983 addr -= pageoff; 984 size += pageoff; 985 size = (vm_size_t) round_page(size); 986 if (size < uap->len) /* wrap */ 987 return (EINVAL); 988 if (size == 0) /* silently allow 0 size */ 989 return (0); 990 tmpaddr = addr + size; /* workaround gcc4 opt */ 991 if (tmpaddr < addr) /* wrap */ 992 return (EINVAL); 993 994 if (atop(size) + vmstats.v_wire_count > vm_page_max_wired) 995 return (EAGAIN); 996 997 /* 998 * We do not need to synchronize against other threads updating ucred; 999 * they update p->ucred, which is synchronized into td_ucred ourselves. 1000 */ 1001 #ifdef pmap_wired_count 1002 if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) > 1003 p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur) { 1004 return (ENOMEM); 1005 } 1006 #else 1007 error = priv_check_cred(td->td_ucred, PRIV_ROOT, 0); 1008 if (error) { 1009 return (error); 1010 } 1011 #endif 1012 error = vm_map_unwire(&p->p_vmspace->vm_map, addr, addr + size, FALSE); 1013 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1014 } 1015 1016 /* 1017 * mlockall(int how) 1018 * 1019 * No requirements 1020 */ 1021 int 1022 sys_mlockall(struct mlockall_args *uap) 1023 { 1024 struct thread *td = curthread; 1025 struct proc *p = td->td_proc; 1026 vm_map_t map = &p->p_vmspace->vm_map; 1027 vm_map_entry_t entry; 1028 int how = uap->how; 1029 int rc = KERN_SUCCESS; 1030 1031 if (((how & MCL_CURRENT) == 0) && ((how & MCL_FUTURE) == 0)) 1032 return (EINVAL); 1033 1034 rc = priv_check_cred(td->td_ucred, PRIV_ROOT, 0); 1035 if (rc) 1036 return (rc); 1037 1038 vm_map_lock(map); 1039 do { 1040 if (how & MCL_CURRENT) { 1041 for(entry = map->header.next; 1042 entry != &map->header; 1043 entry = entry->next); 1044 1045 rc = ENOSYS; 1046 break; 1047 } 1048 1049 if (how & MCL_FUTURE) 1050 map->flags |= MAP_WIREFUTURE; 1051 } while(0); 1052 vm_map_unlock(map); 1053 1054 return (rc); 1055 } 1056 1057 /* 1058 * munlockall(void) 1059 * 1060 * Unwire all user-wired map entries, cancel MCL_FUTURE. 1061 * 1062 * No requirements 1063 */ 1064 int 1065 sys_munlockall(struct munlockall_args *uap) 1066 { 1067 struct thread *td = curthread; 1068 struct proc *p = td->td_proc; 1069 vm_map_t map = &p->p_vmspace->vm_map; 1070 vm_map_entry_t entry; 1071 int rc = KERN_SUCCESS; 1072 1073 vm_map_lock(map); 1074 1075 /* Clear MAP_WIREFUTURE to cancel mlockall(MCL_FUTURE) */ 1076 map->flags &= ~MAP_WIREFUTURE; 1077 1078 retry: 1079 for (entry = map->header.next; 1080 entry != &map->header; 1081 entry = entry->next) { 1082 if ((entry->eflags & MAP_ENTRY_USER_WIRED) == 0) 1083 continue; 1084 1085 /* 1086 * If we encounter an in-transition entry, we release the 1087 * map lock and retry the scan; we do not decrement any 1088 * wired_count more than once because we do not touch 1089 * any entries with MAP_ENTRY_USER_WIRED not set. 1090 * 1091 * There is a potential interleaving with concurrent 1092 * mlockall()s here -- if we abort a scan, an mlockall() 1093 * could start, wire a number of entries before our 1094 * current position in, and then stall itself on this 1095 * or any other in-transition entry. If that occurs, when 1096 * we resume, we will unwire those entries. 1097 */ 1098 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { 1099 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; 1100 ++mycpu->gd_cnt.v_intrans_coll; 1101 ++mycpu->gd_cnt.v_intrans_wait; 1102 vm_map_transition_wait(map, 1); 1103 goto retry; 1104 } 1105 1106 KASSERT(entry->wired_count > 0, 1107 ("wired_count was 0 with USER_WIRED set! %p", entry)); 1108 1109 /* Drop wired count, if it hits zero, unwire the entry */ 1110 entry->eflags &= ~MAP_ENTRY_USER_WIRED; 1111 entry->wired_count--; 1112 if (entry->wired_count == 0) 1113 vm_fault_unwire(map, entry); 1114 } 1115 1116 vm_map_unlock(map); 1117 1118 return (rc); 1119 } 1120 1121 /* 1122 * munlock system call handler 1123 * 1124 * munlock_args(const void *addr, size_t len) 1125 * 1126 * No requirements 1127 */ 1128 int 1129 sys_munlock(struct munlock_args *uap) 1130 { 1131 struct thread *td = curthread; 1132 struct proc *p = td->td_proc; 1133 vm_offset_t addr; 1134 vm_offset_t tmpaddr; 1135 vm_size_t size, pageoff; 1136 int error; 1137 1138 addr = (vm_offset_t) uap->addr; 1139 size = uap->len; 1140 1141 pageoff = (addr & PAGE_MASK); 1142 addr -= pageoff; 1143 size += pageoff; 1144 size = (vm_size_t) round_page(size); 1145 1146 tmpaddr = addr + size; 1147 if (tmpaddr < addr) /* wrap */ 1148 return (EINVAL); 1149 if (size == 0) /* silently allow 0 size */ 1150 return (0); 1151 1152 #ifndef pmap_wired_count 1153 error = priv_check(td, PRIV_ROOT); 1154 if (error) 1155 return (error); 1156 #endif 1157 1158 error = vm_map_unwire(&p->p_vmspace->vm_map, addr, addr + size, TRUE); 1159 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1160 } 1161 1162 /* 1163 * Internal version of mmap. 1164 * Currently used by mmap, exec, and sys5 shared memory. 1165 * Handle is either a vnode pointer or NULL for MAP_ANON. 1166 * 1167 * No requirements 1168 */ 1169 int 1170 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1171 vm_prot_t maxprot, int flags, void *handle, vm_ooffset_t foff) 1172 { 1173 boolean_t fitit; 1174 vm_object_t object; 1175 vm_offset_t eaddr; 1176 vm_size_t esize; 1177 vm_size_t align; 1178 int (*uksmap)(cdev_t dev, vm_page_t fake); 1179 struct vnode *vp; 1180 struct thread *td = curthread; 1181 struct proc *p; 1182 int rv = KERN_SUCCESS; 1183 off_t objsize; 1184 int docow; 1185 int error; 1186 1187 if (size == 0) 1188 return (0); 1189 1190 objsize = round_page(size); 1191 if (objsize < size) 1192 return (EINVAL); 1193 size = objsize; 1194 1195 lwkt_gettoken(&map->token); 1196 1197 /* 1198 * XXX messy code, fixme 1199 * 1200 * NOTE: Overflow checks require discrete statements or GCC4 1201 * will optimize it out. 1202 */ 1203 if ((p = curproc) != NULL && map == &p->p_vmspace->vm_map) { 1204 esize = map->size + size; /* workaround gcc4 opt */ 1205 if (esize < map->size || 1206 esize > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1207 lwkt_reltoken(&map->token); 1208 return(ENOMEM); 1209 } 1210 } 1211 1212 /* 1213 * We currently can only deal with page aligned file offsets. 1214 * The check is here rather than in the syscall because the 1215 * kernel calls this function internally for other mmaping 1216 * operations (such as in exec) and non-aligned offsets will 1217 * cause pmap inconsistencies...so we want to be sure to 1218 * disallow this in all cases. 1219 * 1220 * NOTE: Overflow checks require discrete statements or GCC4 1221 * will optimize it out. 1222 */ 1223 if (foff & PAGE_MASK) { 1224 lwkt_reltoken(&map->token); 1225 return (EINVAL); 1226 } 1227 1228 /* 1229 * Handle alignment. For large memory maps it is possible 1230 * that the MMU can optimize the page table so align anything 1231 * that is a multiple of SEG_SIZE to SEG_SIZE. 1232 * 1233 * Also align any large mapping (bigger than 16x SG_SIZE) to a 1234 * SEG_SIZE address boundary. 1235 */ 1236 if (flags & MAP_SIZEALIGN) { 1237 align = size; 1238 if ((align ^ (align - 1)) != (align << 1) - 1) { 1239 lwkt_reltoken(&map->token); 1240 return (EINVAL); 1241 } 1242 } else if ((flags & MAP_FIXED) == 0 && 1243 ((size & SEG_MASK) == 0 || size > SEG_SIZE * 16)) { 1244 align = SEG_SIZE; 1245 } else { 1246 align = PAGE_SIZE; 1247 } 1248 1249 if ((flags & (MAP_FIXED | MAP_TRYFIXED)) == 0) { 1250 fitit = TRUE; 1251 *addr = round_page(*addr); 1252 } else { 1253 if (*addr != trunc_page(*addr)) { 1254 lwkt_reltoken(&map->token); 1255 return (EINVAL); 1256 } 1257 eaddr = *addr + size; 1258 if (eaddr < *addr) { 1259 lwkt_reltoken(&map->token); 1260 return (EINVAL); 1261 } 1262 fitit = FALSE; 1263 if ((flags & MAP_TRYFIXED) == 0) 1264 vm_map_remove(map, *addr, *addr + size); 1265 } 1266 1267 uksmap = NULL; 1268 1269 /* 1270 * Lookup/allocate object. 1271 */ 1272 if (flags & MAP_ANON) { 1273 /* 1274 * Unnamed anonymous regions always start at 0. 1275 */ 1276 if (handle) { 1277 /* 1278 * Default memory object 1279 */ 1280 object = default_pager_alloc(handle, objsize, 1281 prot, foff); 1282 if (object == NULL) { 1283 lwkt_reltoken(&map->token); 1284 return(ENOMEM); 1285 } 1286 docow = MAP_PREFAULT_PARTIAL; 1287 } else { 1288 /* 1289 * Implicit single instance of a default memory 1290 * object, so we don't need a VM object yet. 1291 */ 1292 foff = 0; 1293 object = NULL; 1294 docow = 0; 1295 } 1296 vp = NULL; 1297 } else { 1298 vp = (struct vnode *)handle; 1299 1300 /* 1301 * Non-anonymous mappings of VCHR (aka not /dev/zero) 1302 * cannot specify MAP_STACK or MAP_VPAGETABLE. 1303 */ 1304 if (vp->v_type == VCHR) { 1305 if (flags & (MAP_STACK | MAP_VPAGETABLE)) { 1306 lwkt_reltoken(&map->token); 1307 return(EINVAL); 1308 } 1309 } 1310 1311 if (vp->v_type == VCHR && vp->v_rdev->si_ops->d_uksmap) { 1312 /* 1313 * Device mappings without a VM object, typically 1314 * sharing permanently allocated kernel memory or 1315 * process-context-specific (per-process) data. 1316 * 1317 * Force them to be shared. 1318 */ 1319 uksmap = vp->v_rdev->si_ops->d_uksmap; 1320 object = NULL; 1321 docow = MAP_PREFAULT_PARTIAL; 1322 flags &= ~(MAP_PRIVATE|MAP_COPY); 1323 flags |= MAP_SHARED; 1324 } else if (vp->v_type == VCHR) { 1325 /* 1326 * Device mappings (device size unknown?). 1327 * Force them to be shared. 1328 */ 1329 error = dev_dmmap_single(vp->v_rdev, &foff, objsize, 1330 &object, prot, NULL); 1331 1332 if (error == ENODEV) { 1333 handle = (void *)(intptr_t)vp->v_rdev; 1334 object = dev_pager_alloc(handle, objsize, prot, foff); 1335 if (object == NULL) { 1336 lwkt_reltoken(&map->token); 1337 return(EINVAL); 1338 } 1339 } else if (error) { 1340 lwkt_reltoken(&map->token); 1341 return(error); 1342 } 1343 1344 docow = MAP_PREFAULT_PARTIAL; 1345 flags &= ~(MAP_PRIVATE|MAP_COPY); 1346 flags |= MAP_SHARED; 1347 } else { 1348 /* 1349 * Regular file mapping (typically). The attribute 1350 * check is for the link count test only. mmapable 1351 * vnodes must already have a VM object assigned. 1352 */ 1353 struct vattr vat; 1354 int error; 1355 1356 error = VOP_GETATTR(vp, &vat); 1357 if (error) { 1358 lwkt_reltoken(&map->token); 1359 return (error); 1360 } 1361 docow = MAP_PREFAULT_PARTIAL; 1362 object = vnode_pager_reference(vp); 1363 if (object == NULL && vp->v_type == VREG) { 1364 lwkt_reltoken(&map->token); 1365 kprintf("Warning: cannot mmap vnode %p, no " 1366 "object\n", vp); 1367 return(EINVAL); 1368 } 1369 1370 /* 1371 * If it is a regular file without any references 1372 * we do not need to sync it. 1373 */ 1374 if (vp->v_type == VREG && vat.va_nlink == 0) { 1375 flags |= MAP_NOSYNC; 1376 } 1377 } 1378 } 1379 1380 /* 1381 * Deal with the adjusted flags 1382 */ 1383 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1384 docow |= MAP_COPY_ON_WRITE; 1385 if (flags & MAP_NOSYNC) 1386 docow |= MAP_DISABLE_SYNCER; 1387 if (flags & MAP_NOCORE) 1388 docow |= MAP_DISABLE_COREDUMP; 1389 1390 /* 1391 * This may place the area in its own page directory if (size) is 1392 * large enough, otherwise it typically returns its argument. 1393 * 1394 * (object can be NULL) 1395 */ 1396 if (fitit) { 1397 *addr = pmap_addr_hint(object, *addr, size); 1398 } 1399 1400 /* 1401 * Stack mappings need special attention. 1402 * 1403 * Mappings that use virtual page tables will default to storing 1404 * the page table at offset 0. 1405 */ 1406 if (uksmap) { 1407 rv = vm_map_find(map, uksmap, vp->v_rdev, 1408 foff, addr, size, 1409 align, fitit, 1410 VM_MAPTYPE_UKSMAP, VM_SUBSYS_MMAP, 1411 prot, maxprot, docow); 1412 } else if (flags & MAP_STACK) { 1413 rv = vm_map_stack(map, *addr, size, flags, 1414 prot, maxprot, docow); 1415 } else if (flags & MAP_VPAGETABLE) { 1416 rv = vm_map_find(map, object, NULL, 1417 foff, addr, size, 1418 align, fitit, 1419 VM_MAPTYPE_VPAGETABLE, VM_SUBSYS_MMAP, 1420 prot, maxprot, docow); 1421 } else { 1422 rv = vm_map_find(map, object, NULL, 1423 foff, addr, size, 1424 align, fitit, 1425 VM_MAPTYPE_NORMAL, VM_SUBSYS_MMAP, 1426 prot, maxprot, docow); 1427 } 1428 1429 if (rv != KERN_SUCCESS) { 1430 /* 1431 * Lose the object reference. Will destroy the 1432 * object if it's an unnamed anonymous mapping 1433 * or named anonymous without other references. 1434 * 1435 * (NOTE: object can be NULL) 1436 */ 1437 vm_object_deallocate(object); 1438 goto out; 1439 } 1440 1441 /* 1442 * Shared memory is also shared with children. 1443 */ 1444 if (flags & (MAP_SHARED|MAP_INHERIT)) { 1445 rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE); 1446 if (rv != KERN_SUCCESS) { 1447 vm_map_remove(map, *addr, *addr + size); 1448 goto out; 1449 } 1450 } 1451 1452 /* If a process has marked all future mappings for wiring, do so */ 1453 if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE)) 1454 vm_map_unwire(map, *addr, *addr + size, FALSE); 1455 1456 /* 1457 * Set the access time on the vnode 1458 */ 1459 if (vp != NULL) 1460 vn_mark_atime(vp, td); 1461 out: 1462 lwkt_reltoken(&map->token); 1463 1464 switch (rv) { 1465 case KERN_SUCCESS: 1466 return (0); 1467 case KERN_INVALID_ADDRESS: 1468 case KERN_NO_SPACE: 1469 return (ENOMEM); 1470 case KERN_PROTECTION_FAILURE: 1471 return (EACCES); 1472 default: 1473 return (EINVAL); 1474 } 1475 } 1476 1477 /* 1478 * Translate a Mach VM return code to zero on success or the appropriate errno 1479 * on failure. 1480 */ 1481 int 1482 vm_mmap_to_errno(int rv) 1483 { 1484 1485 switch (rv) { 1486 case KERN_SUCCESS: 1487 return (0); 1488 case KERN_INVALID_ADDRESS: 1489 case KERN_NO_SPACE: 1490 return (ENOMEM); 1491 case KERN_PROTECTION_FAILURE: 1492 return (EACCES); 1493 default: 1494 return (EINVAL); 1495 } 1496 } 1497