1 /* $NetBSD: subr_copy.c,v 1.19 2023/05/22 14:07:24 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 1998, 1999, 2002, 2007, 2008, 2019 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993 36 * The Regents of the University of California. All rights reserved. 37 * (c) UNIX System Laboratories, Inc. 38 * All or some portions of this file are derived from material licensed 39 * to the University of California by American Telephone and Telegraph 40 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 41 * the permission of UNIX System Laboratories, Inc. 42 * 43 * Copyright (c) 1992, 1993 44 * The Regents of the University of California. All rights reserved. 45 * 46 * This software was developed by the Computer Systems Engineering group 47 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 48 * contributed to Berkeley. 49 * 50 * All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Lawrence Berkeley Laboratory. 54 * 55 * Redistribution and use in source and binary forms, with or without 56 * modification, are permitted provided that the following conditions 57 * are met: 58 * 1. Redistributions of source code must retain the above copyright 59 * notice, this list of conditions and the following disclaimer. 60 * 2. Redistributions in binary form must reproduce the above copyright 61 * notice, this list of conditions and the following disclaimer in the 62 * documentation and/or other materials provided with the distribution. 63 * 3. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_subr.c 8.4 (Berkeley) 2/14/95 80 */ 81 82 #include <sys/cdefs.h> 83 __KERNEL_RCSID(0, "$NetBSD: subr_copy.c,v 1.19 2023/05/22 14:07:24 riastradh Exp $"); 84 85 #define __UFETCHSTORE_PRIVATE 86 #define __UCAS_PRIVATE 87 88 #include <sys/param.h> 89 #include <sys/fcntl.h> 90 #include <sys/proc.h> 91 #include <sys/systm.h> 92 93 #include <uvm/uvm_extern.h> 94 95 void 96 uio_setup_sysspace(struct uio *uio) 97 { 98 99 uio->uio_vmspace = vmspace_kernel(); 100 } 101 102 int 103 uiomove(void *buf, size_t n, struct uio *uio) 104 { 105 struct vmspace *vm = uio->uio_vmspace; 106 struct iovec *iov; 107 size_t cnt; 108 int error = 0; 109 char *cp = buf; 110 111 ASSERT_SLEEPABLE(); 112 113 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE); 114 while (n > 0 && uio->uio_resid) { 115 KASSERT(uio->uio_iovcnt > 0); 116 iov = uio->uio_iov; 117 cnt = iov->iov_len; 118 if (cnt == 0) { 119 KASSERT(uio->uio_iovcnt > 1); 120 uio->uio_iov++; 121 uio->uio_iovcnt--; 122 continue; 123 } 124 if (cnt > n) 125 cnt = n; 126 if (!VMSPACE_IS_KERNEL_P(vm)) { 127 preempt_point(); 128 } 129 130 if (uio->uio_rw == UIO_READ) { 131 error = copyout_vmspace(vm, cp, iov->iov_base, 132 cnt); 133 } else { 134 error = copyin_vmspace(vm, iov->iov_base, cp, 135 cnt); 136 } 137 if (error) { 138 break; 139 } 140 iov->iov_base = (char *)iov->iov_base + cnt; 141 iov->iov_len -= cnt; 142 uio->uio_resid -= cnt; 143 uio->uio_offset += cnt; 144 cp += cnt; 145 KDASSERT(cnt <= n); 146 n -= cnt; 147 } 148 149 return (error); 150 } 151 152 /* 153 * Wrapper for uiomove() that validates the arguments against a known-good 154 * kernel buffer. 155 */ 156 int 157 uiomove_frombuf(void *buf, size_t buflen, struct uio *uio) 158 { 159 size_t offset; 160 161 if (uio->uio_offset < 0 || /* uio->uio_resid < 0 || */ 162 (offset = uio->uio_offset) != uio->uio_offset) 163 return (EINVAL); 164 if (offset >= buflen) 165 return (0); 166 return (uiomove((char *)buf + offset, buflen - offset, uio)); 167 } 168 169 int 170 uiopeek(void *buf, size_t n, struct uio *uio) 171 { 172 struct vmspace *vm = uio->uio_vmspace; 173 struct iovec *iov; 174 size_t cnt; 175 int error = 0; 176 char *cp = buf; 177 size_t resid = uio->uio_resid; 178 int iovcnt = uio->uio_iovcnt; 179 char *base; 180 size_t len; 181 182 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE); 183 184 if (n == 0 || resid == 0) 185 return 0; 186 iov = uio->uio_iov; 187 base = iov->iov_base; 188 len = iov->iov_len; 189 190 while (n > 0 && resid > 0) { 191 KASSERT(iovcnt > 0); 192 cnt = len; 193 if (cnt == 0) { 194 KASSERT(iovcnt > 1); 195 iov++; 196 iovcnt--; 197 base = iov->iov_base; 198 len = iov->iov_len; 199 continue; 200 } 201 if (cnt > n) 202 cnt = n; 203 if (!VMSPACE_IS_KERNEL_P(vm)) { 204 preempt_point(); 205 } 206 207 if (uio->uio_rw == UIO_READ) { 208 error = copyout_vmspace(vm, cp, base, cnt); 209 } else { 210 error = copyin_vmspace(vm, base, cp, cnt); 211 } 212 if (error) { 213 break; 214 } 215 base += cnt; 216 len -= cnt; 217 resid -= cnt; 218 cp += cnt; 219 KDASSERT(cnt <= n); 220 n -= cnt; 221 } 222 223 return error; 224 } 225 226 void 227 uioskip(size_t n, struct uio *uio) 228 { 229 struct iovec *iov; 230 size_t cnt; 231 232 KASSERTMSG(n <= uio->uio_resid, "n=%zu resid=%zu", n, uio->uio_resid); 233 234 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE); 235 while (n > 0 && uio->uio_resid) { 236 KASSERT(uio->uio_iovcnt > 0); 237 iov = uio->uio_iov; 238 cnt = iov->iov_len; 239 if (cnt == 0) { 240 KASSERT(uio->uio_iovcnt > 1); 241 uio->uio_iov++; 242 uio->uio_iovcnt--; 243 continue; 244 } 245 if (cnt > n) 246 cnt = n; 247 iov->iov_base = (char *)iov->iov_base + cnt; 248 iov->iov_len -= cnt; 249 uio->uio_resid -= cnt; 250 uio->uio_offset += cnt; 251 KDASSERT(cnt <= n); 252 n -= cnt; 253 } 254 } 255 256 /* 257 * Give next character to user as result of read. 258 */ 259 int 260 ureadc(int c, struct uio *uio) 261 { 262 struct iovec *iov; 263 264 if (uio->uio_resid <= 0) 265 panic("ureadc: non-positive resid"); 266 again: 267 if (uio->uio_iovcnt <= 0) 268 panic("ureadc: non-positive iovcnt"); 269 iov = uio->uio_iov; 270 if (iov->iov_len <= 0) { 271 uio->uio_iovcnt--; 272 uio->uio_iov++; 273 goto again; 274 } 275 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 276 int error; 277 if ((error = ustore_char(iov->iov_base, c)) != 0) 278 return (error); 279 } else { 280 *(char *)iov->iov_base = c; 281 } 282 iov->iov_base = (char *)iov->iov_base + 1; 283 iov->iov_len--; 284 uio->uio_resid--; 285 uio->uio_offset++; 286 return (0); 287 } 288 289 /* 290 * Like copyin(), but operates on an arbitrary vmspace. 291 */ 292 int 293 copyin_vmspace(struct vmspace *vm, const void *uaddr, void *kaddr, size_t len) 294 { 295 struct iovec iov; 296 struct uio uio; 297 int error; 298 299 if (len == 0) 300 return (0); 301 302 if (VMSPACE_IS_KERNEL_P(vm)) { 303 return kcopy(uaddr, kaddr, len); 304 } 305 if (__predict_true(vm == curproc->p_vmspace)) { 306 return copyin(uaddr, kaddr, len); 307 } 308 309 iov.iov_base = kaddr; 310 iov.iov_len = len; 311 uio.uio_iov = &iov; 312 uio.uio_iovcnt = 1; 313 uio.uio_offset = (off_t)(uintptr_t)uaddr; 314 uio.uio_resid = len; 315 uio.uio_rw = UIO_READ; 316 UIO_SETUP_SYSSPACE(&uio); 317 error = uvm_io(&vm->vm_map, &uio, 0); 318 319 return (error); 320 } 321 322 /* 323 * Like copyout(), but operates on an arbitrary vmspace. 324 */ 325 int 326 copyout_vmspace(struct vmspace *vm, const void *kaddr, void *uaddr, size_t len) 327 { 328 struct iovec iov; 329 struct uio uio; 330 int error; 331 332 if (len == 0) 333 return (0); 334 335 if (VMSPACE_IS_KERNEL_P(vm)) { 336 return kcopy(kaddr, uaddr, len); 337 } 338 if (__predict_true(vm == curproc->p_vmspace)) { 339 return copyout(kaddr, uaddr, len); 340 } 341 342 iov.iov_base = __UNCONST(kaddr); /* XXXUNCONST cast away const */ 343 iov.iov_len = len; 344 uio.uio_iov = &iov; 345 uio.uio_iovcnt = 1; 346 uio.uio_offset = (off_t)(uintptr_t)uaddr; 347 uio.uio_resid = len; 348 uio.uio_rw = UIO_WRITE; 349 UIO_SETUP_SYSSPACE(&uio); 350 error = uvm_io(&vm->vm_map, &uio, 0); 351 352 return (error); 353 } 354 355 /* 356 * Like copyin(), but operates on an arbitrary process. 357 */ 358 int 359 copyin_proc(struct proc *p, const void *uaddr, void *kaddr, size_t len) 360 { 361 struct vmspace *vm; 362 int error; 363 364 error = proc_vmspace_getref(p, &vm); 365 if (error) { 366 return error; 367 } 368 error = copyin_vmspace(vm, uaddr, kaddr, len); 369 uvmspace_free(vm); 370 371 return error; 372 } 373 374 /* 375 * Like copyout(), but operates on an arbitrary process. 376 */ 377 int 378 copyout_proc(struct proc *p, const void *kaddr, void *uaddr, size_t len) 379 { 380 struct vmspace *vm; 381 int error; 382 383 error = proc_vmspace_getref(p, &vm); 384 if (error) { 385 return error; 386 } 387 error = copyout_vmspace(vm, kaddr, uaddr, len); 388 uvmspace_free(vm); 389 390 return error; 391 } 392 393 /* 394 * Like copyin(), but operates on an arbitrary pid. 395 */ 396 int 397 copyin_pid(pid_t pid, const void *uaddr, void *kaddr, size_t len) 398 { 399 struct proc *p; 400 struct vmspace *vm; 401 int error; 402 403 mutex_enter(&proc_lock); 404 p = proc_find(pid); 405 if (p == NULL) { 406 mutex_exit(&proc_lock); 407 return ESRCH; 408 } 409 mutex_enter(p->p_lock); 410 error = proc_vmspace_getref(p, &vm); 411 mutex_exit(p->p_lock); 412 mutex_exit(&proc_lock); 413 414 if (error == 0) { 415 error = copyin_vmspace(vm, uaddr, kaddr, len); 416 uvmspace_free(vm); 417 } 418 return error; 419 } 420 421 /* 422 * Like copyin(), except it operates on kernel addresses when the FKIOCTL 423 * flag is passed in `ioctlflags' from the ioctl call. 424 */ 425 int 426 ioctl_copyin(int ioctlflags, const void *src, void *dst, size_t len) 427 { 428 if (ioctlflags & FKIOCTL) 429 return kcopy(src, dst, len); 430 return copyin(src, dst, len); 431 } 432 433 /* 434 * Like copyout(), except it operates on kernel addresses when the FKIOCTL 435 * flag is passed in `ioctlflags' from the ioctl call. 436 */ 437 int 438 ioctl_copyout(int ioctlflags, const void *src, void *dst, size_t len) 439 { 440 if (ioctlflags & FKIOCTL) 441 return kcopy(src, dst, len); 442 return copyout(src, dst, len); 443 } 444 445 /* 446 * User-space CAS / fetch / store 447 */ 448 449 #ifdef __NO_STRICT_ALIGNMENT 450 #define CHECK_ALIGNMENT(x) __nothing 451 #else /* ! __NO_STRICT_ALIGNMENT */ 452 static bool 453 ufetchstore_aligned(uintptr_t uaddr, size_t size) 454 { 455 return (uaddr & (size - 1)) == 0; 456 } 457 458 #define CHECK_ALIGNMENT() \ 459 do { \ 460 if (!ufetchstore_aligned((uintptr_t)uaddr, sizeof(*uaddr))) \ 461 return EFAULT; \ 462 } while (/*CONSTCOND*/0) 463 #endif /* __NO_STRICT_ALIGNMENT */ 464 465 /* 466 * __HAVE_UCAS_FULL platforms provide _ucas_32() and _ucas_64() themselves. 467 * _RUMPKERNEL also provides it's own _ucas_32() and _ucas_64(). 468 * 469 * In all other cases, we provide generic implementations that work on 470 * all platforms. 471 */ 472 473 #if !defined(__HAVE_UCAS_FULL) && !defined(_RUMPKERNEL) 474 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR) 475 #include <sys/atomic.h> 476 #include <sys/cpu.h> 477 #include <sys/once.h> 478 #include <sys/mutex.h> 479 #include <sys/ipi.h> 480 481 static int ucas_critical_splcookie; 482 static volatile u_int ucas_critical_pausing_cpus; 483 static u_int ucas_critical_ipi; 484 static ONCE_DECL(ucas_critical_init_once) 485 486 static void 487 ucas_critical_cpu_gate(void *arg __unused) 488 { 489 int count = SPINLOCK_BACKOFF_MIN; 490 491 KASSERT(atomic_load_relaxed(&ucas_critical_pausing_cpus) > 0); 492 493 /* 494 * Notify ucas_critical_wait that we have stopped. Using 495 * store-release ensures all our memory operations up to the 496 * IPI happen before the ucas -- no buffered stores on our end 497 * can clobber it later on, for instance. 498 * 499 * Matches atomic_load_acquire in ucas_critical_wait -- turns 500 * the following atomic_dec_uint into a store-release. 501 */ 502 membar_release(); 503 atomic_dec_uint(&ucas_critical_pausing_cpus); 504 505 /* 506 * Wait for ucas_critical_exit to reopen the gate and let us 507 * proceed. Using a load-acquire ensures the ucas happens 508 * before any of our memory operations when we return from the 509 * IPI and proceed -- we won't observe any stale cached value 510 * that the ucas overwrote, for instance. 511 * 512 * Matches atomic_store_release in ucas_critical_exit. 513 */ 514 while (atomic_load_acquire(&ucas_critical_pausing_cpus) != (u_int)-1) { 515 SPINLOCK_BACKOFF(count); 516 } 517 } 518 519 static int 520 ucas_critical_init(void) 521 { 522 523 ucas_critical_ipi = ipi_register(ucas_critical_cpu_gate, NULL); 524 return 0; 525 } 526 527 static void 528 ucas_critical_wait(void) 529 { 530 int count = SPINLOCK_BACKOFF_MIN; 531 532 /* 533 * Wait for all CPUs to stop at the gate. Using a load-acquire 534 * ensures all memory operations before they stop at the gate 535 * happen before the ucas -- no buffered stores in other CPUs 536 * can clobber it later on, for instance. 537 * 538 * Matches membar_release/atomic_dec_uint (store-release) in 539 * ucas_critical_cpu_gate. 540 */ 541 while (atomic_load_acquire(&ucas_critical_pausing_cpus) > 0) { 542 SPINLOCK_BACKOFF(count); 543 } 544 } 545 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */ 546 547 static inline void 548 ucas_critical_enter(lwp_t * const l) 549 { 550 551 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR) 552 if (ncpu > 1) { 553 RUN_ONCE(&ucas_critical_init_once, ucas_critical_init); 554 555 /* 556 * Acquire the mutex first, then go to splhigh() and 557 * broadcast the IPI to lock all of the other CPUs 558 * behind the gate. 559 * 560 * N.B. Going to splhigh() implicitly disables preemption, 561 * so there's no need to do it explicitly. 562 */ 563 mutex_enter(&cpu_lock); 564 ucas_critical_splcookie = splhigh(); 565 ucas_critical_pausing_cpus = ncpu - 1; 566 ipi_trigger_broadcast(ucas_critical_ipi, true); 567 ucas_critical_wait(); 568 return; 569 } 570 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */ 571 572 KPREEMPT_DISABLE(l); 573 } 574 575 static inline void 576 ucas_critical_exit(lwp_t * const l) 577 { 578 579 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR) 580 if (ncpu > 1) { 581 /* 582 * Open the gate and notify all CPUs in 583 * ucas_critical_cpu_gate that they can now proceed. 584 * Using a store-release ensures the ucas happens 585 * before any memory operations they issue after the 586 * IPI -- they won't observe any stale cache of the 587 * target word, for instance. 588 * 589 * Matches atomic_load_acquire in ucas_critical_cpu_gate. 590 */ 591 atomic_store_release(&ucas_critical_pausing_cpus, (u_int)-1); 592 splx(ucas_critical_splcookie); 593 mutex_exit(&cpu_lock); 594 return; 595 } 596 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */ 597 598 KPREEMPT_ENABLE(l); 599 } 600 601 int 602 _ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, uint32_t *ret) 603 { 604 lwp_t * const l = curlwp; 605 uint32_t *uva = ((void *)(uintptr_t)uaddr); 606 int error; 607 608 /* 609 * Wire the user address down to avoid taking a page fault during 610 * the critical section. 611 */ 612 error = uvm_vslock(l->l_proc->p_vmspace, uva, sizeof(*uaddr), 613 VM_PROT_READ | VM_PROT_WRITE); 614 if (error) 615 return error; 616 617 ucas_critical_enter(l); 618 error = _ufetch_32(uva, ret); 619 if (error == 0 && *ret == old) { 620 error = _ustore_32(uva, new); 621 } 622 ucas_critical_exit(l); 623 624 uvm_vsunlock(l->l_proc->p_vmspace, uva, sizeof(*uaddr)); 625 626 return error; 627 } 628 629 #ifdef _LP64 630 int 631 _ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, uint64_t *ret) 632 { 633 lwp_t * const l = curlwp; 634 uint64_t *uva = ((void *)(uintptr_t)uaddr); 635 int error; 636 637 /* 638 * Wire the user address down to avoid taking a page fault during 639 * the critical section. 640 */ 641 error = uvm_vslock(l->l_proc->p_vmspace, uva, sizeof(*uaddr), 642 VM_PROT_READ | VM_PROT_WRITE); 643 if (error) 644 return error; 645 646 ucas_critical_enter(l); 647 error = _ufetch_64(uva, ret); 648 if (error == 0 && *ret == old) { 649 error = _ustore_64(uva, new); 650 } 651 ucas_critical_exit(l); 652 653 uvm_vsunlock(l->l_proc->p_vmspace, uva, sizeof(*uaddr)); 654 655 return error; 656 } 657 #endif /* _LP64 */ 658 #endif /* ! __HAVE_UCAS_FULL && ! _RUMPKERNEL */ 659 660 int 661 ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, uint32_t *ret) 662 { 663 664 ASSERT_SLEEPABLE(); 665 CHECK_ALIGNMENT(); 666 #if (defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)) && \ 667 !defined(_RUMPKERNEL) 668 if (ncpu > 1) { 669 return _ucas_32_mp(uaddr, old, new, ret); 670 } 671 #endif /* __HAVE_UCAS_MP && MULTIPROCESSOR */ 672 return _ucas_32(uaddr, old, new, ret); 673 } 674 675 #ifdef _LP64 676 int 677 ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, uint64_t *ret) 678 { 679 680 ASSERT_SLEEPABLE(); 681 CHECK_ALIGNMENT(); 682 #if (defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)) && \ 683 !defined(_RUMPKERNEL) 684 if (ncpu > 1) { 685 return _ucas_64_mp(uaddr, old, new, ret); 686 } 687 #endif /* __HAVE_UCAS_MP && MULTIPROCESSOR */ 688 return _ucas_64(uaddr, old, new, ret); 689 } 690 #endif /* _LP64 */ 691 692 __strong_alias(ucas_int,ucas_32); 693 #ifdef _LP64 694 __strong_alias(ucas_ptr,ucas_64); 695 #else 696 __strong_alias(ucas_ptr,ucas_32); 697 #endif /* _LP64 */ 698 699 int 700 ufetch_8(const uint8_t *uaddr, uint8_t *valp) 701 { 702 703 ASSERT_SLEEPABLE(); 704 CHECK_ALIGNMENT(); 705 return _ufetch_8(uaddr, valp); 706 } 707 708 int 709 ufetch_16(const uint16_t *uaddr, uint16_t *valp) 710 { 711 712 ASSERT_SLEEPABLE(); 713 CHECK_ALIGNMENT(); 714 return _ufetch_16(uaddr, valp); 715 } 716 717 int 718 ufetch_32(const uint32_t *uaddr, uint32_t *valp) 719 { 720 721 ASSERT_SLEEPABLE(); 722 CHECK_ALIGNMENT(); 723 return _ufetch_32(uaddr, valp); 724 } 725 726 #ifdef _LP64 727 int 728 ufetch_64(const uint64_t *uaddr, uint64_t *valp) 729 { 730 731 ASSERT_SLEEPABLE(); 732 CHECK_ALIGNMENT(); 733 return _ufetch_64(uaddr, valp); 734 } 735 #endif /* _LP64 */ 736 737 __strong_alias(ufetch_char,ufetch_8); 738 __strong_alias(ufetch_short,ufetch_16); 739 __strong_alias(ufetch_int,ufetch_32); 740 #ifdef _LP64 741 __strong_alias(ufetch_long,ufetch_64); 742 __strong_alias(ufetch_ptr,ufetch_64); 743 #else 744 __strong_alias(ufetch_long,ufetch_32); 745 __strong_alias(ufetch_ptr,ufetch_32); 746 #endif /* _LP64 */ 747 748 int 749 ustore_8(uint8_t *uaddr, uint8_t val) 750 { 751 752 ASSERT_SLEEPABLE(); 753 CHECK_ALIGNMENT(); 754 return _ustore_8(uaddr, val); 755 } 756 757 int 758 ustore_16(uint16_t *uaddr, uint16_t val) 759 { 760 761 ASSERT_SLEEPABLE(); 762 CHECK_ALIGNMENT(); 763 return _ustore_16(uaddr, val); 764 } 765 766 int 767 ustore_32(uint32_t *uaddr, uint32_t val) 768 { 769 770 ASSERT_SLEEPABLE(); 771 CHECK_ALIGNMENT(); 772 return _ustore_32(uaddr, val); 773 } 774 775 #ifdef _LP64 776 int 777 ustore_64(uint64_t *uaddr, uint64_t val) 778 { 779 780 ASSERT_SLEEPABLE(); 781 CHECK_ALIGNMENT(); 782 return _ustore_64(uaddr, val); 783 } 784 #endif /* _LP64 */ 785 786 __strong_alias(ustore_char,ustore_8); 787 __strong_alias(ustore_short,ustore_16); 788 __strong_alias(ustore_int,ustore_32); 789 #ifdef _LP64 790 __strong_alias(ustore_long,ustore_64); 791 __strong_alias(ustore_ptr,ustore_64); 792 #else 793 __strong_alias(ustore_long,ustore_32); 794 __strong_alias(ustore_ptr,ustore_32); 795 #endif /* _LP64 */ 796