1 /* $NetBSD: sys_aio.c,v 1.46 2020/02/01 02:23:04 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2007 Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Implementation of POSIX asynchronous I/O. 31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.46 2020/02/01 02:23:04 riastradh Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_ddb.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/condvar.h> 43 #include <sys/file.h> 44 #include <sys/filedesc.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/lwp.h> 48 #include <sys/mutex.h> 49 #include <sys/pool.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/signal.h> 53 #include <sys/signalvar.h> 54 #include <sys/syscall.h> 55 #include <sys/syscallargs.h> 56 #include <sys/syscallvar.h> 57 #include <sys/sysctl.h> 58 #include <sys/systm.h> 59 #include <sys/types.h> 60 #include <sys/vnode.h> 61 #include <sys/atomic.h> 62 #include <sys/module.h> 63 #include <sys/buf.h> 64 65 #include <uvm/uvm_extern.h> 66 67 MODULE(MODULE_CLASS_MISC, aio, NULL); 68 69 /* 70 * System-wide limits and counter of AIO operations. 71 */ 72 u_int aio_listio_max = AIO_LISTIO_MAX; 73 static u_int aio_max = AIO_MAX; 74 static u_int aio_jobs_count; 75 76 static struct sysctllog *aio_sysctl; 77 static struct pool aio_job_pool; 78 static struct pool aio_lio_pool; 79 static void * aio_ehook; 80 81 static void aio_worker(void *); 82 static void aio_process(struct aio_job *); 83 static void aio_sendsig(struct proc *, struct sigevent *); 84 static int aio_enqueue_job(int, void *, struct lio_req *); 85 static void aio_exit(proc_t *, void *); 86 87 static int sysctl_aio_listio_max(SYSCTLFN_PROTO); 88 static int sysctl_aio_max(SYSCTLFN_PROTO); 89 static int sysctl_aio_init(void); 90 91 static const struct syscall_package aio_syscalls[] = { 92 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel }, 93 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error }, 94 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync }, 95 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read }, 96 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return }, 97 { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 }, 98 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write }, 99 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio }, 100 { 0, 0, NULL }, 101 }; 102 103 /* 104 * Tear down all AIO state. 105 */ 106 static int 107 aio_fini(bool interface) 108 { 109 int error; 110 proc_t *p; 111 112 if (interface) { 113 /* Stop syscall activity. */ 114 error = syscall_disestablish(NULL, aio_syscalls); 115 if (error != 0) 116 return error; 117 /* Abort if any processes are using AIO. */ 118 mutex_enter(proc_lock); 119 PROCLIST_FOREACH(p, &allproc) { 120 if (p->p_aio != NULL) 121 break; 122 } 123 mutex_exit(proc_lock); 124 if (p != NULL) { 125 error = syscall_establish(NULL, aio_syscalls); 126 KASSERT(error == 0); 127 return EBUSY; 128 } 129 } 130 if (aio_sysctl != NULL) 131 sysctl_teardown(&aio_sysctl); 132 133 KASSERT(aio_jobs_count == 0); 134 exithook_disestablish(aio_ehook); 135 pool_destroy(&aio_job_pool); 136 pool_destroy(&aio_lio_pool); 137 return 0; 138 } 139 140 /* 141 * Initialize global AIO state. 142 */ 143 static int 144 aio_init(void) 145 { 146 int error; 147 148 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 149 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 150 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 151 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 152 aio_ehook = exithook_establish(aio_exit, NULL); 153 154 error = sysctl_aio_init(); 155 if (error != 0) { 156 (void)aio_fini(false); 157 return error; 158 } 159 error = syscall_establish(NULL, aio_syscalls); 160 if (error != 0) 161 (void)aio_fini(false); 162 return error; 163 } 164 165 /* 166 * Module interface. 167 */ 168 static int 169 aio_modcmd(modcmd_t cmd, void *arg) 170 { 171 172 switch (cmd) { 173 case MODULE_CMD_INIT: 174 return aio_init(); 175 case MODULE_CMD_FINI: 176 return aio_fini(true); 177 default: 178 return ENOTTY; 179 } 180 } 181 182 /* 183 * Initialize Asynchronous I/O data structures for the process. 184 */ 185 static int 186 aio_procinit(struct proc *p) 187 { 188 struct aioproc *aio; 189 struct lwp *l; 190 int error; 191 vaddr_t uaddr; 192 193 /* Allocate and initialize AIO structure */ 194 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP); 195 196 /* Initialize queue and their synchronization structures */ 197 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 198 cv_init(&aio->aio_worker_cv, "aiowork"); 199 cv_init(&aio->done_cv, "aiodone"); 200 TAILQ_INIT(&aio->jobs_queue); 201 202 /* 203 * Create an AIO worker thread. 204 * XXX: Currently, AIO thread is not protected against user's actions. 205 */ 206 uaddr = uvm_uarea_alloc(); 207 if (uaddr == 0) { 208 aio_exit(p, aio); 209 return EAGAIN; 210 } 211 error = lwp_create(curlwp, p, uaddr, 0, NULL, 0, aio_worker, 212 NULL, &l, curlwp->l_class, &curlwp->l_sigmask, &curlwp->l_sigstk); 213 if (error != 0) { 214 uvm_uarea_free(uaddr); 215 aio_exit(p, aio); 216 return error; 217 } 218 219 /* Recheck if we are really first */ 220 mutex_enter(p->p_lock); 221 if (p->p_aio) { 222 mutex_exit(p->p_lock); 223 aio_exit(p, aio); 224 lwp_exit(l); 225 return 0; 226 } 227 p->p_aio = aio; 228 229 /* Complete the initialization of thread, and run it */ 230 aio->aio_worker = l; 231 lwp_lock(l); 232 lwp_changepri(l, MAXPRI_USER); 233 setrunnable(l); 234 /* LWP now unlocked */ 235 mutex_exit(p->p_lock); 236 237 return 0; 238 } 239 240 /* 241 * Exit of Asynchronous I/O subsystem of process. 242 */ 243 static void 244 aio_exit(struct proc *p, void *cookie) 245 { 246 struct aio_job *a_job; 247 struct aioproc *aio; 248 249 if (cookie != NULL) 250 aio = cookie; 251 else if ((aio = p->p_aio) == NULL) 252 return; 253 254 /* Free AIO queue */ 255 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 256 a_job = TAILQ_FIRST(&aio->jobs_queue); 257 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 258 pool_put(&aio_job_pool, a_job); 259 atomic_dec_uint(&aio_jobs_count); 260 } 261 262 /* Destroy and free the entire AIO data structure */ 263 cv_destroy(&aio->aio_worker_cv); 264 cv_destroy(&aio->done_cv); 265 mutex_destroy(&aio->aio_mtx); 266 kmem_free(aio, sizeof(struct aioproc)); 267 } 268 269 /* 270 * AIO worker thread and processor. 271 */ 272 static void 273 aio_worker(void *arg) 274 { 275 struct proc *p = curlwp->l_proc; 276 struct aioproc *aio = p->p_aio; 277 struct aio_job *a_job; 278 struct lio_req *lio; 279 sigset_t oss, nss; 280 int error __diagused, refcnt; 281 282 /* 283 * Make an empty signal mask, so it 284 * handles only SIGKILL and SIGSTOP. 285 */ 286 sigfillset(&nss); 287 mutex_enter(p->p_lock); 288 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 289 mutex_exit(p->p_lock); 290 KASSERT(error == 0); 291 292 for (;;) { 293 /* 294 * Loop for each job in the queue. If there 295 * are no jobs then sleep. 296 */ 297 mutex_enter(&aio->aio_mtx); 298 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 299 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 300 /* 301 * Thread was interrupted - check for 302 * pending exit or suspend. 303 */ 304 mutex_exit(&aio->aio_mtx); 305 lwp_userret(curlwp); 306 mutex_enter(&aio->aio_mtx); 307 } 308 } 309 310 /* Take the job from the queue */ 311 aio->curjob = a_job; 312 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 313 314 atomic_dec_uint(&aio_jobs_count); 315 aio->jobs_count--; 316 317 mutex_exit(&aio->aio_mtx); 318 319 /* Process an AIO operation */ 320 aio_process(a_job); 321 322 /* Copy data structure back to the user-space */ 323 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 324 sizeof(struct aiocb)); 325 326 mutex_enter(&aio->aio_mtx); 327 KASSERT(aio->curjob == a_job); 328 aio->curjob = NULL; 329 330 /* Decrease a reference counter, if there is a LIO structure */ 331 lio = a_job->lio; 332 refcnt = (lio != NULL ? --lio->refcnt : -1); 333 334 /* Notify all suspenders */ 335 cv_broadcast(&aio->done_cv); 336 mutex_exit(&aio->aio_mtx); 337 338 /* Send a signal, if any */ 339 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 340 341 /* Destroy the LIO structure */ 342 if (refcnt == 0) { 343 aio_sendsig(p, &lio->sig); 344 pool_put(&aio_lio_pool, lio); 345 } 346 347 /* Destroy the job */ 348 pool_put(&aio_job_pool, a_job); 349 } 350 351 /* NOTREACHED */ 352 } 353 354 static void 355 aio_process(struct aio_job *a_job) 356 { 357 struct proc *p = curlwp->l_proc; 358 struct aiocb *aiocbp = &a_job->aiocbp; 359 struct file *fp; 360 int fd = aiocbp->aio_fildes; 361 int error = 0; 362 363 KASSERT(a_job->aio_op != 0); 364 365 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 366 struct iovec aiov; 367 struct uio auio; 368 369 if (aiocbp->aio_nbytes > SSIZE_MAX) { 370 error = EINVAL; 371 goto done; 372 } 373 374 fp = fd_getfile(fd); 375 if (fp == NULL) { 376 error = EBADF; 377 goto done; 378 } 379 380 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 381 aiov.iov_len = aiocbp->aio_nbytes; 382 auio.uio_iov = &aiov; 383 auio.uio_iovcnt = 1; 384 auio.uio_resid = aiocbp->aio_nbytes; 385 auio.uio_vmspace = p->p_vmspace; 386 387 if (a_job->aio_op & AIO_READ) { 388 /* 389 * Perform a Read operation 390 */ 391 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 392 393 if ((fp->f_flag & FREAD) == 0) { 394 fd_putfile(fd); 395 error = EBADF; 396 goto done; 397 } 398 auio.uio_rw = UIO_READ; 399 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 400 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 401 } else { 402 /* 403 * Perform a Write operation 404 */ 405 KASSERT(a_job->aio_op & AIO_WRITE); 406 407 if ((fp->f_flag & FWRITE) == 0) { 408 fd_putfile(fd); 409 error = EBADF; 410 goto done; 411 } 412 auio.uio_rw = UIO_WRITE; 413 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 414 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 415 } 416 fd_putfile(fd); 417 418 /* Store the result value */ 419 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 420 a_job->aiocbp._retval = (error == 0) ? 421 a_job->aiocbp.aio_nbytes : -1; 422 423 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 424 /* 425 * Perform a file Sync operation 426 */ 427 struct vnode *vp; 428 429 if ((error = fd_getvnode(fd, &fp)) != 0) 430 goto done; 431 432 if ((fp->f_flag & FWRITE) == 0) { 433 fd_putfile(fd); 434 error = EBADF; 435 goto done; 436 } 437 438 vp = fp->f_vnode; 439 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 440 if (a_job->aio_op & AIO_DSYNC) { 441 error = VOP_FSYNC(vp, fp->f_cred, 442 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 443 } else if (a_job->aio_op & AIO_SYNC) { 444 error = VOP_FSYNC(vp, fp->f_cred, 445 FSYNC_WAIT, 0, 0); 446 } 447 VOP_UNLOCK(vp); 448 fd_putfile(fd); 449 450 /* Store the result value */ 451 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 452 453 } else 454 panic("aio_process: invalid operation code\n"); 455 456 done: 457 /* Job is done, set the error, if any */ 458 a_job->aiocbp._errno = error; 459 a_job->aiocbp._state = JOB_DONE; 460 } 461 462 /* 463 * Send AIO signal. 464 */ 465 static void 466 aio_sendsig(struct proc *p, struct sigevent *sig) 467 { 468 ksiginfo_t ksi; 469 470 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 471 return; 472 473 KSI_INIT(&ksi); 474 ksi.ksi_signo = sig->sigev_signo; 475 ksi.ksi_code = SI_ASYNCIO; 476 ksi.ksi_value = sig->sigev_value; 477 mutex_enter(proc_lock); 478 kpsignal(p, &ksi, NULL); 479 mutex_exit(proc_lock); 480 } 481 482 /* 483 * Enqueue the job. 484 */ 485 static int 486 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 487 { 488 struct proc *p = curlwp->l_proc; 489 struct aioproc *aio; 490 struct aio_job *a_job; 491 struct aiocb aiocbp; 492 struct sigevent *sig; 493 int error; 494 495 /* Non-accurate check for the limit */ 496 if (aio_jobs_count + 1 > aio_max) 497 return EAGAIN; 498 499 /* Get the data structure from user-space */ 500 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 501 if (error) 502 return error; 503 504 /* Check if signal is set, and validate it */ 505 sig = &aiocbp.aio_sigevent; 506 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 507 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 508 return EINVAL; 509 510 /* Buffer and byte count */ 511 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 512 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 513 return EINVAL; 514 515 /* Check the opcode, if LIO_NOP - simply ignore */ 516 if (op == AIO_LIO) { 517 KASSERT(lio != NULL); 518 if (aiocbp.aio_lio_opcode == LIO_WRITE) 519 op = AIO_WRITE; 520 else if (aiocbp.aio_lio_opcode == LIO_READ) 521 op = AIO_READ; 522 else 523 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 524 } else { 525 KASSERT(lio == NULL); 526 } 527 528 /* 529 * Look for already existing job. If found - the job is in-progress. 530 * According to POSIX this is invalid, so return the error. 531 */ 532 aio = p->p_aio; 533 if (aio) { 534 mutex_enter(&aio->aio_mtx); 535 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 536 if (a_job->aiocb_uptr != aiocb_uptr) 537 continue; 538 mutex_exit(&aio->aio_mtx); 539 return EINVAL; 540 } 541 mutex_exit(&aio->aio_mtx); 542 } 543 544 /* 545 * Check if AIO structure is initialized, if not - initialize it. 546 * In LIO case, we did that already. We will recheck this with 547 * the lock in aio_procinit(). 548 */ 549 if (lio == NULL && p->p_aio == NULL) 550 if (aio_procinit(p)) 551 return EAGAIN; 552 aio = p->p_aio; 553 554 /* 555 * Set the state with errno, and copy data 556 * structure back to the user-space. 557 */ 558 aiocbp._state = JOB_WIP; 559 aiocbp._errno = EINPROGRESS; 560 aiocbp._retval = -1; 561 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 562 if (error) 563 return error; 564 565 /* Allocate and initialize a new AIO job */ 566 a_job = pool_get(&aio_job_pool, PR_WAITOK | PR_ZERO); 567 568 /* 569 * Set the data. 570 * Store the user-space pointer for searching. Since we 571 * are storing only per proc pointers - it is safe. 572 */ 573 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 574 a_job->aiocb_uptr = aiocb_uptr; 575 a_job->aio_op |= op; 576 a_job->lio = lio; 577 578 /* 579 * Add the job to the queue, update the counters, and 580 * notify the AIO worker thread to handle the job. 581 */ 582 mutex_enter(&aio->aio_mtx); 583 584 /* Fail, if the limit was reached */ 585 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 586 aio->jobs_count >= aio_listio_max) { 587 atomic_dec_uint(&aio_jobs_count); 588 mutex_exit(&aio->aio_mtx); 589 pool_put(&aio_job_pool, a_job); 590 return EAGAIN; 591 } 592 593 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 594 aio->jobs_count++; 595 if (lio) 596 lio->refcnt++; 597 cv_signal(&aio->aio_worker_cv); 598 599 mutex_exit(&aio->aio_mtx); 600 601 /* 602 * One would handle the errors only with aio_error() function. 603 * This way is appropriate according to POSIX. 604 */ 605 return 0; 606 } 607 608 /* 609 * Syscall functions. 610 */ 611 612 int 613 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap, 614 register_t *retval) 615 { 616 /* { 617 syscallarg(int) fildes; 618 syscallarg(struct aiocb *) aiocbp; 619 } */ 620 struct proc *p = l->l_proc; 621 struct aioproc *aio; 622 struct aio_job *a_job; 623 struct aiocb *aiocbp_ptr; 624 struct lio_req *lio; 625 struct filedesc *fdp = p->p_fd; 626 unsigned int cn, errcnt, fildes; 627 fdtab_t *dt; 628 629 TAILQ_HEAD(, aio_job) tmp_jobs_list; 630 631 /* Check for invalid file descriptor */ 632 fildes = (unsigned int)SCARG(uap, fildes); 633 dt = atomic_load_consume(&fdp->fd_dt); 634 if (fildes >= dt->dt_nfiles) 635 return EBADF; 636 if (dt->dt_ff[fildes] == NULL || dt->dt_ff[fildes]->ff_file == NULL) 637 return EBADF; 638 639 /* Check if AIO structure is initialized */ 640 if (p->p_aio == NULL) { 641 *retval = AIO_NOTCANCELED; 642 return 0; 643 } 644 645 aio = p->p_aio; 646 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 647 648 mutex_enter(&aio->aio_mtx); 649 650 /* Cancel the jobs, and remove them from the queue */ 651 cn = 0; 652 TAILQ_INIT(&tmp_jobs_list); 653 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 654 if (aiocbp_ptr) { 655 if (aiocbp_ptr != a_job->aiocb_uptr) 656 continue; 657 if (fildes != a_job->aiocbp.aio_fildes) { 658 mutex_exit(&aio->aio_mtx); 659 return EBADF; 660 } 661 } else if (a_job->aiocbp.aio_fildes != fildes) 662 continue; 663 664 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 665 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 666 667 /* Decrease the counters */ 668 atomic_dec_uint(&aio_jobs_count); 669 aio->jobs_count--; 670 lio = a_job->lio; 671 if (lio != NULL && --lio->refcnt != 0) 672 a_job->lio = NULL; 673 674 cn++; 675 if (aiocbp_ptr) 676 break; 677 } 678 679 /* There are canceled jobs */ 680 if (cn) 681 *retval = AIO_CANCELED; 682 683 /* We cannot cancel current job */ 684 a_job = aio->curjob; 685 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 686 (a_job->aiocb_uptr == aiocbp_ptr))) 687 *retval = AIO_NOTCANCELED; 688 689 mutex_exit(&aio->aio_mtx); 690 691 /* Free the jobs after the lock */ 692 errcnt = 0; 693 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 694 a_job = TAILQ_FIRST(&tmp_jobs_list); 695 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 696 /* Set the errno and copy structures back to the user-space */ 697 a_job->aiocbp._errno = ECANCELED; 698 a_job->aiocbp._state = JOB_DONE; 699 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 700 sizeof(struct aiocb))) 701 errcnt++; 702 /* Send a signal if any */ 703 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 704 if (a_job->lio) { 705 lio = a_job->lio; 706 aio_sendsig(p, &lio->sig); 707 pool_put(&aio_lio_pool, lio); 708 } 709 pool_put(&aio_job_pool, a_job); 710 } 711 712 if (errcnt) 713 return EFAULT; 714 715 /* Set a correct return value */ 716 if (*retval == 0) 717 *retval = AIO_ALLDONE; 718 719 return 0; 720 } 721 722 int 723 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap, 724 register_t *retval) 725 { 726 /* { 727 syscallarg(const struct aiocb *) aiocbp; 728 } */ 729 struct proc *p = l->l_proc; 730 struct aioproc *aio = p->p_aio; 731 struct aiocb aiocbp; 732 int error; 733 734 if (aio == NULL) 735 return EINVAL; 736 737 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 738 if (error) 739 return error; 740 741 if (aiocbp._state == JOB_NONE) 742 return EINVAL; 743 744 *retval = aiocbp._errno; 745 746 return 0; 747 } 748 749 int 750 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap, 751 register_t *retval) 752 { 753 /* { 754 syscallarg(int) op; 755 syscallarg(struct aiocb *) aiocbp; 756 } */ 757 int op = SCARG(uap, op); 758 759 if ((op != O_DSYNC) && (op != O_SYNC)) 760 return EINVAL; 761 762 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 763 764 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 765 } 766 767 int 768 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap, 769 register_t *retval) 770 { 771 /* { 772 syscallarg(struct aiocb *) aiocbp; 773 } */ 774 775 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 776 } 777 778 int 779 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap, 780 register_t *retval) 781 { 782 /* { 783 syscallarg(struct aiocb *) aiocbp; 784 } */ 785 struct proc *p = l->l_proc; 786 struct aioproc *aio = p->p_aio; 787 struct aiocb aiocbp; 788 int error; 789 790 if (aio == NULL) 791 return EINVAL; 792 793 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 794 if (error) 795 return error; 796 797 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 798 return EINVAL; 799 800 *retval = aiocbp._retval; 801 802 /* Reset the internal variables */ 803 aiocbp._errno = 0; 804 aiocbp._retval = -1; 805 aiocbp._state = JOB_NONE; 806 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 807 808 return error; 809 } 810 811 int 812 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap, 813 register_t *retval) 814 { 815 /* { 816 syscallarg(const struct aiocb *const[]) list; 817 syscallarg(int) nent; 818 syscallarg(const struct timespec *) timeout; 819 } */ 820 struct aiocb **list; 821 struct timespec ts; 822 int error, nent; 823 824 nent = SCARG(uap, nent); 825 if (nent <= 0 || nent > aio_listio_max) 826 return EAGAIN; 827 828 if (SCARG(uap, timeout)) { 829 /* Convert timespec to ticks */ 830 error = copyin(SCARG(uap, timeout), &ts, 831 sizeof(struct timespec)); 832 if (error) 833 return error; 834 } 835 836 list = kmem_alloc(nent * sizeof(*list), KM_SLEEP); 837 error = copyin(SCARG(uap, list), list, nent * sizeof(*list)); 838 if (error) 839 goto out; 840 error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL); 841 out: 842 kmem_free(list, nent * sizeof(*list)); 843 return error; 844 } 845 846 int 847 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent, 848 struct timespec *ts) 849 { 850 struct proc *p = l->l_proc; 851 struct aioproc *aio; 852 struct aio_job *a_job; 853 int i, error, timo; 854 855 if (p->p_aio == NULL) 856 return EAGAIN; 857 aio = p->p_aio; 858 859 if (ts) { 860 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000)); 861 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0) 862 timo = 1; 863 if (timo <= 0) 864 return EAGAIN; 865 } else 866 timo = 0; 867 868 mutex_enter(&aio->aio_mtx); 869 for (;;) { 870 for (i = 0; i < nent; i++) { 871 872 /* Skip NULL entries */ 873 if (aiocbp_list[i] == NULL) 874 continue; 875 876 /* Skip current job */ 877 if (aio->curjob) { 878 a_job = aio->curjob; 879 if (a_job->aiocb_uptr == aiocbp_list[i]) 880 continue; 881 } 882 883 /* Look for a job in the queue */ 884 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 885 if (a_job->aiocb_uptr == aiocbp_list[i]) 886 break; 887 888 if (a_job == NULL) { 889 struct aiocb aiocbp; 890 891 mutex_exit(&aio->aio_mtx); 892 893 /* Check if the job is done. */ 894 error = copyin(aiocbp_list[i], &aiocbp, 895 sizeof(struct aiocb)); 896 if (error == 0 && aiocbp._state != JOB_DONE) { 897 mutex_enter(&aio->aio_mtx); 898 continue; 899 } 900 return error; 901 } 902 } 903 904 /* Wait for a signal or when timeout occurs */ 905 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 906 if (error) { 907 if (error == EWOULDBLOCK) 908 error = EAGAIN; 909 break; 910 } 911 } 912 mutex_exit(&aio->aio_mtx); 913 return error; 914 } 915 916 int 917 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap, 918 register_t *retval) 919 { 920 /* { 921 syscallarg(struct aiocb *) aiocbp; 922 } */ 923 924 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 925 } 926 927 int 928 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap, 929 register_t *retval) 930 { 931 /* { 932 syscallarg(int) mode; 933 syscallarg(struct aiocb *const[]) list; 934 syscallarg(int) nent; 935 syscallarg(struct sigevent *) sig; 936 } */ 937 struct proc *p = l->l_proc; 938 struct aioproc *aio; 939 struct aiocb **aiocbp_list; 940 struct lio_req *lio; 941 int i, error, errcnt, mode, nent; 942 943 mode = SCARG(uap, mode); 944 nent = SCARG(uap, nent); 945 946 /* Non-accurate checks for the limit and invalid values */ 947 if (nent < 1 || nent > aio_listio_max) 948 return EINVAL; 949 if (aio_jobs_count + nent > aio_max) 950 return EAGAIN; 951 952 /* Check if AIO structure is initialized, if not - initialize it */ 953 if (p->p_aio == NULL) 954 if (aio_procinit(p)) 955 return EAGAIN; 956 aio = p->p_aio; 957 958 /* Create a LIO structure */ 959 lio = pool_get(&aio_lio_pool, PR_WAITOK); 960 lio->refcnt = 1; 961 error = 0; 962 963 switch (mode) { 964 case LIO_WAIT: 965 memset(&lio->sig, 0, sizeof(struct sigevent)); 966 break; 967 case LIO_NOWAIT: 968 /* Check for signal, validate it */ 969 if (SCARG(uap, sig)) { 970 struct sigevent *sig = &lio->sig; 971 972 error = copyin(SCARG(uap, sig), &lio->sig, 973 sizeof(struct sigevent)); 974 if (error == 0 && 975 (sig->sigev_signo < 0 || 976 sig->sigev_signo >= NSIG || 977 sig->sigev_notify < SIGEV_NONE || 978 sig->sigev_notify > SIGEV_SA)) 979 error = EINVAL; 980 } else 981 memset(&lio->sig, 0, sizeof(struct sigevent)); 982 break; 983 default: 984 error = EINVAL; 985 break; 986 } 987 988 if (error != 0) { 989 pool_put(&aio_lio_pool, lio); 990 return error; 991 } 992 993 /* Get the list from user-space */ 994 aiocbp_list = kmem_alloc(nent * sizeof(*aiocbp_list), KM_SLEEP); 995 error = copyin(SCARG(uap, list), aiocbp_list, 996 nent * sizeof(*aiocbp_list)); 997 if (error) { 998 mutex_enter(&aio->aio_mtx); 999 goto err; 1000 } 1001 1002 /* Enqueue all jobs */ 1003 errcnt = 0; 1004 for (i = 0; i < nent; i++) { 1005 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 1006 /* 1007 * According to POSIX, in such error case it may 1008 * fail with other I/O operations initiated. 1009 */ 1010 if (error) 1011 errcnt++; 1012 } 1013 1014 mutex_enter(&aio->aio_mtx); 1015 1016 /* Return an error, if any */ 1017 if (errcnt) { 1018 error = EIO; 1019 goto err; 1020 } 1021 1022 if (mode == LIO_WAIT) { 1023 /* 1024 * Wait for AIO completion. In such case, 1025 * the LIO structure will be freed here. 1026 */ 1027 while (lio->refcnt > 1 && error == 0) 1028 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 1029 if (error) 1030 error = EINTR; 1031 } 1032 1033 err: 1034 if (--lio->refcnt != 0) 1035 lio = NULL; 1036 mutex_exit(&aio->aio_mtx); 1037 if (lio != NULL) { 1038 aio_sendsig(p, &lio->sig); 1039 pool_put(&aio_lio_pool, lio); 1040 } 1041 kmem_free(aiocbp_list, nent * sizeof(*aiocbp_list)); 1042 return error; 1043 } 1044 1045 /* 1046 * SysCtl 1047 */ 1048 1049 static int 1050 sysctl_aio_listio_max(SYSCTLFN_ARGS) 1051 { 1052 struct sysctlnode node; 1053 int error, newsize; 1054 1055 node = *rnode; 1056 node.sysctl_data = &newsize; 1057 1058 newsize = aio_listio_max; 1059 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1060 if (error || newp == NULL) 1061 return error; 1062 1063 if (newsize < 1 || newsize > aio_max) 1064 return EINVAL; 1065 aio_listio_max = newsize; 1066 1067 return 0; 1068 } 1069 1070 static int 1071 sysctl_aio_max(SYSCTLFN_ARGS) 1072 { 1073 struct sysctlnode node; 1074 int error, newsize; 1075 1076 node = *rnode; 1077 node.sysctl_data = &newsize; 1078 1079 newsize = aio_max; 1080 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1081 if (error || newp == NULL) 1082 return error; 1083 1084 if (newsize < 1 || newsize < aio_listio_max) 1085 return EINVAL; 1086 aio_max = newsize; 1087 1088 return 0; 1089 } 1090 1091 static int 1092 sysctl_aio_init(void) 1093 { 1094 int rv; 1095 1096 aio_sysctl = NULL; 1097 1098 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1099 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1100 CTLTYPE_INT, "posix_aio", 1101 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1102 "Asynchronous I/O option to which the " 1103 "system attempts to conform"), 1104 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1105 CTL_KERN, CTL_CREATE, CTL_EOL); 1106 1107 if (rv != 0) 1108 return rv; 1109 1110 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1111 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1112 CTLTYPE_INT, "aio_listio_max", 1113 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1114 "operations in a single list I/O call"), 1115 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1116 CTL_KERN, CTL_CREATE, CTL_EOL); 1117 1118 if (rv != 0) 1119 return rv; 1120 1121 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1122 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1123 CTLTYPE_INT, "aio_max", 1124 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1125 "operations"), 1126 sysctl_aio_max, 0, &aio_max, 0, 1127 CTL_KERN, CTL_CREATE, CTL_EOL); 1128 1129 return rv; 1130 } 1131 1132 /* 1133 * Debugging 1134 */ 1135 #if defined(DDB) 1136 void 1137 aio_print_jobs(void (*pr)(const char *, ...)) 1138 { 1139 struct proc *p = curlwp->l_proc; 1140 struct aioproc *aio; 1141 struct aio_job *a_job; 1142 struct aiocb *aiocbp; 1143 1144 if (p == NULL) { 1145 (*pr)("AIO: We are not in the processes right now.\n"); 1146 return; 1147 } 1148 1149 aio = p->p_aio; 1150 if (aio == NULL) { 1151 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1152 return; 1153 } 1154 1155 (*pr)("AIO: PID = %d\n", p->p_pid); 1156 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1157 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1158 1159 if (aio->curjob) { 1160 a_job = aio->curjob; 1161 (*pr)("\nAIO current job:\n"); 1162 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1163 a_job->aio_op, a_job->aiocbp._errno, 1164 a_job->aiocbp._state, a_job->aiocb_uptr); 1165 aiocbp = &a_job->aiocbp; 1166 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1167 aiocbp->aio_fildes, aiocbp->aio_offset, 1168 aiocbp->aio_buf, aiocbp->aio_nbytes); 1169 } 1170 1171 (*pr)("\nAIO queue:\n"); 1172 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1173 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1174 a_job->aio_op, a_job->aiocbp._errno, 1175 a_job->aiocbp._state, a_job->aiocb_uptr); 1176 aiocbp = &a_job->aiocbp; 1177 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1178 aiocbp->aio_fildes, aiocbp->aio_offset, 1179 aiocbp->aio_buf, aiocbp->aio_nbytes); 1180 } 1181 } 1182 #endif /* defined(DDB) */ 1183