1 /* $NetBSD: sys_aio.c,v 1.6 2007/07/11 00:40:42 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 17 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 19 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * TODO: 30 * 1. Additional work for VCHR and maybe VBLK devices. 31 * 2. Consider making the job-finding O(n) per one file descriptor. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.6 2007/07/11 00:40:42 rmind Exp $"); 36 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/condvar.h> 41 #include <sys/file.h> 42 #include <sys/filedesc.h> 43 #include <sys/kernel.h> 44 #include <sys/kmem.h> 45 #include <sys/lwp.h> 46 #include <sys/mutex.h> 47 #include <sys/pool.h> 48 #include <sys/proc.h> 49 #include <sys/queue.h> 50 #include <sys/signal.h> 51 #include <sys/signalvar.h> 52 #include <sys/syscallargs.h> 53 #include <sys/sysctl.h> 54 #include <sys/systm.h> 55 #include <sys/types.h> 56 #include <sys/vnode.h> 57 58 #include <uvm/uvm_extern.h> 59 60 /* 61 * System-wide limits and counter of AIO operations. 62 * XXXSMP: We should spin-lock it, or modify atomically. 63 */ 64 static u_int aio_listio_max = AIO_LISTIO_MAX; 65 static u_int aio_max = AIO_MAX; 66 static u_int aio_jobs_count; 67 68 static struct pool aio_job_pool; 69 static struct pool aio_lio_pool; 70 71 /* Prototypes */ 72 void aio_worker(void *); 73 static void aio_process(struct aio_job *); 74 static void aio_sendsig(struct proc *, struct sigevent *); 75 static int aio_enqueue_job(int, void *, struct lio_req *); 76 77 /* 78 * Initialize the AIO system. 79 */ 80 void 81 aio_sysinit(void) 82 { 83 84 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 85 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 86 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 87 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 88 } 89 90 /* 91 * Initialize Asynchronous I/O data structures for the process. 92 */ 93 int 94 aio_init(struct proc *p) 95 { 96 struct aioproc *aio; 97 struct lwp *l; 98 bool inmem; 99 vaddr_t uaddr; 100 101 /* Allocate and initialize AIO structure */ 102 aio = kmem_zalloc(sizeof(struct aioproc), KM_NOSLEEP); 103 if (aio == NULL) 104 return EAGAIN; 105 106 /* Initialize queue and their synchronization structures */ 107 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 108 cv_init(&aio->aio_worker_cv, "aiowork"); 109 cv_init(&aio->done_cv, "aiodone"); 110 TAILQ_INIT(&aio->jobs_queue); 111 112 /* 113 * Create an AIO worker thread. 114 * XXX: Currently, AIO thread is not protected against user's actions. 115 */ 116 inmem = uvm_uarea_alloc(&uaddr); 117 if (uaddr == 0) { 118 aio_exit(p, aio); 119 return EAGAIN; 120 } 121 if (newlwp(curlwp, p, uaddr, inmem, 0, NULL, 0, 122 aio_worker, NULL, &l)) { 123 uvm_uarea_free(uaddr); 124 aio_exit(p, aio); 125 return EAGAIN; 126 } 127 128 /* Recheck if we are really first */ 129 mutex_enter(&p->p_mutex); 130 if (p->p_aio) { 131 mutex_exit(&p->p_mutex); 132 aio_exit(p, aio); 133 lwp_exit(l); 134 return 0; 135 } 136 p->p_aio = aio; 137 mutex_exit(&p->p_mutex); 138 139 /* Complete the initialization of thread, and run it */ 140 mutex_enter(&p->p_smutex); 141 aio->aio_worker = l; 142 p->p_nrlwps++; 143 lwp_lock(l); 144 l->l_stat = LSRUN; 145 l->l_usrpri = PUSER - 1; /* XXX */ 146 sched_enqueue(l, false); 147 lwp_unlock(l); 148 mutex_exit(&p->p_smutex); 149 150 return 0; 151 } 152 153 /* 154 * Exit of Asynchronous I/O subsystem of process. 155 */ 156 void 157 aio_exit(struct proc *p, struct aioproc *aio) 158 { 159 struct aio_job *a_job; 160 161 if (aio == NULL) 162 return; 163 164 /* Free AIO queue */ 165 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 166 a_job = TAILQ_FIRST(&aio->jobs_queue); 167 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 168 pool_put(&aio_job_pool, a_job); 169 aio_jobs_count--; /* XXXSMP */ 170 } 171 172 /* Destroy and free the entire AIO data structure */ 173 cv_destroy(&aio->aio_worker_cv); 174 cv_destroy(&aio->done_cv); 175 mutex_destroy(&aio->aio_mtx); 176 kmem_free(aio, sizeof(struct aioproc)); 177 } 178 179 /* 180 * AIO worker thread and processor. 181 */ 182 void 183 aio_worker(void *arg) 184 { 185 struct proc *p = curlwp->l_proc; 186 struct aioproc *aio = p->p_aio; 187 struct aio_job *a_job; 188 struct lio_req *lio; 189 sigset_t oss, nss; 190 int error, refcnt; 191 192 /* 193 * Make an empty signal mask, so it 194 * handles only SIGKILL and SIGSTOP. 195 */ 196 sigfillset(&nss); 197 mutex_enter(&p->p_smutex); 198 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 199 mutex_exit(&p->p_smutex); 200 KASSERT(error == 0); 201 202 for (;;) { 203 /* 204 * Loop for each job in the queue. If there 205 * are no jobs then sleep. 206 */ 207 mutex_enter(&aio->aio_mtx); 208 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 209 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 210 /* 211 * Thread was interrupted - check for 212 * pending exit or suspend. 213 */ 214 mutex_exit(&aio->aio_mtx); 215 lwp_userret(curlwp); 216 mutex_enter(&aio->aio_mtx); 217 } 218 } 219 220 /* Take the job from the queue */ 221 aio->curjob = a_job; 222 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 223 224 aio_jobs_count--; /* XXXSMP */ 225 aio->jobs_count--; 226 227 mutex_exit(&aio->aio_mtx); 228 229 /* Process an AIO operation */ 230 aio_process(a_job); 231 232 /* Copy data structure back to the user-space */ 233 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 234 sizeof(struct aiocb)); 235 236 mutex_enter(&aio->aio_mtx); 237 aio->curjob = NULL; 238 239 /* Decrease a reference counter, if there is a LIO structure */ 240 lio = a_job->lio; 241 refcnt = (lio != NULL ? --lio->refcnt : -1); 242 243 /* Notify all suspenders */ 244 cv_broadcast(&aio->done_cv); 245 mutex_exit(&aio->aio_mtx); 246 247 /* Send a signal, if any */ 248 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 249 250 /* Destroy the LIO structure */ 251 if (refcnt == 0) { 252 aio_sendsig(p, &lio->sig); 253 pool_put(&aio_lio_pool, lio); 254 } 255 256 /* Destroy the the job */ 257 pool_put(&aio_job_pool, a_job); 258 } 259 260 /* NOTREACHED */ 261 } 262 263 static void 264 aio_process(struct aio_job *a_job) 265 { 266 struct proc *p = curlwp->l_proc; 267 struct aiocb *aiocbp = &a_job->aiocbp; 268 struct file *fp; 269 struct filedesc *fdp = p->p_fd; 270 int fd = aiocbp->aio_fildes; 271 int error = 0; 272 273 KASSERT(fdp != NULL); 274 KASSERT(a_job->aio_op != 0); 275 276 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 277 struct iovec aiov; 278 struct uio auio; 279 280 if (aiocbp->aio_nbytes > SSIZE_MAX) { 281 error = EINVAL; 282 goto done; 283 } 284 285 fp = fd_getfile(fdp, fd); 286 if (fp == NULL) { 287 error = EBADF; 288 goto done; 289 } 290 291 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 292 aiov.iov_len = aiocbp->aio_nbytes; 293 auio.uio_iov = &aiov; 294 auio.uio_iovcnt = 1; 295 auio.uio_resid = aiocbp->aio_nbytes; 296 auio.uio_vmspace = p->p_vmspace; 297 298 FILE_USE(fp); 299 if (a_job->aio_op & AIO_READ) { 300 /* 301 * Perform a Read operation 302 */ 303 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 304 305 if ((fp->f_flag & FREAD) == 0) { 306 FILE_UNUSE(fp, curlwp); 307 error = EBADF; 308 goto done; 309 } 310 auio.uio_rw = UIO_READ; 311 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 312 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 313 } else { 314 /* 315 * Perform a Write operation 316 */ 317 KASSERT(a_job->aio_op & AIO_WRITE); 318 319 if ((fp->f_flag & FWRITE) == 0) { 320 FILE_UNUSE(fp, curlwp); 321 error = EBADF; 322 goto done; 323 } 324 auio.uio_rw = UIO_WRITE; 325 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 326 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 327 } 328 FILE_UNUSE(fp, curlwp); 329 330 /* Store the result value */ 331 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 332 a_job->aiocbp._retval = (error == 0) ? 333 a_job->aiocbp.aio_nbytes : -1; 334 335 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 336 /* 337 * Perform a file Sync operation 338 */ 339 struct vnode *vp; 340 341 if ((error = getvnode(fdp, fd, &fp)) != 0) 342 goto done; 343 344 if ((fp->f_flag & FWRITE) == 0) { 345 FILE_UNUSE(fp, curlwp); 346 error = EBADF; 347 goto done; 348 } 349 350 vp = (struct vnode *)fp->f_data; 351 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 352 if (a_job->aio_op & AIO_DSYNC) { 353 error = VOP_FSYNC(vp, fp->f_cred, 354 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0, curlwp); 355 } else if (a_job->aio_op & AIO_SYNC) { 356 error = VOP_FSYNC(vp, fp->f_cred, 357 FSYNC_WAIT, 0, 0, curlwp); 358 if (error == 0 && bioops.io_fsync != NULL && 359 vp->v_mount && 360 (vp->v_mount->mnt_flag & MNT_SOFTDEP)) 361 (*bioops.io_fsync)(vp, 0); 362 } 363 VOP_UNLOCK(vp, 0); 364 FILE_UNUSE(fp, curlwp); 365 366 /* Store the result value */ 367 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 368 369 } else 370 panic("aio_process: invalid operation code\n"); 371 372 done: 373 /* Job is done, set the error, if any */ 374 a_job->aiocbp._errno = error; 375 a_job->aiocbp._state = JOB_DONE; 376 } 377 378 /* 379 * Send AIO signal. 380 */ 381 static void 382 aio_sendsig(struct proc *p, struct sigevent *sig) 383 { 384 ksiginfo_t ksi; 385 386 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 387 return; 388 389 KSI_INIT(&ksi); 390 ksi.ksi_signo = sig->sigev_signo; 391 ksi.ksi_code = SI_ASYNCIO; 392 ksi.ksi_value = sig->sigev_value; 393 mutex_enter(&proclist_mutex); 394 kpsignal(p, &ksi, NULL); 395 mutex_exit(&proclist_mutex); 396 } 397 398 /* 399 * Enqueue the job. 400 */ 401 static int 402 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 403 { 404 struct proc *p = curlwp->l_proc; 405 struct aioproc *aio; 406 struct aio_job *a_job; 407 struct aiocb aiocbp; 408 struct sigevent *sig; 409 int error; 410 411 /* Check for the limit */ 412 if (aio_jobs_count + 1 > aio_max) /* XXXSMP */ 413 return EAGAIN; 414 415 /* Get the data structure from user-space */ 416 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 417 if (error) 418 return error; 419 420 /* Check if signal is set, and validate it */ 421 sig = &aiocbp.aio_sigevent; 422 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 423 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 424 return EINVAL; 425 426 /* Buffer and byte count */ 427 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 428 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 429 return EINVAL; 430 431 /* Check the opcode, if LIO_NOP - simply ignore */ 432 if (op == AIO_LIO) { 433 KASSERT(lio != NULL); 434 if (aiocbp.aio_lio_opcode == LIO_WRITE) 435 op = AIO_WRITE; 436 else if (aiocbp.aio_lio_opcode == LIO_READ) 437 op = AIO_READ; 438 else 439 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 440 } else { 441 KASSERT(lio == NULL); 442 } 443 444 /* 445 * Look for already existing job. If found - the job is in-progress. 446 * According to POSIX this is invalid, so return the error. 447 */ 448 aio = p->p_aio; 449 if (aio) { 450 mutex_enter(&aio->aio_mtx); 451 if (aio->curjob) { 452 a_job = aio->curjob; 453 if (a_job->aiocb_uptr == aiocb_uptr) { 454 mutex_exit(&aio->aio_mtx); 455 return EINVAL; 456 } 457 } 458 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 459 if (a_job->aiocb_uptr != aiocb_uptr) 460 continue; 461 mutex_exit(&aio->aio_mtx); 462 return EINVAL; 463 } 464 mutex_exit(&aio->aio_mtx); 465 } 466 467 /* 468 * Check if AIO structure is initialized, if not - initialize it. 469 * In LIO case, we did that already. We will recheck this with 470 * the lock in aio_init(). 471 */ 472 if (lio == NULL && p->p_aio == NULL) 473 if (aio_init(p)) 474 return EAGAIN; 475 aio = p->p_aio; 476 477 /* 478 * Set the state with errno, and copy data 479 * structure back to the user-space. 480 */ 481 aiocbp._state = JOB_WIP; 482 aiocbp._errno = EINPROGRESS; 483 aiocbp._retval = -1; 484 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 485 if (error) 486 return error; 487 488 /* Allocate and initialize a new AIO job */ 489 a_job = pool_get(&aio_job_pool, PR_WAITOK); 490 memset(a_job, 0, sizeof(struct aio_job)); 491 492 /* 493 * Set the data. 494 * Store the user-space pointer for searching. Since we 495 * are storing only per proc pointers - it is safe. 496 */ 497 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 498 a_job->aiocb_uptr = aiocb_uptr; 499 a_job->aio_op |= op; 500 a_job->lio = lio; 501 502 /* 503 * Add the job to the queue, update the counters, and 504 * notify the AIO worker thread to handle the job. 505 */ 506 mutex_enter(&aio->aio_mtx); 507 508 /* Fail, if the limit was reached */ 509 if (aio->jobs_count >= aio_listio_max) { 510 mutex_exit(&aio->aio_mtx); 511 pool_put(&aio_job_pool, a_job); 512 return EAGAIN; 513 } 514 515 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 516 aio_jobs_count++; /* XXXSMP */ 517 aio->jobs_count++; 518 if (lio) 519 lio->refcnt++; 520 cv_signal(&aio->aio_worker_cv); 521 522 mutex_exit(&aio->aio_mtx); 523 524 /* 525 * One would handle the errors only with aio_error() function. 526 * This way is appropriate according to POSIX. 527 */ 528 return 0; 529 } 530 531 /* 532 * Syscall functions. 533 */ 534 535 int 536 sys_aio_cancel(struct lwp *l, void *v, register_t *retval) 537 { 538 struct sys_aio_cancel_args /* { 539 syscallarg(int) fildes; 540 syscallarg(struct aiocb *) aiocbp; 541 } */ *uap = v; 542 struct proc *p = l->l_proc; 543 struct aioproc *aio; 544 struct aio_job *a_job; 545 struct aiocb *aiocbp_ptr; 546 struct lio_req *lio; 547 struct filedesc *fdp = p->p_fd; 548 unsigned int cn, errcnt, fildes; 549 550 TAILQ_HEAD(, aio_job) tmp_jobs_list; 551 552 /* Check for invalid file descriptor */ 553 fildes = (unsigned int)SCARG(uap, fildes); 554 if (fildes >= fdp->fd_nfiles || fdp->fd_ofiles[fildes] == NULL) 555 return EBADF; 556 557 /* Check if AIO structure is initialized */ 558 if (p->p_aio == NULL) { 559 *retval = AIO_NOTCANCELED; 560 return 0; 561 } 562 563 aio = p->p_aio; 564 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 565 566 mutex_enter(&aio->aio_mtx); 567 568 /* Cancel the jobs, and remove them from the queue */ 569 cn = 0; 570 TAILQ_INIT(&tmp_jobs_list); 571 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 572 if (aiocbp_ptr) { 573 if (aiocbp_ptr != a_job->aiocb_uptr) 574 continue; 575 if (fildes != a_job->aiocbp.aio_fildes) { 576 mutex_exit(&aio->aio_mtx); 577 return EBADF; 578 } 579 } else if (a_job->aiocbp.aio_fildes != fildes) 580 continue; 581 582 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 583 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 584 585 /* Decrease the counters */ 586 aio_jobs_count--; /* XXXSMP */ 587 aio->jobs_count--; 588 lio = a_job->lio; 589 if (lio != NULL && --lio->refcnt != 0) 590 a_job->lio = NULL; 591 592 cn++; 593 if (aiocbp_ptr) 594 break; 595 } 596 597 /* There are canceled jobs */ 598 if (cn) 599 *retval = AIO_CANCELED; 600 601 /* We cannot cancel current job */ 602 a_job = aio->curjob; 603 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 604 (a_job->aiocb_uptr == aiocbp_ptr))) 605 *retval = AIO_NOTCANCELED; 606 607 mutex_exit(&aio->aio_mtx); 608 609 /* Free the jobs after the lock */ 610 errcnt = 0; 611 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 612 a_job = TAILQ_FIRST(&tmp_jobs_list); 613 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 614 /* Set the errno and copy structures back to the user-space */ 615 a_job->aiocbp._errno = ECANCELED; 616 a_job->aiocbp._state = JOB_DONE; 617 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 618 sizeof(struct aiocb))) 619 errcnt++; 620 /* Send a signal if any */ 621 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 622 if (a_job->lio) { 623 lio = a_job->lio; 624 aio_sendsig(p, &lio->sig); 625 pool_put(&aio_lio_pool, lio); 626 } 627 pool_put(&aio_job_pool, a_job); 628 } 629 630 if (errcnt) 631 return EFAULT; 632 633 /* Set a correct return value */ 634 if (*retval == 0) 635 *retval = AIO_ALLDONE; 636 637 return 0; 638 } 639 640 int 641 sys_aio_error(struct lwp *l, void *v, register_t *retval) 642 { 643 struct sys_aio_error_args /* { 644 syscallarg(const struct aiocb *) aiocbp; 645 } */ *uap = v; 646 struct proc *p = l->l_proc; 647 struct aioproc *aio = p->p_aio; 648 struct aiocb aiocbp; 649 int error; 650 651 if (aio == NULL) 652 return EINVAL; 653 654 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 655 if (error) 656 return error; 657 658 if (aiocbp._state == JOB_NONE) 659 return EINVAL; 660 661 *retval = aiocbp._errno; 662 663 return 0; 664 } 665 666 int 667 sys_aio_fsync(struct lwp *l, void *v, register_t *retval) 668 { 669 struct sys_aio_fsync_args /* { 670 syscallarg(int) op; 671 syscallarg(struct aiocb *) aiocbp; 672 } */ *uap = v; 673 int op = SCARG(uap, op); 674 675 if ((op != O_DSYNC) && (op != O_SYNC)) 676 return EINVAL; 677 678 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 679 680 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 681 } 682 683 int 684 sys_aio_read(struct lwp *l, void *v, register_t *retval) 685 { 686 struct sys_aio_read_args /* { 687 syscallarg(struct aiocb *) aiocbp; 688 } */ *uap = v; 689 690 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 691 } 692 693 int 694 sys_aio_return(struct lwp *l, void *v, register_t *retval) 695 { 696 struct sys_aio_return_args /* { 697 syscallarg(struct aiocb *) aiocbp; 698 } */ *uap = v; 699 struct proc *p = l->l_proc; 700 struct aioproc *aio = p->p_aio; 701 struct aiocb aiocbp; 702 int error; 703 704 if (aio == NULL) 705 return EINVAL; 706 707 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 708 if (error) 709 return error; 710 711 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 712 return EINVAL; 713 714 *retval = aiocbp._retval; 715 716 /* Reset the internal variables */ 717 aiocbp._errno = 0; 718 aiocbp._retval = -1; 719 aiocbp._state = JOB_NONE; 720 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 721 722 return error; 723 } 724 725 int 726 sys_aio_suspend(struct lwp *l, void *v, register_t *retval) 727 { 728 struct sys_aio_suspend_args /* { 729 syscallarg(const struct aiocb *const[]) list; 730 syscallarg(int) nent; 731 syscallarg(const struct timespec *) timeout; 732 } */ *uap = v; 733 struct proc *p = l->l_proc; 734 struct aioproc *aio; 735 struct aio_job *a_job; 736 struct aiocb **aiocbp_list; 737 struct timespec ts; 738 int i, error, nent, timo; 739 740 if (p->p_aio == NULL) 741 return EAGAIN; 742 aio = p->p_aio; 743 744 nent = SCARG(uap, nent); 745 if (nent <= 0 || nent > aio_listio_max) 746 return EAGAIN; 747 748 if (SCARG(uap, timeout)) { 749 /* Convert timespec to ticks */ 750 error = copyin(SCARG(uap, timeout), &ts, 751 sizeof(struct timespec)); 752 if (error) 753 return error; 754 timo = mstohz((ts.tv_sec * 1000) + (ts.tv_nsec / 1000000)); 755 if (timo == 0 && ts.tv_sec == 0 && ts.tv_nsec > 0) 756 timo = 1; 757 if (timo <= 0) 758 return EAGAIN; 759 } else 760 timo = 0; 761 762 /* Get the list from user-space */ 763 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 764 error = copyin(SCARG(uap, list), aiocbp_list, 765 nent * sizeof(struct aiocb)); 766 if (error) { 767 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 768 return error; 769 } 770 771 mutex_enter(&aio->aio_mtx); 772 for (;;) { 773 774 for (i = 0; i < nent; i++) { 775 776 /* Skip NULL entries */ 777 if (aiocbp_list[i] == NULL) 778 continue; 779 780 /* Skip current job */ 781 if (aio->curjob) { 782 a_job = aio->curjob; 783 if (a_job->aiocb_uptr == aiocbp_list[i]) 784 continue; 785 } 786 787 /* Look for a job in the queue */ 788 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 789 if (a_job->aiocb_uptr == aiocbp_list[i]) 790 break; 791 792 if (a_job == NULL) { 793 struct aiocb aiocbp; 794 795 mutex_exit(&aio->aio_mtx); 796 797 error = copyin(aiocbp_list[i], &aiocbp, 798 sizeof(struct aiocb)); 799 if (error == 0 && aiocbp._state != JOB_DONE) { 800 mutex_enter(&aio->aio_mtx); 801 continue; 802 } 803 804 kmem_free(aiocbp_list, 805 nent * sizeof(struct aio_job)); 806 return error; 807 } 808 } 809 810 /* Wait for a signal or when timeout occurs */ 811 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 812 if (error) { 813 if (error == EWOULDBLOCK) 814 error = EAGAIN; 815 break; 816 } 817 } 818 mutex_exit(&aio->aio_mtx); 819 820 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 821 return error; 822 } 823 824 int 825 sys_aio_write(struct lwp *l, void *v, register_t *retval) 826 { 827 struct sys_aio_write_args /* { 828 syscallarg(struct aiocb *) aiocbp; 829 } */ *uap = v; 830 831 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 832 } 833 834 int 835 sys_lio_listio(struct lwp *l, void *v, register_t *retval) 836 { 837 struct sys_lio_listio_args /* { 838 syscallarg(int) mode; 839 syscallarg(struct aiocb *const[]) list; 840 syscallarg(int) nent; 841 syscallarg(struct sigevent *) sig; 842 } */ *uap = v; 843 struct proc *p = l->l_proc; 844 struct aioproc *aio; 845 struct aiocb **aiocbp_list; 846 struct lio_req *lio; 847 int i, error, errcnt, mode, nent; 848 849 mode = SCARG(uap, mode); 850 nent = SCARG(uap, nent); 851 852 /* Check for the limits, and invalid values */ 853 if (nent < 1 || nent > aio_listio_max) 854 return EINVAL; 855 if (aio_jobs_count + nent > aio_max) /* XXXSMP */ 856 return EAGAIN; 857 858 /* Check if AIO structure is initialized, if not - initialize it */ 859 if (p->p_aio == NULL) 860 if (aio_init(p)) 861 return EAGAIN; 862 aio = p->p_aio; 863 864 /* Create a LIO structure */ 865 lio = pool_get(&aio_lio_pool, PR_WAITOK); 866 lio->refcnt = 1; 867 error = 0; 868 869 switch (mode) { 870 case LIO_WAIT: 871 memset(&lio->sig, 0, sizeof(struct sigevent)); 872 break; 873 case LIO_NOWAIT: 874 /* Check for signal, validate it */ 875 if (SCARG(uap, sig)) { 876 struct sigevent *sig = &lio->sig; 877 878 error = copyin(SCARG(uap, sig), &lio->sig, 879 sizeof(struct sigevent)); 880 if (error == 0 && 881 (sig->sigev_signo < 0 || 882 sig->sigev_signo >= NSIG || 883 sig->sigev_notify < SIGEV_NONE || 884 sig->sigev_notify > SIGEV_SA)) 885 error = EINVAL; 886 } else 887 memset(&lio->sig, 0, sizeof(struct sigevent)); 888 break; 889 default: 890 error = EINVAL; 891 break; 892 } 893 894 if (error != 0) { 895 pool_put(&aio_lio_pool, lio); 896 return error; 897 } 898 899 /* Get the list from user-space */ 900 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 901 error = copyin(SCARG(uap, list), aiocbp_list, 902 nent * sizeof(struct aiocb)); 903 if (error) { 904 mutex_enter(&aio->aio_mtx); 905 goto err; 906 } 907 908 /* Enqueue all jobs */ 909 errcnt = 0; 910 for (i = 0; i < nent; i++) { 911 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 912 /* 913 * According to POSIX, in such error case it may 914 * fail with other I/O operations initiated. 915 */ 916 if (error) 917 errcnt++; 918 } 919 920 mutex_enter(&aio->aio_mtx); 921 922 /* Return an error, if any */ 923 if (errcnt) { 924 error = EIO; 925 goto err; 926 } 927 928 if (mode == LIO_WAIT) { 929 /* 930 * Wait for AIO completion. In such case, 931 * the LIO structure will be freed here. 932 */ 933 while (lio->refcnt > 1 && error == 0) 934 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 935 if (error) 936 error = EINTR; 937 } 938 939 err: 940 if (--lio->refcnt != 0) 941 lio = NULL; 942 mutex_exit(&aio->aio_mtx); 943 if (lio != NULL) { 944 aio_sendsig(p, &lio->sig); 945 pool_put(&aio_lio_pool, lio); 946 } 947 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 948 return error; 949 } 950 951 /* 952 * SysCtl 953 */ 954 955 static int 956 sysctl_aio_listio_max(SYSCTLFN_ARGS) 957 { 958 struct sysctlnode node; 959 int error, newsize; 960 961 node = *rnode; 962 node.sysctl_data = &newsize; 963 964 newsize = aio_listio_max; 965 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 966 if (error || newp == NULL) 967 return error; 968 969 /* XXXSMP */ 970 if (newsize < 1 || newsize > aio_max) 971 return EINVAL; 972 aio_listio_max = newsize; 973 974 return 0; 975 } 976 977 static int 978 sysctl_aio_max(SYSCTLFN_ARGS) 979 { 980 struct sysctlnode node; 981 int error, newsize; 982 983 node = *rnode; 984 node.sysctl_data = &newsize; 985 986 newsize = aio_max; 987 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 988 if (error || newp == NULL) 989 return error; 990 991 /* XXXSMP */ 992 if (newsize < 1 || newsize < aio_listio_max) 993 return EINVAL; 994 aio_max = newsize; 995 996 return 0; 997 } 998 999 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup") 1000 { 1001 1002 sysctl_createv(clog, 0, NULL, NULL, 1003 CTLFLAG_PERMANENT, 1004 CTLTYPE_NODE, "kern", NULL, 1005 NULL, 0, NULL, 0, 1006 CTL_KERN, CTL_EOL); 1007 sysctl_createv(clog, 0, NULL, NULL, 1008 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1009 CTLTYPE_INT, "posix_aio", 1010 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1011 "Asynchronous I/O option to which the " 1012 "system attempts to conform"), 1013 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1014 CTL_KERN, CTL_CREATE, CTL_EOL); 1015 sysctl_createv(clog, 0, NULL, NULL, 1016 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1017 CTLTYPE_INT, "aio_listio_max", 1018 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1019 "operations in a single list I/O call"), 1020 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1021 CTL_KERN, CTL_CREATE, CTL_EOL); 1022 sysctl_createv(clog, 0, NULL, NULL, 1023 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1024 CTLTYPE_INT, "aio_max", 1025 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1026 "operations"), 1027 sysctl_aio_max, 0, &aio_max, 0, 1028 CTL_KERN, CTL_CREATE, CTL_EOL); 1029 } 1030 1031 /* 1032 * Debugging 1033 */ 1034 #if defined(DDB) 1035 void 1036 aio_print_jobs(void (*pr)(const char *, ...)) 1037 { 1038 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1039 struct aioproc *aio; 1040 struct aio_job *a_job; 1041 struct aiocb *aiocbp; 1042 1043 if (p == NULL) { 1044 (*pr)("AIO: We are not in the processes right now.\n"); 1045 return; 1046 } 1047 1048 aio = p->p_aio; 1049 if (aio == NULL) { 1050 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1051 return; 1052 } 1053 1054 (*pr)("AIO: PID = %d\n", p->p_pid); 1055 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1056 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1057 1058 if (aio->curjob) { 1059 a_job = aio->curjob; 1060 (*pr)("\nAIO current job:\n"); 1061 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1062 a_job->aio_op, a_job->aiocbp._errno, 1063 a_job->aiocbp._state, a_job->aiocb_uptr); 1064 aiocbp = &a_job->aiocbp; 1065 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1066 aiocbp->aio_fildes, aiocbp->aio_offset, 1067 aiocbp->aio_buf, aiocbp->aio_nbytes); 1068 } 1069 1070 (*pr)("\nAIO queue:\n"); 1071 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1072 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1073 a_job->aio_op, a_job->aiocbp._errno, 1074 a_job->aiocbp._state, a_job->aiocb_uptr); 1075 aiocbp = &a_job->aiocbp; 1076 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1077 aiocbp->aio_fildes, aiocbp->aio_offset, 1078 aiocbp->aio_buf, aiocbp->aio_nbytes); 1079 } 1080 } 1081 #endif /* defined(DDB) */ 1082