1 /* $NetBSD: sys_aio.c,v 1.13 2007/11/30 17:39:43 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO: 31 * 1. Additional work for VCHR and maybe VBLK devices. 32 * 2. Consider making the job-finding O(n) per one file descriptor. 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.13 2007/11/30 17:39:43 rmind Exp $"); 37 38 #include "opt_ddb.h" 39 40 #include <sys/param.h> 41 #include <sys/condvar.h> 42 #include <sys/file.h> 43 #include <sys/filedesc.h> 44 #include <sys/kernel.h> 45 #include <sys/kmem.h> 46 #include <sys/lwp.h> 47 #include <sys/mutex.h> 48 #include <sys/pool.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/signal.h> 52 #include <sys/signalvar.h> 53 #include <sys/syscallargs.h> 54 #include <sys/sysctl.h> 55 #include <sys/systm.h> 56 #include <sys/types.h> 57 #include <sys/vnode.h> 58 #include <sys/atomic.h> 59 60 #include <uvm/uvm_extern.h> 61 62 /* 63 * System-wide limits and counter of AIO operations. 64 */ 65 static u_int aio_listio_max = AIO_LISTIO_MAX; 66 static u_int aio_max = AIO_MAX; 67 static u_int aio_jobs_count; 68 69 static struct pool aio_job_pool; 70 static struct pool aio_lio_pool; 71 72 /* Prototypes */ 73 void aio_worker(void *); 74 static void aio_process(struct aio_job *); 75 static void aio_sendsig(struct proc *, struct sigevent *); 76 static int aio_enqueue_job(int, void *, struct lio_req *); 77 78 /* 79 * Initialize the AIO system. 80 */ 81 void 82 aio_sysinit(void) 83 { 84 85 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 86 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 87 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 88 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 89 } 90 91 /* 92 * Initialize Asynchronous I/O data structures for the process. 93 */ 94 int 95 aio_init(struct proc *p) 96 { 97 struct aioproc *aio; 98 struct lwp *l; 99 int error; 100 bool inmem; 101 vaddr_t uaddr; 102 103 /* Allocate and initialize AIO structure */ 104 aio = kmem_zalloc(sizeof(struct aioproc), KM_NOSLEEP); 105 if (aio == NULL) 106 return EAGAIN; 107 108 /* Initialize queue and their synchronization structures */ 109 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 110 cv_init(&aio->aio_worker_cv, "aiowork"); 111 cv_init(&aio->done_cv, "aiodone"); 112 TAILQ_INIT(&aio->jobs_queue); 113 114 /* 115 * Create an AIO worker thread. 116 * XXX: Currently, AIO thread is not protected against user's actions. 117 */ 118 inmem = uvm_uarea_alloc(&uaddr); 119 if (uaddr == 0) { 120 aio_exit(p, aio); 121 return EAGAIN; 122 } 123 error = lwp_create(curlwp, p, uaddr, inmem, 0, NULL, 0, aio_worker, 124 NULL, &l, curlwp->l_class); 125 if (error != 0) { 126 uvm_uarea_free(uaddr, curcpu()); 127 aio_exit(p, aio); 128 return error; 129 } 130 131 /* Recheck if we are really first */ 132 mutex_enter(&p->p_mutex); 133 if (p->p_aio) { 134 mutex_exit(&p->p_mutex); 135 aio_exit(p, aio); 136 lwp_exit(l); 137 return 0; 138 } 139 p->p_aio = aio; 140 mutex_exit(&p->p_mutex); 141 142 /* Complete the initialization of thread, and run it */ 143 mutex_enter(&p->p_smutex); 144 aio->aio_worker = l; 145 p->p_nrlwps++; 146 lwp_lock(l); 147 l->l_stat = LSRUN; 148 l->l_priority = MAXPRI_USER; 149 sched_enqueue(l, false); 150 lwp_unlock(l); 151 mutex_exit(&p->p_smutex); 152 153 return 0; 154 } 155 156 /* 157 * Exit of Asynchronous I/O subsystem of process. 158 */ 159 void 160 aio_exit(struct proc *p, struct aioproc *aio) 161 { 162 struct aio_job *a_job; 163 164 if (aio == NULL) 165 return; 166 167 /* Free AIO queue */ 168 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 169 a_job = TAILQ_FIRST(&aio->jobs_queue); 170 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 171 pool_put(&aio_job_pool, a_job); 172 atomic_dec_uint(&aio_jobs_count); 173 } 174 175 /* Destroy and free the entire AIO data structure */ 176 cv_destroy(&aio->aio_worker_cv); 177 cv_destroy(&aio->done_cv); 178 mutex_destroy(&aio->aio_mtx); 179 kmem_free(aio, sizeof(struct aioproc)); 180 } 181 182 /* 183 * AIO worker thread and processor. 184 */ 185 void 186 aio_worker(void *arg) 187 { 188 struct proc *p = curlwp->l_proc; 189 struct aioproc *aio = p->p_aio; 190 struct aio_job *a_job; 191 struct lio_req *lio; 192 sigset_t oss, nss; 193 int error, refcnt; 194 195 /* 196 * Make an empty signal mask, so it 197 * handles only SIGKILL and SIGSTOP. 198 */ 199 sigfillset(&nss); 200 mutex_enter(&p->p_smutex); 201 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 202 mutex_exit(&p->p_smutex); 203 KASSERT(error == 0); 204 205 for (;;) { 206 /* 207 * Loop for each job in the queue. If there 208 * are no jobs then sleep. 209 */ 210 mutex_enter(&aio->aio_mtx); 211 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 212 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 213 /* 214 * Thread was interrupted - check for 215 * pending exit or suspend. 216 */ 217 mutex_exit(&aio->aio_mtx); 218 lwp_userret(curlwp); 219 mutex_enter(&aio->aio_mtx); 220 } 221 } 222 223 /* Take the job from the queue */ 224 aio->curjob = a_job; 225 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 226 227 atomic_dec_uint(&aio_jobs_count); 228 aio->jobs_count--; 229 230 mutex_exit(&aio->aio_mtx); 231 232 /* Process an AIO operation */ 233 aio_process(a_job); 234 235 /* Copy data structure back to the user-space */ 236 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 237 sizeof(struct aiocb)); 238 239 mutex_enter(&aio->aio_mtx); 240 aio->curjob = NULL; 241 242 /* Decrease a reference counter, if there is a LIO structure */ 243 lio = a_job->lio; 244 refcnt = (lio != NULL ? --lio->refcnt : -1); 245 246 /* Notify all suspenders */ 247 cv_broadcast(&aio->done_cv); 248 mutex_exit(&aio->aio_mtx); 249 250 /* Send a signal, if any */ 251 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 252 253 /* Destroy the LIO structure */ 254 if (refcnt == 0) { 255 aio_sendsig(p, &lio->sig); 256 pool_put(&aio_lio_pool, lio); 257 } 258 259 /* Destroy the the job */ 260 pool_put(&aio_job_pool, a_job); 261 } 262 263 /* NOTREACHED */ 264 } 265 266 static void 267 aio_process(struct aio_job *a_job) 268 { 269 struct proc *p = curlwp->l_proc; 270 struct aiocb *aiocbp = &a_job->aiocbp; 271 struct file *fp; 272 struct filedesc *fdp = p->p_fd; 273 int fd = aiocbp->aio_fildes; 274 int error = 0; 275 276 KASSERT(fdp != NULL); 277 KASSERT(a_job->aio_op != 0); 278 279 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 280 struct iovec aiov; 281 struct uio auio; 282 283 if (aiocbp->aio_nbytes > SSIZE_MAX) { 284 error = EINVAL; 285 goto done; 286 } 287 288 fp = fd_getfile(fdp, fd); 289 if (fp == NULL) { 290 error = EBADF; 291 goto done; 292 } 293 294 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 295 aiov.iov_len = aiocbp->aio_nbytes; 296 auio.uio_iov = &aiov; 297 auio.uio_iovcnt = 1; 298 auio.uio_resid = aiocbp->aio_nbytes; 299 auio.uio_vmspace = p->p_vmspace; 300 301 FILE_USE(fp); 302 if (a_job->aio_op & AIO_READ) { 303 /* 304 * Perform a Read operation 305 */ 306 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 307 308 if ((fp->f_flag & FREAD) == 0) { 309 FILE_UNUSE(fp, curlwp); 310 error = EBADF; 311 goto done; 312 } 313 auio.uio_rw = UIO_READ; 314 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 315 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 316 } else { 317 /* 318 * Perform a Write operation 319 */ 320 KASSERT(a_job->aio_op & AIO_WRITE); 321 322 if ((fp->f_flag & FWRITE) == 0) { 323 FILE_UNUSE(fp, curlwp); 324 error = EBADF; 325 goto done; 326 } 327 auio.uio_rw = UIO_WRITE; 328 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 329 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 330 } 331 FILE_UNUSE(fp, curlwp); 332 333 /* Store the result value */ 334 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 335 a_job->aiocbp._retval = (error == 0) ? 336 a_job->aiocbp.aio_nbytes : -1; 337 338 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 339 /* 340 * Perform a file Sync operation 341 */ 342 struct vnode *vp; 343 344 if ((error = getvnode(fdp, fd, &fp)) != 0) 345 goto done; 346 347 if ((fp->f_flag & FWRITE) == 0) { 348 FILE_UNUSE(fp, curlwp); 349 error = EBADF; 350 goto done; 351 } 352 353 vp = (struct vnode *)fp->f_data; 354 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 355 if (a_job->aio_op & AIO_DSYNC) { 356 error = VOP_FSYNC(vp, fp->f_cred, 357 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 358 } else if (a_job->aio_op & AIO_SYNC) { 359 error = VOP_FSYNC(vp, fp->f_cred, 360 FSYNC_WAIT, 0, 0); 361 if (error == 0 && bioopsp != NULL && 362 vp->v_mount && 363 (vp->v_mount->mnt_flag & MNT_SOFTDEP)) 364 bioopsp->io_fsync(vp, 0); 365 } 366 VOP_UNLOCK(vp, 0); 367 FILE_UNUSE(fp, curlwp); 368 369 /* Store the result value */ 370 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 371 372 } else 373 panic("aio_process: invalid operation code\n"); 374 375 done: 376 /* Job is done, set the error, if any */ 377 a_job->aiocbp._errno = error; 378 a_job->aiocbp._state = JOB_DONE; 379 } 380 381 /* 382 * Send AIO signal. 383 */ 384 static void 385 aio_sendsig(struct proc *p, struct sigevent *sig) 386 { 387 ksiginfo_t ksi; 388 389 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 390 return; 391 392 KSI_INIT(&ksi); 393 ksi.ksi_signo = sig->sigev_signo; 394 ksi.ksi_code = SI_ASYNCIO; 395 ksi.ksi_value = sig->sigev_value; 396 mutex_enter(&proclist_mutex); 397 kpsignal(p, &ksi, NULL); 398 mutex_exit(&proclist_mutex); 399 } 400 401 /* 402 * Enqueue the job. 403 */ 404 static int 405 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 406 { 407 struct proc *p = curlwp->l_proc; 408 struct aioproc *aio; 409 struct aio_job *a_job; 410 struct aiocb aiocbp; 411 struct sigevent *sig; 412 int error; 413 414 /* Non-accurate check for the limit */ 415 if (aio_jobs_count + 1 > aio_max) 416 return EAGAIN; 417 418 /* Get the data structure from user-space */ 419 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 420 if (error) 421 return error; 422 423 /* Check if signal is set, and validate it */ 424 sig = &aiocbp.aio_sigevent; 425 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 426 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 427 return EINVAL; 428 429 /* Buffer and byte count */ 430 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 431 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 432 return EINVAL; 433 434 /* Check the opcode, if LIO_NOP - simply ignore */ 435 if (op == AIO_LIO) { 436 KASSERT(lio != NULL); 437 if (aiocbp.aio_lio_opcode == LIO_WRITE) 438 op = AIO_WRITE; 439 else if (aiocbp.aio_lio_opcode == LIO_READ) 440 op = AIO_READ; 441 else 442 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 443 } else { 444 KASSERT(lio == NULL); 445 } 446 447 /* 448 * Look for already existing job. If found - the job is in-progress. 449 * According to POSIX this is invalid, so return the error. 450 */ 451 aio = p->p_aio; 452 if (aio) { 453 mutex_enter(&aio->aio_mtx); 454 if (aio->curjob) { 455 a_job = aio->curjob; 456 if (a_job->aiocb_uptr == aiocb_uptr) { 457 mutex_exit(&aio->aio_mtx); 458 return EINVAL; 459 } 460 } 461 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 462 if (a_job->aiocb_uptr != aiocb_uptr) 463 continue; 464 mutex_exit(&aio->aio_mtx); 465 return EINVAL; 466 } 467 mutex_exit(&aio->aio_mtx); 468 } 469 470 /* 471 * Check if AIO structure is initialized, if not - initialize it. 472 * In LIO case, we did that already. We will recheck this with 473 * the lock in aio_init(). 474 */ 475 if (lio == NULL && p->p_aio == NULL) 476 if (aio_init(p)) 477 return EAGAIN; 478 aio = p->p_aio; 479 480 /* 481 * Set the state with errno, and copy data 482 * structure back to the user-space. 483 */ 484 aiocbp._state = JOB_WIP; 485 aiocbp._errno = EINPROGRESS; 486 aiocbp._retval = -1; 487 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 488 if (error) 489 return error; 490 491 /* Allocate and initialize a new AIO job */ 492 a_job = pool_get(&aio_job_pool, PR_WAITOK); 493 memset(a_job, 0, sizeof(struct aio_job)); 494 495 /* 496 * Set the data. 497 * Store the user-space pointer for searching. Since we 498 * are storing only per proc pointers - it is safe. 499 */ 500 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 501 a_job->aiocb_uptr = aiocb_uptr; 502 a_job->aio_op |= op; 503 a_job->lio = lio; 504 505 /* 506 * Add the job to the queue, update the counters, and 507 * notify the AIO worker thread to handle the job. 508 */ 509 mutex_enter(&aio->aio_mtx); 510 511 /* Fail, if the limit was reached */ 512 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 513 aio->jobs_count >= aio_listio_max) { 514 atomic_dec_uint(&aio_jobs_count); 515 mutex_exit(&aio->aio_mtx); 516 pool_put(&aio_job_pool, a_job); 517 return EAGAIN; 518 } 519 520 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 521 aio->jobs_count++; 522 if (lio) 523 lio->refcnt++; 524 cv_signal(&aio->aio_worker_cv); 525 526 mutex_exit(&aio->aio_mtx); 527 528 /* 529 * One would handle the errors only with aio_error() function. 530 * This way is appropriate according to POSIX. 531 */ 532 return 0; 533 } 534 535 /* 536 * Syscall functions. 537 */ 538 539 int 540 sys_aio_cancel(struct lwp *l, void *v, register_t *retval) 541 { 542 struct sys_aio_cancel_args /* { 543 syscallarg(int) fildes; 544 syscallarg(struct aiocb *) aiocbp; 545 } */ *uap = v; 546 struct proc *p = l->l_proc; 547 struct aioproc *aio; 548 struct aio_job *a_job; 549 struct aiocb *aiocbp_ptr; 550 struct lio_req *lio; 551 struct filedesc *fdp = p->p_fd; 552 unsigned int cn, errcnt, fildes; 553 554 TAILQ_HEAD(, aio_job) tmp_jobs_list; 555 556 /* Check for invalid file descriptor */ 557 fildes = (unsigned int)SCARG(uap, fildes); 558 if (fildes >= fdp->fd_nfiles || fdp->fd_ofiles[fildes] == NULL) 559 return EBADF; 560 561 /* Check if AIO structure is initialized */ 562 if (p->p_aio == NULL) { 563 *retval = AIO_NOTCANCELED; 564 return 0; 565 } 566 567 aio = p->p_aio; 568 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 569 570 mutex_enter(&aio->aio_mtx); 571 572 /* Cancel the jobs, and remove them from the queue */ 573 cn = 0; 574 TAILQ_INIT(&tmp_jobs_list); 575 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 576 if (aiocbp_ptr) { 577 if (aiocbp_ptr != a_job->aiocb_uptr) 578 continue; 579 if (fildes != a_job->aiocbp.aio_fildes) { 580 mutex_exit(&aio->aio_mtx); 581 return EBADF; 582 } 583 } else if (a_job->aiocbp.aio_fildes != fildes) 584 continue; 585 586 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 587 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 588 589 /* Decrease the counters */ 590 atomic_dec_uint(&aio_jobs_count); 591 aio->jobs_count--; 592 lio = a_job->lio; 593 if (lio != NULL && --lio->refcnt != 0) 594 a_job->lio = NULL; 595 596 cn++; 597 if (aiocbp_ptr) 598 break; 599 } 600 601 /* There are canceled jobs */ 602 if (cn) 603 *retval = AIO_CANCELED; 604 605 /* We cannot cancel current job */ 606 a_job = aio->curjob; 607 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 608 (a_job->aiocb_uptr == aiocbp_ptr))) 609 *retval = AIO_NOTCANCELED; 610 611 mutex_exit(&aio->aio_mtx); 612 613 /* Free the jobs after the lock */ 614 errcnt = 0; 615 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 616 a_job = TAILQ_FIRST(&tmp_jobs_list); 617 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 618 /* Set the errno and copy structures back to the user-space */ 619 a_job->aiocbp._errno = ECANCELED; 620 a_job->aiocbp._state = JOB_DONE; 621 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 622 sizeof(struct aiocb))) 623 errcnt++; 624 /* Send a signal if any */ 625 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 626 if (a_job->lio) { 627 lio = a_job->lio; 628 aio_sendsig(p, &lio->sig); 629 pool_put(&aio_lio_pool, lio); 630 } 631 pool_put(&aio_job_pool, a_job); 632 } 633 634 if (errcnt) 635 return EFAULT; 636 637 /* Set a correct return value */ 638 if (*retval == 0) 639 *retval = AIO_ALLDONE; 640 641 return 0; 642 } 643 644 int 645 sys_aio_error(struct lwp *l, void *v, register_t *retval) 646 { 647 struct sys_aio_error_args /* { 648 syscallarg(const struct aiocb *) aiocbp; 649 } */ *uap = v; 650 struct proc *p = l->l_proc; 651 struct aioproc *aio = p->p_aio; 652 struct aiocb aiocbp; 653 int error; 654 655 if (aio == NULL) 656 return EINVAL; 657 658 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 659 if (error) 660 return error; 661 662 if (aiocbp._state == JOB_NONE) 663 return EINVAL; 664 665 *retval = aiocbp._errno; 666 667 return 0; 668 } 669 670 int 671 sys_aio_fsync(struct lwp *l, void *v, register_t *retval) 672 { 673 struct sys_aio_fsync_args /* { 674 syscallarg(int) op; 675 syscallarg(struct aiocb *) aiocbp; 676 } */ *uap = v; 677 int op = SCARG(uap, op); 678 679 if ((op != O_DSYNC) && (op != O_SYNC)) 680 return EINVAL; 681 682 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 683 684 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 685 } 686 687 int 688 sys_aio_read(struct lwp *l, void *v, register_t *retval) 689 { 690 struct sys_aio_read_args /* { 691 syscallarg(struct aiocb *) aiocbp; 692 } */ *uap = v; 693 694 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 695 } 696 697 int 698 sys_aio_return(struct lwp *l, void *v, register_t *retval) 699 { 700 struct sys_aio_return_args /* { 701 syscallarg(struct aiocb *) aiocbp; 702 } */ *uap = v; 703 struct proc *p = l->l_proc; 704 struct aioproc *aio = p->p_aio; 705 struct aiocb aiocbp; 706 int error; 707 708 if (aio == NULL) 709 return EINVAL; 710 711 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 712 if (error) 713 return error; 714 715 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 716 return EINVAL; 717 718 *retval = aiocbp._retval; 719 720 /* Reset the internal variables */ 721 aiocbp._errno = 0; 722 aiocbp._retval = -1; 723 aiocbp._state = JOB_NONE; 724 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 725 726 return error; 727 } 728 729 int 730 sys_aio_suspend(struct lwp *l, void *v, register_t *retval) 731 { 732 struct sys_aio_suspend_args /* { 733 syscallarg(const struct aiocb *const[]) list; 734 syscallarg(int) nent; 735 syscallarg(const struct timespec *) timeout; 736 } */ *uap = v; 737 struct proc *p = l->l_proc; 738 struct aioproc *aio; 739 struct aio_job *a_job; 740 struct aiocb **aiocbp_list; 741 struct timespec ts; 742 int i, error, nent, timo; 743 744 if (p->p_aio == NULL) 745 return EAGAIN; 746 aio = p->p_aio; 747 748 nent = SCARG(uap, nent); 749 if (nent <= 0 || nent > aio_listio_max) 750 return EAGAIN; 751 752 if (SCARG(uap, timeout)) { 753 /* Convert timespec to ticks */ 754 error = copyin(SCARG(uap, timeout), &ts, 755 sizeof(struct timespec)); 756 if (error) 757 return error; 758 timo = mstohz((ts.tv_sec * 1000) + (ts.tv_nsec / 1000000)); 759 if (timo == 0 && ts.tv_sec == 0 && ts.tv_nsec > 0) 760 timo = 1; 761 if (timo <= 0) 762 return EAGAIN; 763 } else 764 timo = 0; 765 766 /* Get the list from user-space */ 767 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 768 error = copyin(SCARG(uap, list), aiocbp_list, 769 nent * sizeof(struct aiocb)); 770 if (error) { 771 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 772 return error; 773 } 774 775 mutex_enter(&aio->aio_mtx); 776 for (;;) { 777 778 for (i = 0; i < nent; i++) { 779 780 /* Skip NULL entries */ 781 if (aiocbp_list[i] == NULL) 782 continue; 783 784 /* Skip current job */ 785 if (aio->curjob) { 786 a_job = aio->curjob; 787 if (a_job->aiocb_uptr == aiocbp_list[i]) 788 continue; 789 } 790 791 /* Look for a job in the queue */ 792 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 793 if (a_job->aiocb_uptr == aiocbp_list[i]) 794 break; 795 796 if (a_job == NULL) { 797 struct aiocb aiocbp; 798 799 mutex_exit(&aio->aio_mtx); 800 801 error = copyin(aiocbp_list[i], &aiocbp, 802 sizeof(struct aiocb)); 803 if (error == 0 && aiocbp._state != JOB_DONE) { 804 mutex_enter(&aio->aio_mtx); 805 continue; 806 } 807 808 kmem_free(aiocbp_list, 809 nent * sizeof(struct aio_job)); 810 return error; 811 } 812 } 813 814 /* Wait for a signal or when timeout occurs */ 815 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 816 if (error) { 817 if (error == EWOULDBLOCK) 818 error = EAGAIN; 819 break; 820 } 821 } 822 mutex_exit(&aio->aio_mtx); 823 824 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 825 return error; 826 } 827 828 int 829 sys_aio_write(struct lwp *l, void *v, register_t *retval) 830 { 831 struct sys_aio_write_args /* { 832 syscallarg(struct aiocb *) aiocbp; 833 } */ *uap = v; 834 835 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 836 } 837 838 int 839 sys_lio_listio(struct lwp *l, void *v, register_t *retval) 840 { 841 struct sys_lio_listio_args /* { 842 syscallarg(int) mode; 843 syscallarg(struct aiocb *const[]) list; 844 syscallarg(int) nent; 845 syscallarg(struct sigevent *) sig; 846 } */ *uap = v; 847 struct proc *p = l->l_proc; 848 struct aioproc *aio; 849 struct aiocb **aiocbp_list; 850 struct lio_req *lio; 851 int i, error, errcnt, mode, nent; 852 853 mode = SCARG(uap, mode); 854 nent = SCARG(uap, nent); 855 856 /* Non-accurate checks for the limit and invalid values */ 857 if (nent < 1 || nent > aio_listio_max) 858 return EINVAL; 859 if (aio_jobs_count + nent > aio_max) 860 return EAGAIN; 861 862 /* Check if AIO structure is initialized, if not - initialize it */ 863 if (p->p_aio == NULL) 864 if (aio_init(p)) 865 return EAGAIN; 866 aio = p->p_aio; 867 868 /* Create a LIO structure */ 869 lio = pool_get(&aio_lio_pool, PR_WAITOK); 870 lio->refcnt = 1; 871 error = 0; 872 873 switch (mode) { 874 case LIO_WAIT: 875 memset(&lio->sig, 0, sizeof(struct sigevent)); 876 break; 877 case LIO_NOWAIT: 878 /* Check for signal, validate it */ 879 if (SCARG(uap, sig)) { 880 struct sigevent *sig = &lio->sig; 881 882 error = copyin(SCARG(uap, sig), &lio->sig, 883 sizeof(struct sigevent)); 884 if (error == 0 && 885 (sig->sigev_signo < 0 || 886 sig->sigev_signo >= NSIG || 887 sig->sigev_notify < SIGEV_NONE || 888 sig->sigev_notify > SIGEV_SA)) 889 error = EINVAL; 890 } else 891 memset(&lio->sig, 0, sizeof(struct sigevent)); 892 break; 893 default: 894 error = EINVAL; 895 break; 896 } 897 898 if (error != 0) { 899 pool_put(&aio_lio_pool, lio); 900 return error; 901 } 902 903 /* Get the list from user-space */ 904 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 905 error = copyin(SCARG(uap, list), aiocbp_list, 906 nent * sizeof(struct aiocb)); 907 if (error) { 908 mutex_enter(&aio->aio_mtx); 909 goto err; 910 } 911 912 /* Enqueue all jobs */ 913 errcnt = 0; 914 for (i = 0; i < nent; i++) { 915 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 916 /* 917 * According to POSIX, in such error case it may 918 * fail with other I/O operations initiated. 919 */ 920 if (error) 921 errcnt++; 922 } 923 924 mutex_enter(&aio->aio_mtx); 925 926 /* Return an error, if any */ 927 if (errcnt) { 928 error = EIO; 929 goto err; 930 } 931 932 if (mode == LIO_WAIT) { 933 /* 934 * Wait for AIO completion. In such case, 935 * the LIO structure will be freed here. 936 */ 937 while (lio->refcnt > 1 && error == 0) 938 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 939 if (error) 940 error = EINTR; 941 } 942 943 err: 944 if (--lio->refcnt != 0) 945 lio = NULL; 946 mutex_exit(&aio->aio_mtx); 947 if (lio != NULL) { 948 aio_sendsig(p, &lio->sig); 949 pool_put(&aio_lio_pool, lio); 950 } 951 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 952 return error; 953 } 954 955 /* 956 * SysCtl 957 */ 958 959 static int 960 sysctl_aio_listio_max(SYSCTLFN_ARGS) 961 { 962 struct sysctlnode node; 963 int error, newsize; 964 965 node = *rnode; 966 node.sysctl_data = &newsize; 967 968 newsize = aio_listio_max; 969 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 970 if (error || newp == NULL) 971 return error; 972 973 if (newsize < 1 || newsize > aio_max) 974 return EINVAL; 975 aio_listio_max = newsize; 976 977 return 0; 978 } 979 980 static int 981 sysctl_aio_max(SYSCTLFN_ARGS) 982 { 983 struct sysctlnode node; 984 int error, newsize; 985 986 node = *rnode; 987 node.sysctl_data = &newsize; 988 989 newsize = aio_max; 990 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 991 if (error || newp == NULL) 992 return error; 993 994 if (newsize < 1 || newsize < aio_listio_max) 995 return EINVAL; 996 aio_max = newsize; 997 998 return 0; 999 } 1000 1001 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup") 1002 { 1003 1004 sysctl_createv(clog, 0, NULL, NULL, 1005 CTLFLAG_PERMANENT, 1006 CTLTYPE_NODE, "kern", NULL, 1007 NULL, 0, NULL, 0, 1008 CTL_KERN, CTL_EOL); 1009 sysctl_createv(clog, 0, NULL, NULL, 1010 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1011 CTLTYPE_INT, "posix_aio", 1012 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1013 "Asynchronous I/O option to which the " 1014 "system attempts to conform"), 1015 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1016 CTL_KERN, CTL_CREATE, CTL_EOL); 1017 sysctl_createv(clog, 0, NULL, NULL, 1018 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1019 CTLTYPE_INT, "aio_listio_max", 1020 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1021 "operations in a single list I/O call"), 1022 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1023 CTL_KERN, CTL_CREATE, CTL_EOL); 1024 sysctl_createv(clog, 0, NULL, NULL, 1025 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1026 CTLTYPE_INT, "aio_max", 1027 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1028 "operations"), 1029 sysctl_aio_max, 0, &aio_max, 0, 1030 CTL_KERN, CTL_CREATE, CTL_EOL); 1031 } 1032 1033 /* 1034 * Debugging 1035 */ 1036 #if defined(DDB) 1037 void 1038 aio_print_jobs(void (*pr)(const char *, ...)) 1039 { 1040 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1041 struct aioproc *aio; 1042 struct aio_job *a_job; 1043 struct aiocb *aiocbp; 1044 1045 if (p == NULL) { 1046 (*pr)("AIO: We are not in the processes right now.\n"); 1047 return; 1048 } 1049 1050 aio = p->p_aio; 1051 if (aio == NULL) { 1052 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1053 return; 1054 } 1055 1056 (*pr)("AIO: PID = %d\n", p->p_pid); 1057 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1058 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1059 1060 if (aio->curjob) { 1061 a_job = aio->curjob; 1062 (*pr)("\nAIO current job:\n"); 1063 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1064 a_job->aio_op, a_job->aiocbp._errno, 1065 a_job->aiocbp._state, a_job->aiocb_uptr); 1066 aiocbp = &a_job->aiocbp; 1067 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1068 aiocbp->aio_fildes, aiocbp->aio_offset, 1069 aiocbp->aio_buf, aiocbp->aio_nbytes); 1070 } 1071 1072 (*pr)("\nAIO queue:\n"); 1073 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1074 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1075 a_job->aio_op, a_job->aiocbp._errno, 1076 a_job->aiocbp._state, a_job->aiocb_uptr); 1077 aiocbp = &a_job->aiocbp; 1078 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1079 aiocbp->aio_fildes, aiocbp->aio_offset, 1080 aiocbp->aio_buf, aiocbp->aio_nbytes); 1081 } 1082 } 1083 #endif /* defined(DDB) */ 1084