Lines Matching +full:trim +full:- +full:data +full:- +full:valid
1 /*-
2 * Copyright (c) 2013-2015 Gleb Smirnoff <glebius@FreeBSD.org>
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
87 * of the data.
116 KASSERT(sfs->count == 0, ("sendfile sync %p still busy", sfs));
118 cv_destroy(&sfs->cv);
119 mtx_destroy(&sfs->mtx);
126 mtx_lock(&sfs->mtx);
127 KASSERT(sfs->count > 0, ("sendfile sync %p not busy", sfs));
128 if (--sfs->count == 0) {
129 if (!sfs->waiting) {
134 cv_signal(&sfs->cv);
137 mtx_unlock(&sfs->mtx);
157 if (req->newptr)
173 KASSERT(m->m_flags & M_EXT && m->m_ext.ext_type == EXT_SFBUF,
176 sf = m->m_ext.ext_arg1;
178 flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;
183 if (m->m_ext.ext_flags & EXT_FLAG_SYNC) {
184 struct sendfile_sync *sfs = m->m_ext.ext_arg2;
198 cache_last = m->m_ext.ext_flags & EXT_FLAG_CACHE_LAST;
199 flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;
201 for (i = 0; i < m->m_epg_npgs; i++) {
202 if (cache_last && i == m->m_epg_npgs - 1)
204 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
208 if (m->m_ext.ext_flags & EXT_FLAG_SYNC) {
209 struct sendfile_sync *sfs = m->m_ext.ext_arg1;
215 * Helper function to calculate how much data to put into page i of n.
223 return (omin(PAGE_SIZE - (off & PAGE_MASK), len));
225 if (i == n - 1 && ((off + len) & PAGE_MASK) > 0)
256 *space -= xfsize(old - 1, old, off, *space);
257 old--;
265 *space -= xfsize(0, old, off, *space);
270 *space -= (old - new) * PAGE_SIZE;
276 * Wait for all in-flight ios to complete, we must not unwire pages
282 while (atomic_load_int(&sfio->nios) != 1)
297 sfio->error = error;
300 * Restore the valid page pointers. They are already
306 * unbusied the swapped-in pages, they can become
312 sfio->pa[(pa[0]->pindex - sfio->pindex0) + i] =
313 pa[i] = vm_page_relookup(sfio->obj,
314 pa[0]->pindex + i);
323 if (!refcount_release(&sfio->nios))
327 for (i = 1; i < sfio->npages; i++) {
328 if (sfio->pa[i] == NULL)
330 KASSERT(vm_page_wired(sfio->pa[i]),
331 ("sfio %p page %d %p not wired", sfio, i, sfio->pa[i]));
334 KASSERT(sfio->pa[0]->object == sfio->pa[i]->object,
336 sfio->pa[i], sfio->pa[0]->object, sfio->pa[i]->object));
337 KASSERT(sfio->pa[0]->pindex + i == sfio->pa[i]->pindex,
339 sfio->pa[i], (uintmax_t)sfio->pa[0]->pindex,
340 (uintmax_t)sfio->pa[i]->pindex));
344 vm_object_pip_wakeup(sfio->obj);
346 if (sfio->m == NULL) {
351 * pru_send hadn't been executed - nothing had been sent
354 MPASS((curthread->td_pflags & TDP_KTHREAD) == 0);
360 if ((sfio->m->m_flags & M_EXTPG) != 0)
361 KASSERT(sfio->tls == sfio->m->m_epg_tls,
364 KASSERT(sfio->tls == NULL,
365 ("non-ext_pgs mbuf with TLS session"));
367 so = sfio->so;
368 CURVNET_SET(so->so_vnet);
369 if (__predict_false(sfio->error)) {
371 * I/O operation failed. The state of data in the socket
381 so->so_proto->pr_abort(so);
382 so->so_error = EIO;
384 mb_free_notready(sfio->m, sfio->npages);
386 } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
396 ktls_enqueue(sfio->m, so, sfio->npages);
400 (void)so->so_proto->pr_ready(so, sfio->m, sfio->npages);
411 * Iterate through pages vector and request paging for non-valid pages.
420 pa = sfio->pa;
421 npages = sfio->npages;
424 sfio->pindex0 = OFF_TO_IDX(off);
440 /* Skip valid pages. */
465 MPASS(pa[i]->dirty == 0);
476 count = min(a + 1, npages - i);
479 * We should not pagein into a valid page because
481 * e.g. a buffer, thus we substitute any valid pages
490 * First trim the end of the run consisting of the
491 * valid pages, then replace the rest of the valid
495 for (j = i + count - 1; j > i; j--) {
500 count--;
507 * The last page in the run pa[i + count - 1] is
508 * guaranteed to be invalid by the trim above, so it
509 * is not replaced with bogus, thus -1 in the loop end
512 MPASS(pa[i + count - 1]->valid != VM_PAGE_BITS_ALL);
513 for (j = i + 1; j < i + count - 1; j++) {
523 refcount_acquire(&sfio->nios);
582 if (fp->f_type == DTYPE_VNODE) {
583 vp = fp->f_vnode;
585 if (vp->v_type != VREG) {
589 *bsize = vp->v_mount->mnt_stat.f_iosize;
590 obj = vp->v_object;
601 if (obj->type == OBJT_VNODE) {
603 *obj_size = obj->un_pager.vnp.vnp_size;
605 error = vn_getsize_locked(vp, obj_size, td->td_ucred);
610 } else if (fp->f_type == DTYPE_SHM) {
611 shmfd = fp->f_data;
612 obj = shmfd->shm_object;
614 *obj_size = shmfd->shm_size;
620 if ((obj->flags & OBJ_DEAD) != 0) {
658 *so = (*sock_fp)->f_data;
659 if ((*so)->so_type != SOCK_STREAM)
662 * SCTP one-to-one style sockets currently don't work with
665 if ((*so)->so_proto->pr_protocol == IPPROTO_SCTP)
715 error = mac_socket_check_send(td->td_ucred, so);
725 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF);
726 cv_init(&sfs->cv, "sendfile");
727 sfs->waiting = true;
730 rem = nbytes ? omin(nbytes, obj_size - offset) : obj_size - offset;
735 * XXXRW: Historically this has assumed non-interruptibility, so now
742 tls = ktls_hold(so->so_snd.sb_tls_info);
779 SOCKBUF_LOCK(&so->so_snd);
780 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2)
781 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2;
783 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
785 SOCKBUF_UNLOCK(&so->so_snd);
787 } else if (so->so_error) {
788 error = so->so_error;
789 so->so_error = 0;
790 SOCKBUF_UNLOCK(&so->so_snd);
793 if ((so->so_state & SS_ISCONNECTED) == 0) {
794 SOCKBUF_UNLOCK(&so->so_snd);
799 space = sbspace(&so->so_snd);
802 space < so->so_snd.sb_lowat)) {
803 if (so->so_state & SS_NBIO) {
804 SOCKBUF_UNLOCK(&so->so_snd);
821 SOCKBUF_UNLOCK(&so->so_snd);
826 SOCKBUF_UNLOCK(&so->so_snd);
834 if (hdr_uio != NULL && hdr_uio->uio_resid > 0) {
835 hdr_uio->uio_td = td;
836 hdr_uio->uio_rw = UIO_WRITE;
840 tls->params.max_frame_len, M_EXTPG);
846 space -= hdrlen;
867 if (obj->type == OBJT_VNODE) {
869 nobj_size = obj->un_pager.vnp.vnp_size;
872 error = VOP_GETATTR(vp, &va, td->td_ucred);
887 rem -= off;
899 space -= (PAGE_SIZE - (off & PAGE_MASK));
902 space += (PAGE_SIZE - (off & PAGE_MASK));
919 rhpages = howmany(rem + (off & PAGE_MASK), PAGE_SIZE) -
924 rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -
929 refcount_init(&sfio->nios, 1);
930 sfio->obj = obj;
931 sfio->error = 0;
932 sfio->m = NULL;
933 sfio->npages = npages;
936 * This doesn't use ktls_hold() because sfio->m will
937 * also have a reference on 'tls' that will be valid
940 sfio->tls = tls;
956 pa = sfio->pa;
962 * been tested with UNIX-domain sockets.
967 so->so_proto->pr_protocol == IPPROTO_TCP)
975 max_pgs = num_pages(tls->params.max_frame_len);
981 ext_pgs_idx = max_pgs - 1;
987 * trim the array. Can happen only with SF_NODISKIO.
992 sfio->npages = i;
998 pga = vm_page_relookup(obj, sfio->pindex0 + i);
1009 m0->m_ext.ext_flags |=
1017 if ((npages - i <= max_pgs) &&
1020 m0->m_ext.ext_flags |=
1024 m0->m_ext.ext_flags |=
1026 m0->m_ext.ext_arg1 = sfs;
1027 mtx_lock(&sfs->mtx);
1028 sfs->count++;
1029 mtx_unlock(&sfs->mtx);
1035 mtail->m_next = m0;
1039 m0->m_epg_1st_off =
1043 mtail->m_flags |= M_NOTREADY;
1044 m0->m_epg_nrdy++;
1047 m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
1048 m0->m_epg_npgs++;
1050 m0->m_epg_last_len = xfs;
1052 mtail->m_len += xfs;
1053 mtail->m_ext.ext_size += PAGE_SIZE;
1078 sfio->npages = i;
1083 m0->m_ext.ext_buf = (char *)sf_buf_kva(sf);
1084 m0->m_ext.ext_size = PAGE_SIZE;
1085 m0->m_ext.ext_arg1 = sf;
1086 m0->m_ext.ext_type = EXT_SFBUF;
1087 m0->m_ext.ext_flags = EXT_FLAG_EMBREF;
1088 m0->m_ext.ext_free = sendfile_free_mext;
1092 * if the page is truncated, and we got more data to
1097 (i != npages - 1 ||
1100 m0->m_ext.ext_flags |= EXT_FLAG_NOCACHE;
1102 m0->m_ext.ext_flags |= EXT_FLAG_SYNC;
1103 m0->m_ext.ext_arg2 = sfs;
1104 mtx_lock(&sfs->mtx);
1105 sfs->count++;
1106 mtx_unlock(&sfs->mtx);
1108 m0->m_ext.ext_count = 1;
1109 m0->m_flags |= (M_EXT | M_RDONLY);
1111 m0->m_flags |= M_NOTREADY;
1112 m0->m_data = (char *)sf_buf_kva(sf) +
1114 m0->m_len = xfsize(i, npages, off, space);
1118 mtail->m_next = m0;
1129 rem -= space;
1137 m0 = mhtail->m_next = m;
1155 CURVNET_SET(so->so_vnet);
1163 * which happens if all data is cached in VM, or if
1165 * sfio is NULL, then we can send data right now
1171 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
1172 error = so->so_proto->pr_send(so,
1182 error = so->so_proto->pr_send(so, 0, m, NULL,
1185 sfio->so = so;
1186 sfio->m = m0;
1188 error = so->so_proto->pr_send(so, PRUS_NOTREADY, m,
1193 if (so->so_proto->pr_protocol == IPPROTO_TCP) {
1219 sbytes += td->td_retval[0];
1227 * If there was no error we have to clear td->td_retval[0]
1231 td->td_retval[0] = 0;
1246 mtx_lock(&sfs->mtx);
1247 if (sfs->count != 0)
1248 error = cv_wait_sig(&sfs->cv, &sfs->mtx);
1249 if (sfs->count == 0) {
1252 sfs->waiting = false;
1253 mtx_unlock(&sfs->mtx);
1278 * we send only the header/trailer and no payload data.
1280 if (uap->offset < 0)
1286 if (uap->hdtr != NULL) {
1287 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr));
1302 if (uap->nbytes > hdr_uio->uio_resid)
1303 uap->nbytes -= hdr_uio->uio_resid;
1305 uap->nbytes = 0;
1317 AUDIT_ARG_FD(uap->fd);
1323 if ((error = fget_read(td, uap->fd, &cap_pread_rights, &fp)) != 0)
1326 error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset,
1327 uap->nbytes, &sbytes, uap->flags, td);
1330 if (uap->sbytes != NULL)
1331 (void)copyout(&sbytes, uap->sbytes, sizeof(off_t));
1363 args.fd = uap->fd;
1364 args.s = uap->s;
1365 args.offset = uap->offset;
1366 args.nbytes = uap->nbytes;
1367 args.hdtr = uap->hdtr;
1368 args.sbytes = uap->sbytes;
1369 args.flags = uap->flags;