xref: /illumos-gate/usr/src/uts/common/fs/fifofs/fifosubr.c (revision ea7b7d8ad0ddfa6a32fb675e11cc106f686a5bb5)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/

/*
 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2024 Oxide Computer Company
 */

/*
 * The routines defined in this file are supporting routines for FIFOFS
 * file system type.
 */
#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/debug.h>
#include <sys/errno.h>
#include <sys/time.h>
#include <sys/kmem.h>
#include <sys/inline.h>
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <sys/var.h>
#include <sys/vfs.h>
#include <sys/vfs_opreg.h>
#include <sys/vnode.h>
#include <sys/mode.h>
#include <sys/signal.h>
#include <sys/user.h>
#include <sys/uio.h>
#include <sys/flock.h>
#include <sys/stream.h>
#include <sys/fs/fifonode.h>
#include <sys/strsubr.h>
#include <sys/stropts.h>
#include <sys/cmn_err.h>
#include <fs/fs_subr.h>
#include <sys/ddi.h>


#if FIFODEBUG
int Fifo_fastmode = 1;		/* pipes/fifos will be opened in fast mode */
int Fifo_verbose = 0;		/* msg when switching out of fast mode */
int Fifohiwat = FIFOHIWAT;	/* Modifiable FIFO high water mark */
#endif

/*
 * This is the loadable module wrapper.
 */
#include <sys/modctl.h>

extern struct qinit fifo_strdata;

struct vfsops *fifo_vfsops;

static vfsdef_t vfw = {
	VFSDEF_VERSION,
	"fifofs",
	fifoinit,
	VSW_ZMOUNT,
	NULL
};

/*
 * Module linkage information for the kernel.
 */
extern struct mod_ops mod_fsops;

static struct modlfs modlfs = {
	&mod_fsops, "filesystem for fifo", &vfw
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modlfs, NULL
};

int
_init()
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * Define data structures within this file.
 * XXX should the hash size be configurable ?
 */
#define	FIFOSHFT	5
#define	FIFO_HASHSZ	63

#if ((FIFO_HASHSZ & (FIFO_HASHSZ - 1)) == 0)
#define	FIFOHASH(vp) (((uintptr_t)(vp) >> FIFOSHFT) & (FIFO_HASHSZ - 1))
#else
#define	FIFOHASH(vp) (((uintptr_t)(vp) >> FIFOSHFT) % FIFO_HASHSZ)
#endif

fifonode_t	*fifoalloc[FIFO_HASHSZ];
dev_t		fifodev;
struct vfs	*fifovfsp;
int		fifofstype;

kmutex_t ftable_lock;
static kmutex_t fino_lock;
struct kmem_cache *fnode_cache;
struct kmem_cache *pipe_cache;

static void fifoinsert(fifonode_t *);
static fifonode_t *fifofind(vnode_t *);
static int fifo_connld(struct vnode **, int, cred_t *);
static void fifo_fastturnoff(fifonode_t *);

static void fifo_reinit_vp(vnode_t *);

static void fnode_destructor(void *, void *);

/*
 * Constructor/destructor routines for fifos and pipes.
 *
 * In the interest of code sharing, we define a common fifodata structure
 * which consists of a fifolock and one or two fnodes.  A fifo contains
 * one fnode; a pipe contains two.  The fifolock is shared by the fnodes,
 * each of which points to it:
 *
 *	--> -->	---------  --- ---
 *	|   |	| lock	|   |	|
 *	|   |	---------   |	|
 *	|   |	|	|  fifo	|
 *	|   --- | fnode	|   |	|
 *	|	|	|   |  pipe
 *	|	---------  ---	|
 *	|	|	|	|
 *	------- | fnode	|	|
 *		|	|	|
 *		---------      ---
 *
 * Since the fifolock is at the beginning of the fifodata structure,
 * the fifolock address is the same as the fifodata address.  Thus,
 * we can determine the fifodata address from any of its member fnodes.
 * This is essential for fifo_inactive.
 *
 * The fnode constructor is designed to handle any fifodata structure,
 * deducing the number of fnodes from the total size.  Thus, the fnode
 * constructor does most of the work for the pipe constructor.
 */
static int
fnode_constructor(void *buf, void *cdrarg, int kmflags)
{
	fifodata_t *fdp = buf;
	fifolock_t *flp = &fdp->fifo_lock;
	fifonode_t *fnp = &fdp->fifo_fnode[0];
	size_t size = (uintptr_t)cdrarg;

	mutex_init(&flp->flk_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&flp->flk_wait_cv, NULL, CV_DEFAULT, NULL);
	flp->flk_ocsync = 0;

	while ((char *)fnp < (char *)buf + size) {

		vnode_t *vp;

		vp = vn_alloc(kmflags);
		if (vp == NULL) {
			fnp->fn_vnode = NULL; /* mark for destructor */
			fnode_destructor(buf, cdrarg);
			return (-1);
		}
		fnp->fn_vnode = vp;

		fnp->fn_lock = flp;
		fnp->fn_open = 0;
		fnp->fn_dest = fnp;
		fnp->fn_mp = NULL;
		fnp->fn_count = 0;
		fnp->fn_rsynccnt = 0;
		fnp->fn_wsynccnt = 0;
		fnp->fn_wwaitcnt = 0;
		fnp->fn_insync = 0;
		fnp->fn_pcredp = NULL;
		fnp->fn_cpid = -1;
		/*
		 * 32-bit stat(2) may fail if fn_ino isn't initialized
		 */
		fnp->fn_ino = 0;

		cv_init(&fnp->fn_wait_cv, NULL, CV_DEFAULT, NULL);

		vn_setops(vp, fifo_vnodeops);
		vp->v_stream = NULL;
		vp->v_type = VFIFO;
		vp->v_data = (caddr_t)fnp;
		vp->v_flag = VNOMAP | VNOSWAP;
		vn_exists(vp);
		fnp++;
	}
	return (0);
}

static void
fnode_destructor(void *buf, void *cdrarg)
{
	fifodata_t *fdp = buf;
	fifolock_t *flp = &fdp->fifo_lock;
	fifonode_t *fnp = &fdp->fifo_fnode[0];
	size_t size = (uintptr_t)cdrarg;

	mutex_destroy(&flp->flk_lock);
	cv_destroy(&flp->flk_wait_cv);
	ASSERT(flp->flk_ocsync == 0);

	while ((char *)fnp < (char *)buf + size) {

		vnode_t *vp = FTOV(fnp);

		if (vp == NULL) {
			return; /* constructor failed here */
		}

		ASSERT(fnp->fn_mp == NULL);
		ASSERT(fnp->fn_count == 0);
		ASSERT(fnp->fn_lock == flp);
		ASSERT(fnp->fn_open == 0);
		ASSERT(fnp->fn_insync == 0);
		ASSERT(fnp->fn_rsynccnt == 0 && fnp->fn_wsynccnt == 0);
		ASSERT(fnp->fn_wwaitcnt == 0);
		ASSERT(fnp->fn_pcredp == NULL);
		ASSERT(vn_matchops(vp, fifo_vnodeops));
		ASSERT(vp->v_stream == NULL);
		ASSERT(vp->v_type == VFIFO);
		ASSERT(vp->v_data == (caddr_t)fnp);
		ASSERT((vp->v_flag & (VNOMAP|VNOSWAP)) == (VNOMAP|VNOSWAP));

		cv_destroy(&fnp->fn_wait_cv);
		vn_invalid(vp);
		vn_free(vp);

		fnp++;
	}
}

static int
pipe_constructor(void *buf, void *cdrarg, int kmflags)
{
	fifodata_t *fdp = buf;
	fifonode_t *fnp1 = &fdp->fifo_fnode[0];
	fifonode_t *fnp2 = &fdp->fifo_fnode[1];
	vnode_t *vp1;
	vnode_t *vp2;

	(void) fnode_constructor(buf, cdrarg, kmflags);

	vp1 = FTOV(fnp1);
	vp2 = FTOV(fnp2);

	vp1->v_vfsp	= vp2->v_vfsp		= fifovfsp;
	vp1->v_rdev	= vp2->v_rdev		= fifodev;
	fnp1->fn_realvp	= fnp2->fn_realvp	= NULL;
	fnp1->fn_dest	= fnp2;
	fnp2->fn_dest	= fnp1;

	return (0);
}

static void
pipe_destructor(void *buf, void *cdrarg)
{
#ifdef DEBUG
	fifodata_t *fdp = buf;
	fifonode_t *fnp1 = &fdp->fifo_fnode[0];
	fifonode_t *fnp2 = &fdp->fifo_fnode[1];
	vnode_t *vp1 = FTOV(fnp1);
	vnode_t *vp2 = FTOV(fnp2);

	ASSERT(vp1->v_vfsp == fifovfsp);
	ASSERT(vp2->v_vfsp == fifovfsp);
	ASSERT(vp1->v_rdev == fifodev);
	ASSERT(vp2->v_rdev == fifodev);
#endif
	fnode_destructor(buf, cdrarg);
}

/*
 * Reinitialize a FIFO vnode (uses normal vnode reinit, but ensures that
 * vnode type and flags are reset).
 */

static void fifo_reinit_vp(vnode_t *vp)
{
	vn_reinit(vp);
	vp->v_type = VFIFO;
	vp->v_flag &= VROOT;
	vp->v_flag |= VNOMAP | VNOSWAP;
}

/*
 * Save file system type/index, initialize vfs operations vector, get
 * unique device number for FIFOFS and initialize the FIFOFS hash.
 * Create and initialize a "generic" vfs pointer that will be placed
 * in the v_vfsp field of each pipe's vnode.
 */
int
fifoinit(int fstype, char *name)
{
	static const fs_operation_def_t fifo_vfsops_template[] = {
		NULL, NULL
	};
	int error;
	major_t dev;

	fifofstype = fstype;
	error = vfs_setfsops(fstype, fifo_vfsops_template, &fifo_vfsops);
	if (error != 0) {
		cmn_err(CE_WARN, "fifoinit: bad vfs ops template");
		return (error);
	}

	error = vn_make_ops(name, fifo_vnodeops_template, &fifo_vnodeops);
	if (error != 0) {
		(void) vfs_freevfsops_by_type(fstype);
		cmn_err(CE_WARN, "fifoinit: bad vnode ops template");
		return (error);
	}

	if ((dev = getudev()) == (major_t)-1) {
		cmn_err(CE_WARN, "fifoinit: can't get unique device number");
		dev = 0;
	}
	fifodev = makedevice(dev, 0);

	fifovfsp = fs_vfsp_global(fifo_vfsops, fifodev, fifofstype, 1024);

	mutex_init(&ftable_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&fino_lock, NULL, MUTEX_DEFAULT, NULL);

	/*
	 * vnodes are cached aligned
	 */
	fnode_cache = kmem_cache_create("fnode_cache",
	    sizeof (fifodata_t) - sizeof (fifonode_t), 32,
	    fnode_constructor, fnode_destructor, NULL,
	    (void *)(sizeof (fifodata_t) - sizeof (fifonode_t)), NULL, 0);

	pipe_cache = kmem_cache_create("pipe_cache", sizeof (fifodata_t), 32,
	    pipe_constructor, pipe_destructor, NULL,
	    (void *)(sizeof (fifodata_t)), NULL, 0);

#if FIFODEBUG
	if (Fifohiwat < FIFOHIWAT)
		Fifohiwat = FIFOHIWAT;
#endif /* FIFODEBUG */
	fifo_strdata.qi_minfo->mi_hiwat = Fifohiwat;

	return (0);
}

/*
 * Provide a shadow for a vnode.  We create a new shadow before checking for an
 * existing one, to minimize the amount of time we need to hold ftable_lock.
 * If a vp already has a shadow in the hash list, return its shadow.  If not,
 * we hash the new vnode and return its pointer to the caller.
 */
vnode_t *
fifovp(vnode_t *vp, cred_t *crp)
{
	fifonode_t *fnp;
	fifonode_t *spec_fnp;   /* Speculative fnode ptr. */
	fifodata_t *fdp;
	vnode_t *newvp;
	struct vattr va;
	vnode_t	*rvp;

	ASSERT(vp != NULL);

	fdp = kmem_cache_alloc(fnode_cache, KM_SLEEP);

	fdp->fifo_lock.flk_ref = 1;
	fnp = &fdp->fifo_fnode[0];

	/*
	 * Its possible that fifo nodes on different lofs mountpoints
	 * shadow the same real filesystem fifo node.
	 * In this case its necessary to get and store the realvp.
	 * This way different fifo nodes sharing the same real vnode
	 * can use realvp for communication.
	 */

	if (VOP_REALVP(vp, &rvp, NULL) == 0)
			vp = rvp;

	fnp->fn_realvp	= vp;
	fnp->fn_wcnt	= 0;
	fnp->fn_rcnt	= 0;

#if FIFODEBUG
	if (! Fifo_fastmode) {
		fnp->fn_flag	= 0;
	} else {
		fnp->fn_flag	= FIFOFAST;
	}
#else /* FIFODEBUG */
	fnp->fn_flag	= FIFOFAST;
#endif /* FIFODEBUG */

	/*
	 * initialize the times from vp.
	 */
	va.va_mask = AT_TIMES;
	if (VOP_GETATTR(vp, &va, 0, crp, NULL) == 0) {
		fnp->fn_atime = va.va_atime;
		fnp->fn_mtime = va.va_mtime;
		fnp->fn_ctime = va.va_ctime;
	} else {
		fnp->fn_atime.tv_sec = 0;
		fnp->fn_atime.tv_nsec = 0;
		fnp->fn_mtime.tv_sec = 0;
		fnp->fn_mtime.tv_nsec = 0;
		fnp->fn_ctime.tv_sec = 0;
		fnp->fn_ctime.tv_nsec = 0;
	}

	/*
	 * Grab the VP here to avoid holding locks
	 * whilst trying to acquire others.
	 */

	VN_HOLD(vp);

	mutex_enter(&ftable_lock);

	if ((spec_fnp = fifofind(vp)) != NULL) {
		mutex_exit(&ftable_lock);

		/*
		 * Release the vnode and free up our pre-prepared fnode.
		 * Zero the lock reference just to explicitly signal
		 * this is unused.
		 */
		VN_RELE(vp);
		fdp->fifo_lock.flk_ref = 0;
		kmem_cache_free(fnode_cache, fdp);

		return (FTOV(spec_fnp));
	}

	newvp = FTOV(fnp);
	fifo_reinit_vp(newvp);
	/*
	 * Since the fifo vnode's v_vfsp needs to point to the
	 * underlying filesystem's vfsp we need to bump up the
	 * underlying filesystem's vfs reference count.
	 * The count is decremented when the fifo node is
	 * inactivated.
	 */

	VFS_HOLD(vp->v_vfsp);
	newvp->v_vfsp = vp->v_vfsp;
	newvp->v_rdev = vp->v_rdev;
	newvp->v_flag |= (vp->v_flag & VROOT);

	fifoinsert(fnp);
	mutex_exit(&ftable_lock);

	return (newvp);
}

/*
 * Create a pipe end by...
 * allocating a vnode-fifonode pair and initializing the fifonode.
 */
void
makepipe(vnode_t **vpp1, vnode_t **vpp2)
{
	fifonode_t *fnp1;
	fifonode_t *fnp2;
	vnode_t *nvp1;
	vnode_t *nvp2;
	fifodata_t *fdp;
	timestruc_t now;

	fdp = kmem_cache_alloc(pipe_cache, KM_SLEEP);
	fdp->fifo_lock.flk_ref = 2;
	fnp1 = &fdp->fifo_fnode[0];
	fnp2 = &fdp->fifo_fnode[1];

	fnp1->fn_wcnt	= fnp2->fn_wcnt		= 1;
	fnp1->fn_rcnt	= fnp2->fn_rcnt		= 1;
#if FIFODEBUG
	if (! Fifo_fastmode) {
		fnp1->fn_flag	= fnp2->fn_flag		= ISPIPE;
	} else {
		fnp1->fn_flag	= fnp2->fn_flag		= ISPIPE | FIFOFAST;
	}
#else /* FIFODEBUG */
	fnp1->fn_flag	= fnp2->fn_flag		= ISPIPE | FIFOFAST;
#endif /* FIFODEBUG */
	gethrestime(&now);
	fnp1->fn_atime	= fnp2->fn_atime	= now;
	fnp1->fn_mtime	= fnp2->fn_mtime	= now;
	fnp1->fn_ctime	= fnp2->fn_ctime	= now;

	*vpp1 = nvp1 = FTOV(fnp1);
	*vpp2 = nvp2 = FTOV(fnp2);

	fifo_reinit_vp(nvp1);		/* Reinitialize vnodes for reuse... */
	fifo_reinit_vp(nvp2);
	nvp1->v_vfsp = fifovfsp;	/* Need to re-establish VFS & device */
	nvp2->v_vfsp = fifovfsp;	/* before we can reuse this vnode. */
	nvp1->v_rdev = fifodev;
	nvp2->v_rdev = fifodev;
}

/*
 * Attempt to establish a unique pipe id.  Only un-named pipes use this
 * routine.
 */
ino_t
fifogetid(void)
{
	static ino_t fifo_ino = 0;
	ino_t fino;

	mutex_enter(&fino_lock);
	fino = fifo_ino++;
	mutex_exit(&fino_lock);
	return (fino);
}


/*
 * Stream a pipe/FIFO.
 * The FIFOCONNLD flag is used when CONNLD has been pushed on the stream.
 * If the flag is set, a new vnode is created by calling fifo_connld().
 * Connld logic was moved to fifo_connld() to speed up the open
 * operation, simplify the connld/fifo interaction, and remove inherent
 * race conditions between the connld module and fifos.
 * This routine is single threaded for two reasons.
 * 1) connld requests are synchronous; that is, they must block
 *    until the server does an I_RECVFD (oh, well).  Single threading is
 *    the simplest way to accomplish this.
 * 2) fifo_close() must not send M_HANGUP or M_ERROR while we are
 *    in stropen. Stropen() has a tendency to reset things and
 *    we would like streams to remember that a hangup occurred.
 */
int
fifo_stropen(vnode_t **vpp, int flag, cred_t *crp, int dotwist, int lockheld)
{
	int error = 0;
	vnode_t *oldvp = *vpp;
	fifonode_t *fnp = VTOF(*vpp);
	dev_t pdev = 0;
	int firstopen = 0;
	fifolock_t *fn_lock;

	fn_lock = fnp->fn_lock;
	if (!lockheld)
		mutex_enter(&fn_lock->flk_lock);
	ASSERT(MUTEX_HELD(&fnp->fn_lock->flk_lock));

	/*
	 * FIFO is in the process of opening. Wait for it
	 * to complete before starting another open on it
	 * This prevents races associated with connld open
	 */
	while (fnp->fn_flag & FIFOOPEN) {
		if (!cv_wait_sig(&fnp->fn_wait_cv, &fn_lock->flk_lock)) {
			fifo_cleanup(oldvp, flag);
			if (!lockheld)
				mutex_exit(&fn_lock->flk_lock);
			return (EINTR);
		}
	}

	/*
	 * The other end of the pipe is almost closed so
	 * reject any other open on this end of the pipe
	 * This only happens with a pipe mounted under namefs
	 */
	if ((fnp->fn_flag & (FIFOCLOSE|ISPIPE)) == (FIFOCLOSE|ISPIPE)) {
		fifo_cleanup(oldvp, flag);
		cv_broadcast(&fnp->fn_wait_cv);
		if (!lockheld)
			mutex_exit(&fn_lock->flk_lock);
		return (ENXIO);
	}

	fnp->fn_flag |= FIFOOPEN;

	/*
	 * can't allow close to happen while we are
	 * in the middle of stropen().
	 * M_HANGUP and M_ERROR could leave the stream in a strange state
	 */
	while (fn_lock->flk_ocsync)
		cv_wait(&fn_lock->flk_wait_cv, &fn_lock->flk_lock);

	fn_lock->flk_ocsync = 1;

	if (fnp->fn_flag & FIFOCONNLD) {
		/*
		 * This is a reopen, so we should release the fifo lock
		 * just in case some strange module pushed on connld
		 * has some odd side effect.
		 * Note: this stropen is on the oldvp.  It will
		 * have no impact on the connld vp returned and
		 * strclose() will only be called when we release
		 * flk_ocsync
		 */
		mutex_exit(&fn_lock->flk_lock);
		if ((error = stropen(oldvp, &pdev, flag, crp)) != 0) {
			mutex_enter(&fn_lock->flk_lock);
			fifo_cleanup(oldvp, flag);
			fn_lock->flk_ocsync = 0;
			cv_broadcast(&fn_lock->flk_wait_cv);
			goto out;
		}
		/*
		 * streams open done, allow close on other end if
		 * required.  Do this now.. it could
		 * be a very long time before fifo_connld returns.
		 */
		mutex_enter(&fn_lock->flk_lock);
		/*
		 * we need to fake an open here so that if this
		 * end of the pipe closes, we don't loose the
		 * stream head (kind of like single threading
		 * open and close for this end of the pipe)
		 * We'll need to call fifo_close() to do clean
		 * up in case this end of the pipe was closed
		 * down while we were in fifo_connld()
		 */
		ASSERT(fnp->fn_open > 0);
		fnp->fn_open++;
		fn_lock->flk_ocsync = 0;
		cv_broadcast(&fn_lock->flk_wait_cv);
		mutex_exit(&fn_lock->flk_lock);
		/*
		 * Connld has been pushed onto the pipe
		 * Create new pipe on behalf of connld
		 */
		if (error = fifo_connld(vpp, flag, crp)) {
			(void) fifo_close(oldvp, flag, 1, 0, crp, NULL);
			mutex_enter(&fn_lock->flk_lock);
			goto out;
		}
		/*
		 * undo fake open.  We need to call fifo_close
		 * because some other thread could have done
		 * a close and detach of the named pipe while
		 * we were in fifo_connld(), so
		 * we want to make sure the close completes (yuk)
		 */
		(void) fifo_close(oldvp, flag, 1, 0, crp, NULL);
		/*
		 * fifo_connld has changed the vp, so we
		 * need to re-initialize locals
		 */
		fnp = VTOF(*vpp);
		fn_lock = fnp->fn_lock;
		mutex_enter(&fn_lock->flk_lock);
	} else {
		/*
		 * release lock in case there are modules pushed that
		 * could have some strange side effect
		 */

		mutex_exit(&fn_lock->flk_lock);

		/*
		 * If this is the first open of a fifo (dotwist
		 * will be non-zero) we will need to twist the queues.
		 */
		if (oldvp->v_stream == NULL)
			firstopen = 1;


		/*
		 * normal open of pipe/fifo
		 */

		if ((error = stropen(oldvp, &pdev, flag, crp)) != 0) {
			mutex_enter(&fn_lock->flk_lock);
			fifo_cleanup(oldvp, flag);
			ASSERT(fnp->fn_open != 0 || oldvp->v_stream == NULL);
			fn_lock->flk_ocsync = 0;
			cv_broadcast(&fn_lock->flk_wait_cv);
			goto out;
		}
		mutex_enter(&fn_lock->flk_lock);

		/*
		 * twist the ends of the fifo together
		 */
		if (dotwist && firstopen)
			strmate(*vpp, *vpp);

		/*
		 * Show that this open has succeeded
		 * and allow closes or other opens to proceed
		 */
		fnp->fn_open++;
		fn_lock->flk_ocsync = 0;
		cv_broadcast(&fn_lock->flk_wait_cv);
	}
out:
	fnp->fn_flag &= ~FIFOOPEN;
	if (error == 0) {
		fnp->fn_flag |= FIFOISOPEN;
		/*
		 * If this is a FIFO and has the close flag set
		 * and there are now writers, clear the close flag
		 * Note: close flag only gets set when last writer
		 * on a FIFO goes away.
		 */
		if (((fnp->fn_flag & (ISPIPE|FIFOCLOSE)) == FIFOCLOSE) &&
		    fnp->fn_wcnt > 0)
			fnp->fn_flag &= ~FIFOCLOSE;
	}
	cv_broadcast(&fnp->fn_wait_cv);
	if (!lockheld)
		mutex_exit(&fn_lock->flk_lock);
	return (error);
}

/*
 * Clean up the state of a FIFO and/or mounted pipe in the
 * event that a fifo_open() was interrupted while the
 * process was blocked.
 */
void
fifo_cleanup(vnode_t *vp, int flag)
{
	fifonode_t *fnp = VTOF(vp);

	ASSERT(MUTEX_HELD(&fnp->fn_lock->flk_lock));

	cleanlocks(vp, curproc->p_pid, 0);
	cleanshares(vp, curproc->p_pid);
	if (flag & FREAD) {
		fnp->fn_rcnt--;
	}
	if (flag & FWRITE) {
		fnp->fn_wcnt--;
	}
	cv_broadcast(&fnp->fn_wait_cv);
}


/*
 * Insert a fifonode-vnode pair onto the fifoalloc hash list.
 */
static void
fifoinsert(fifonode_t *fnp)
{
	int idx = FIFOHASH(fnp->fn_realvp);

	/*
	 * We don't need to hold fn_lock since we're holding ftable_lock and
	 * this routine is only called right after we've allocated an fnode.
	 * FIFO is inserted at head of NULL terminated doubly linked list.
	 */

	ASSERT(MUTEX_HELD(&ftable_lock));
	fnp->fn_backp = NULL;
	fnp->fn_nextp = fifoalloc[idx];
	fifoalloc[idx] = fnp;
	if (fnp->fn_nextp)
		fnp->fn_nextp->fn_backp = fnp;
}

/*
 * Find a fifonode-vnode pair on the fifoalloc hash list.
 * vp is a vnode to be shadowed. If it's on the hash list,
 * it already has a shadow, therefore return its corresponding
 * fifonode.
 */
static fifonode_t *
fifofind(vnode_t *vp)
{
	fifonode_t *fnode;

	ASSERT(MUTEX_HELD(&ftable_lock));
	for (fnode = fifoalloc[FIFOHASH(vp)]; fnode; fnode = fnode->fn_nextp) {
		if (fnode->fn_realvp == vp) {
			VN_HOLD(FTOV(fnode));
			return (fnode);
		}
	}
	return (NULL);
}

/*
 * Remove a fifonode-vnode pair from the fifoalloc hash list.
 * This routine is called from the fifo_inactive() routine when a
 * FIFO is being released.
 * If the link to be removed is the only link, set fifoalloc to NULL.
 */
void
fiforemove(fifonode_t *fnp)
{
	int idx = FIFOHASH(fnp->fn_realvp);
	fifonode_t *fnode;

	ASSERT(MUTEX_HELD(&ftable_lock));
	fnode = fifoalloc[idx];
	/*
	 * fast path... only 1 FIFO in this list entry
	 */
	if (fnode != NULL && fnode == fnp &&
	    !fnode->fn_nextp && !fnode->fn_backp) {
		fifoalloc[idx] = NULL;
	} else {

		for (;  fnode;  fnode = fnode->fn_nextp) {
			if (fnode == fnp) {
				/*
				 * if we are first entry
				 */
				if (fnp == fifoalloc[idx])
					fifoalloc[idx] = fnp->fn_nextp;
				if (fnode->fn_nextp)
					fnode->fn_nextp->fn_backp =
					    fnode->fn_backp;
				if (fnode->fn_backp)
					fnode->fn_backp->fn_nextp =
					    fnode->fn_nextp;
				break;
			}
		}
	}
}

/*
 * Flush all data from a fifo's message queue
 */

void
fifo_fastflush(fifonode_t *fnp)
{
	mblk_t *bp;
	ASSERT(MUTEX_HELD(&fnp->fn_lock->flk_lock));

	if ((bp = fnp->fn_mp) != NULL) {
		fnp->fn_mp = NULL;
		fnp->fn_count = 0;
		freemsg(bp);
	}
	fifo_wakewriter(fnp->fn_dest, fnp->fn_lock);
}

/*
 * Note:  This routine is single threaded
 *  Protected by FIFOOPEN flag (i.e. flk_lock is not held)
 *  Upon successful completion, the original fifo is unlocked
 *  and FIFOOPEN is cleared for the original vpp.
 *  The new fifo returned has FIFOOPEN set.
 */
static int
fifo_connld(struct vnode **vpp, int flag, cred_t *crp)
{
	struct vnode *vp1;
	struct vnode *vp2;
	struct fifonode *oldfnp;
	struct fifonode *fn_dest;
	int error;
	struct file *filep;
	struct fifolock *fn_lock;
	cred_t *c;

	/*
	 * Get two vnodes that will represent the pipe ends for the new pipe.
	 */
	makepipe(&vp1, &vp2);

	/*
	 * Allocate a file descriptor and file pointer for one of the pipe
	 * ends. The file descriptor will be used to send that pipe end to
	 * the process on the other end of this stream. Note that we get
	 * the file structure only, there is no file list entry allocated.
	 */
	if (error = falloc(vp1, FWRITE|FREAD, &filep, NULL)) {
		VN_RELE(vp1);
		VN_RELE(vp2);
		return (error);
	}
	mutex_exit(&filep->f_tlock);
	oldfnp = VTOF(*vpp);
	fn_lock = oldfnp->fn_lock;
	fn_dest = oldfnp->fn_dest;

	/*
	 * Create two new stream heads and attach them to the two vnodes for
	 * the new pipe.
	 */
	if ((error = fifo_stropen(&vp1, FREAD|FWRITE, filep->f_cred, 0, 0)) !=
	    0 ||
	    (error = fifo_stropen(&vp2, flag, filep->f_cred, 0, 0)) != 0) {
#if DEBUG
		cmn_err(CE_NOTE, "fifo stropen failed error 0x%x", error);
#endif
		/*
		 * this will call fifo_close and VN_RELE on vp1
		 */
		(void) closef(filep);
		VN_RELE(vp2);
		return (error);
	}

	/*
	 * twist the ends of the pipe together
	 */
	strmate(vp1, vp2);

	/*
	 * Set our end to busy in open
	 * Note: Don't need lock around this because we're the only
	 * one who knows about it
	 */
	VTOF(vp2)->fn_flag |= FIFOOPEN;

	mutex_enter(&fn_lock->flk_lock);

	fn_dest->fn_flag |= FIFOSEND;
	/*
	 * check to make sure neither end of pipe has gone away
	 */
	if (!(fn_dest->fn_flag & FIFOISOPEN)) {
		error = ENXIO;
		fn_dest->fn_flag &= ~FIFOSEND;
		mutex_exit(&fn_lock->flk_lock);
		/*
		 * this will call fifo_close and VN_RELE on vp1
		 */
		goto out;
	}
	mutex_exit(&fn_lock->flk_lock);

	/*
	 * Tag the sender's credential on the pipe descriptor.
	 */
	crhold(VTOF(vp1)->fn_pcredp = crp);
	VTOF(vp1)->fn_cpid = curproc->p_pid;

	/*
	 * send the file descriptor to other end of pipe
	 */
	if (error = do_sendfp((*vpp)->v_stream, filep, crp)) {
		mutex_enter(&fn_lock->flk_lock);
		fn_dest->fn_flag &= ~FIFOSEND;
		mutex_exit(&fn_lock->flk_lock);
		/*
		 * this will call fifo_close and VN_RELE on vp1
		 */
		goto out;
	}

	mutex_enter(&fn_lock->flk_lock);
	/*
	 * Wait for other end to receive file descriptor
	 * FIFOCLOSE indicates that one or both sides of the pipe
	 * have gone away.
	 */
	while ((fn_dest->fn_flag & (FIFOCLOSE | FIFOSEND)) == FIFOSEND) {
		if (!cv_wait_sig(&oldfnp->fn_wait_cv, &fn_lock->flk_lock)) {
			error = EINTR;
			fn_dest->fn_flag &= ~FIFOSEND;
			mutex_exit(&fn_lock->flk_lock);
			goto out;
		}
	}
	/*
	 * If either end of pipe has gone away and the other end did not
	 * receive pipe, reject the connld open
	 */
	if ((fn_dest->fn_flag & FIFOSEND)) {
		error = ENXIO;
		fn_dest->fn_flag &= ~FIFOSEND;
		mutex_exit(&fn_lock->flk_lock);
		goto out;
	}

	oldfnp->fn_flag &= ~FIFOOPEN;
	cv_broadcast(&oldfnp->fn_wait_cv);
	mutex_exit(&fn_lock->flk_lock);

	VN_RELE(*vpp);
	*vpp = vp2;
	(void) closef(filep);
	return (0);
out:
	c = filep->f_cred;
	crhold(c);
	(void) closef(filep);
	VTOF(vp2)->fn_flag &= ~FIFOOPEN;
	(void) fifo_close(vp2, flag, 1, (offset_t)0, c, NULL);
	crfree(c);
	VN_RELE(vp2);
	return (error);
}

/*
 * Disable fastpath mode.
 */
void
fifo_fastoff(fifonode_t *fnp)
{
	ASSERT(MUTEX_HELD(&fnp->fn_lock->flk_lock));
	ASSERT(FTOV(fnp)->v_stream);

	/* FIFOSTAYFAST is set => FIFOFAST is set */
	while ((fnp->fn_flag & FIFOSTAYFAST) || ((fnp->fn_flag & ISPIPE) &&
	    (fnp->fn_dest->fn_flag & FIFOSTAYFAST))) {
		ASSERT(fnp->fn_flag & FIFOFAST);
		/* indicate someone is waiting to turn into stream mode */
		fnp->fn_flag |= FIFOWAITMODE;
		cv_wait(&fnp->fn_wait_cv, &fnp->fn_lock->flk_lock);
		fnp->fn_flag &= ~FIFOWAITMODE;
	}

	/* as we may have relased the lock, test the FIFOFAST flag here */
	if (!(fnp->fn_flag & FIFOFAST))
		return;
#if FIFODEBUG
	if (Fifo_verbose)
		cmn_err(CE_NOTE, "Fifo reverting to streams mode\n");
#endif

	fifo_fastturnoff(fnp);
	if (fnp->fn_flag & ISPIPE) {
		fifo_fastturnoff(fnp->fn_dest);
	}
}


/*
 * flk_lock must be held while calling fifo_fastturnoff() to
 * preserve data ordering (no reads or writes allowed)
 */

static void
fifo_fastturnoff(fifonode_t *fnp)
{
	fifonode_t *fn_dest = fnp->fn_dest;
	mblk_t	*fn_mp;
	int	fn_flag;

	ASSERT(MUTEX_HELD(&fnp->fn_lock->flk_lock));
	/*
	 * Note: This end can't be closed if there
	 * is stuff in fn_mp
	 */
	if ((fn_mp = fnp->fn_mp) != NULL) {
		ASSERT(fnp->fn_flag & FIFOISOPEN);
		ASSERT(FTOV(fnp)->v_stream != NULL);
		ASSERT(FTOV(fnp)->v_stream->sd_wrq != NULL);
		ASSERT(RD(FTOV(fnp)->v_stream->sd_wrq) != NULL);
		ASSERT(strvp2wq(FTOV(fnp)) != NULL);
		fnp->fn_mp = NULL;
		fnp->fn_count = 0;
		/*
		 * Don't need to drop flk_lock across the put()
		 * since we're just moving the message from the fifo
		 * node to the STREAM head...
		 */
		put(RD(strvp2wq(FTOV(fnp))), fn_mp);
	}

	/*
	 * Need to re-issue any pending poll requests
	 * so that the STREAMS framework sees them
	 * Writers would be waiting on fnp and readers on fn_dest
	 */
	if ((fnp->fn_flag & (FIFOISOPEN | FIFOPOLLW)) ==
	    (FIFOISOPEN | FIFOPOLLW)) {
		strpollwakeup(FTOV(fnp), POLLWRNORM);
	}
	fn_flag = fn_dest->fn_flag;
	if ((fn_flag & FIFOISOPEN) == FIFOISOPEN) {
		if ((fn_flag & (FIFOPOLLR | FIFOPOLLRBAND))) {
			strpollwakeup(FTOV(fn_dest), POLLIN|POLLRDNORM);
		}
	}
	/*
	 * wake up any sleeping processes so they can notice we went
	 * to streams mode
	 */
	fnp->fn_flag &= ~(FIFOFAST|FIFOWANTW|FIFOWANTR);
	cv_broadcast(&fnp->fn_wait_cv);
}

/*
 * Alternative version of fifo_fastoff()
 * optimized for putmsg/getmsg.
 */
void
fifo_vfastoff(vnode_t *vp)
{
	fifonode_t	*fnp = VTOF(vp);

	mutex_enter(&fnp->fn_lock->flk_lock);
	if (!(fnp->fn_flag & FIFOFAST)) {
		mutex_exit(&fnp->fn_lock->flk_lock);
		return;
	}
	fifo_fastoff(fnp);
	mutex_exit(&fnp->fn_lock->flk_lock);
}

/*
 * Wake any sleeping writers, poll and send signals if necessary
 * This module is only called when we drop below the hi water mark
 * FIFOWANTW indicates that a process is sleeping in fifo_write()
 * FIFOHIWATW indicates that we have either attempted a poll or
 * non-blocking write and were over the high water mark
 * This routine assumes a low water mark of 0.
 */

void
fifo_wakewriter(fifonode_t *fn_dest, fifolock_t *fn_lock)
{
	int fn_dflag = fn_dest->fn_flag;

	ASSERT(MUTEX_HELD(&fn_lock->flk_lock));
	ASSERT(fn_dest->fn_dest->fn_count < Fifohiwat);
	if ((fn_dflag & FIFOWANTW)) {
		cv_broadcast(&fn_dest->fn_wait_cv);
	}
	if ((fn_dflag & (FIFOHIWATW | FIFOISOPEN)) ==
	    (FIFOHIWATW | FIFOISOPEN)) {
		if (fn_dflag & FIFOPOLLW)
			strpollwakeup(FTOV(fn_dest), POLLWRNORM);
		if (fn_dflag & FIFOSETSIG)
			str_sendsig(FTOV(fn_dest), S_WRNORM, 0, 0);
	}
	/*
	 * FIFOPOLLW can't be set without setting FIFOHIWAT
	 * This allows us to clear both here.
	 */
	fn_dest->fn_flag = fn_dflag & ~(FIFOWANTW | FIFOHIWATW | FIFOPOLLW);
}

/*
 * wake up any sleeping readers, poll or send signal if needed
 * FIFOWANTR indicates that a process is waiting in fifo_read() for data
 * FIFOSETSIG indicates that SIGPOLL should be sent to process
 * FIFOPOLLR indicates that a poll request for reading on the fifo was made
 */

void
fifo_wakereader(fifonode_t *fn_dest, fifolock_t *fn_lock)
{
	int fn_dflag = fn_dest->fn_flag;

	ASSERT(MUTEX_HELD(&fn_lock->flk_lock));
	if (fn_dflag & FIFOWANTR) {
		cv_broadcast(&fn_dest->fn_wait_cv);
	}
	if (fn_dflag & FIFOISOPEN) {
		if (fn_dflag & FIFOPOLLR)
			strpollwakeup(FTOV(fn_dest), POLLIN | POLLRDNORM);
		if (fn_dflag & FIFOSETSIG)
			str_sendsig(FTOV(fn_dest), S_INPUT | S_RDNORM, 0, 0);
	}
	fn_dest->fn_flag = fn_dflag & ~(FIFOWANTR | FIFOPOLLR);
}