xref: /onnv-gate/usr/src/uts/common/inet/tcp/tcp_fusion.c (revision 12644:4f9a0cd40c5f)
1741Smasputra /*
2741Smasputra  * CDDL HEADER START
3741Smasputra  *
4741Smasputra  * The contents of this file are subject to the terms of the
52024Skrishna  * Common Development and Distribution License (the "License").
62024Skrishna  * You may not use this file except in compliance with the License.
7741Smasputra  *
8741Smasputra  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9741Smasputra  * or http://www.opensolaris.org/os/licensing.
10741Smasputra  * See the License for the specific language governing permissions
11741Smasputra  * and limitations under the License.
12741Smasputra  *
13741Smasputra  * When distributing Covered Code, include this CDDL HEADER in each
14741Smasputra  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15741Smasputra  * If applicable, add the following below this CDDL HEADER, with the
16741Smasputra  * fields enclosed by brackets "[]" replaced with your own identifying
17741Smasputra  * information: Portions Copyright [yyyy] [name of copyright owner]
18741Smasputra  *
19741Smasputra  * CDDL HEADER END
20741Smasputra  */
21741Smasputra /*
2212056SKacheong.Poon@Sun.COM  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23741Smasputra  */
24741Smasputra 
25741Smasputra #include <sys/types.h>
26741Smasputra #include <sys/stream.h>
27741Smasputra #include <sys/strsun.h>
28741Smasputra #include <sys/strsubr.h>
29741Smasputra #include <sys/debug.h>
302958Sdr146992 #include <sys/sdt.h>
31741Smasputra #include <sys/cmn_err.h>
32741Smasputra #include <sys/tihdr.h>
33741Smasputra 
34741Smasputra #include <inet/common.h>
355240Snordmark #include <inet/optcom.h>
36741Smasputra #include <inet/ip.h>
378485SPeter.Memishian@Sun.COM #include <inet/ip_if.h>
38741Smasputra #include <inet/ip_impl.h>
39741Smasputra #include <inet/tcp.h>
40741Smasputra #include <inet/tcp_impl.h>
41741Smasputra #include <inet/ipsec_impl.h>
42741Smasputra #include <inet/ipclassifier.h>
43741Smasputra #include <inet/ipp_common.h>
447828SBrian.Ruthven@Sun.COM #include <inet/ip_if.h>
45741Smasputra 
46741Smasputra /*
47741Smasputra  * This file implements TCP fusion - a protocol-less data path for TCP
48741Smasputra  * loopback connections.  The fusion of two local TCP endpoints occurs
49741Smasputra  * at connection establishment time.  Various conditions (see details
50741Smasputra  * in tcp_fuse()) need to be met for fusion to be successful.  If it
51741Smasputra  * fails, we fall back to the regular TCP data path; if it succeeds,
52741Smasputra  * both endpoints proceed to use tcp_fuse_output() as the transmit path.
53741Smasputra  * tcp_fuse_output() enqueues application data directly onto the peer's
549993SAnders.Persson@Sun.COM  * receive queue; no protocol processing is involved.
55741Smasputra  *
563429Svi117747  * Sychronization is handled by squeue and the mutex tcp_non_sq_lock.
57741Smasputra  * One of the requirements for fusion to succeed is that both endpoints
58741Smasputra  * need to be using the same squeue.  This ensures that neither side
599993SAnders.Persson@Sun.COM  * can disappear while the other side is still sending data. Flow
609993SAnders.Persson@Sun.COM  * control information is manipulated outside the squeue, so the
619993SAnders.Persson@Sun.COM  * tcp_non_sq_lock must be held when touching tcp_flow_stopped.
62741Smasputra  */
63741Smasputra 
64741Smasputra /*
65741Smasputra  * Setting this to false means we disable fusion altogether and
66741Smasputra  * loopback connections would go through the protocol paths.
67741Smasputra  */
68741Smasputra boolean_t do_tcp_fusion = B_TRUE;
69741Smasputra 
70741Smasputra /*
71741Smasputra  * This routine gets called by the eager tcp upon changing state from
72741Smasputra  * SYN_RCVD to ESTABLISHED.  It fuses a direct path between itself
73741Smasputra  * and the active connect tcp such that the regular tcp processings
74741Smasputra  * may be bypassed under allowable circumstances.  Because the fusion
75741Smasputra  * requires both endpoints to be in the same squeue, it does not work
76741Smasputra  * for simultaneous active connects because there is no easy way to
77741Smasputra  * switch from one squeue to another once the connection is created.
78741Smasputra  * This is different from the eager tcp case where we assign it the
79741Smasputra  * same squeue as the one given to the active connect tcp during open.
80741Smasputra  */
81741Smasputra void
tcp_fuse(tcp_t * tcp,uchar_t * iphdr,tcpha_t * tcpha)8211042SErik.Nordmark@Sun.COM tcp_fuse(tcp_t *tcp, uchar_t *iphdr, tcpha_t *tcpha)
83741Smasputra {
8411042SErik.Nordmark@Sun.COM 	conn_t		*peer_connp, *connp = tcp->tcp_connp;
8511042SErik.Nordmark@Sun.COM 	tcp_t		*peer_tcp;
863448Sdh155122 	tcp_stack_t	*tcps = tcp->tcp_tcps;
873448Sdh155122 	netstack_t	*ns;
883448Sdh155122 	ip_stack_t	*ipst = tcps->tcps_netstack->netstack_ip;
89741Smasputra 
90741Smasputra 	ASSERT(!tcp->tcp_fused);
91741Smasputra 	ASSERT(tcp->tcp_loopback);
92741Smasputra 	ASSERT(tcp->tcp_loopback_peer == NULL);
93741Smasputra 	/*
9411042SErik.Nordmark@Sun.COM 	 * We need to inherit conn_rcvbuf of the listener tcp,
9510312SRao.Shoaib@Sun.COM 	 * but we can't really use tcp_listener since we get here after
9611042SErik.Nordmark@Sun.COM 	 * sending up T_CONN_IND and tcp_tli_accept() may be called
9710312SRao.Shoaib@Sun.COM 	 * independently, at which point tcp_listener is cleared;
9810312SRao.Shoaib@Sun.COM 	 * this is why we use tcp_saved_listener. The listener itself
9910312SRao.Shoaib@Sun.COM 	 * is guaranteed to be around until tcp_accept_finish() is called
10010312SRao.Shoaib@Sun.COM 	 * on this eager -- this won't happen until we're done since we're
10110312SRao.Shoaib@Sun.COM 	 * inside the eager's perimeter now.
102741Smasputra 	 */
10311042SErik.Nordmark@Sun.COM 	ASSERT(tcp->tcp_saved_listener != NULL);
104741Smasputra 	/*
105741Smasputra 	 * Lookup peer endpoint; search for the remote endpoint having
106741Smasputra 	 * the reversed address-port quadruplet in ESTABLISHED state,
107741Smasputra 	 * which is guaranteed to be unique in the system.  Zone check
108741Smasputra 	 * is applied accordingly for loopback address, but not for
109741Smasputra 	 * local address since we want fusion to happen across Zones.
110741Smasputra 	 */
11111042SErik.Nordmark@Sun.COM 	if (connp->conn_ipversion == IPV4_VERSION) {
112741Smasputra 		peer_connp = ipcl_conn_tcp_lookup_reversed_ipv4(connp,
11311042SErik.Nordmark@Sun.COM 		    (ipha_t *)iphdr, tcpha, ipst);
114741Smasputra 	} else {
115741Smasputra 		peer_connp = ipcl_conn_tcp_lookup_reversed_ipv6(connp,
11611042SErik.Nordmark@Sun.COM 		    (ip6_t *)iphdr, tcpha, ipst);
117741Smasputra 	}
118741Smasputra 
119741Smasputra 	/*
120741Smasputra 	 * We can only proceed if peer exists, resides in the same squeue
1219992SAnders.Persson@Sun.COM 	 * as our conn and is not raw-socket. We also restrict fusion to
1229992SAnders.Persson@Sun.COM 	 * endpoints of the same type (STREAMS or non-STREAMS). The squeue
1239992SAnders.Persson@Sun.COM 	 * assignment of this eager tcp was done earlier at the time of SYN
1249992SAnders.Persson@Sun.COM 	 * processing in ip_fanout_tcp{_v6}.  Note that similar squeues by
1259992SAnders.Persson@Sun.COM 	 * itself doesn't guarantee a safe condition to fuse, hence we perform
126741Smasputra 	 * additional tests below.
127741Smasputra 	 */
128741Smasputra 	ASSERT(peer_connp == NULL || peer_connp != connp);
129741Smasputra 	if (peer_connp == NULL || peer_connp->conn_sqp != connp->conn_sqp ||
1309992SAnders.Persson@Sun.COM 	    !IPCL_IS_TCP(peer_connp) ||
1319992SAnders.Persson@Sun.COM 	    IPCL_IS_NONSTR(connp) != IPCL_IS_NONSTR(peer_connp)) {
132741Smasputra 		if (peer_connp != NULL) {
1333448Sdh155122 			TCP_STAT(tcps, tcp_fusion_unqualified);
134741Smasputra 			CONN_DEC_REF(peer_connp);
135741Smasputra 		}
136741Smasputra 		return;
137741Smasputra 	}
138741Smasputra 	peer_tcp = peer_connp->conn_tcp;	/* active connect tcp */
139741Smasputra 
140741Smasputra 	ASSERT(peer_tcp != NULL && peer_tcp != tcp && !peer_tcp->tcp_fused);
1419532SErik.Nordmark@Sun.COM 	ASSERT(peer_tcp->tcp_loopback_peer == NULL);
142741Smasputra 	ASSERT(peer_connp->conn_sqp == connp->conn_sqp);
143741Smasputra 
144741Smasputra 	/*
1459532SErik.Nordmark@Sun.COM 	 * Due to IRE changes the peer and us might not agree on tcp_loopback.
1469532SErik.Nordmark@Sun.COM 	 * We bail in that case.
1479532SErik.Nordmark@Sun.COM 	 */
1489532SErik.Nordmark@Sun.COM 	if (!peer_tcp->tcp_loopback) {
1499532SErik.Nordmark@Sun.COM 		TCP_STAT(tcps, tcp_fusion_unqualified);
1509532SErik.Nordmark@Sun.COM 		CONN_DEC_REF(peer_connp);
1519532SErik.Nordmark@Sun.COM 		return;
1529532SErik.Nordmark@Sun.COM 	}
1539532SErik.Nordmark@Sun.COM 	/*
154741Smasputra 	 * Fuse the endpoints; we perform further checks against both
155741Smasputra 	 * tcp endpoints to ensure that a fusion is allowed to happen.
156741Smasputra 	 */
1573448Sdh155122 	ns = tcps->tcps_netstack;
1583448Sdh155122 	ipst = ns->netstack_ip;
1593448Sdh155122 
160741Smasputra 	if (!tcp->tcp_unfusable && !peer_tcp->tcp_unfusable &&
161*12644SAnders.Persson@Sun.COM 	    tcp->tcp_xmit_head == NULL && peer_tcp->tcp_xmit_head == NULL) {
162741Smasputra 		mblk_t *mp;
16311042SErik.Nordmark@Sun.COM 		queue_t *peer_rq = peer_connp->conn_rq;
164741Smasputra 
1658348SEric.Yu@Sun.COM 		ASSERT(!TCP_IS_DETACHED(peer_tcp));
16611042SErik.Nordmark@Sun.COM 		ASSERT(tcp->tcp_fused_sigurg_mp == NULL);
16711042SErik.Nordmark@Sun.COM 		ASSERT(peer_tcp->tcp_fused_sigurg_mp == NULL);
168741Smasputra 
169741Smasputra 		/*
170741Smasputra 		 * We need to drain data on both endpoints during unfuse.
171741Smasputra 		 * If we need to send up SIGURG at the time of draining,
172741Smasputra 		 * we want to be sure that an mblk is readily available.
173741Smasputra 		 * This is why we pre-allocate the M_PCSIG mblks for both
174741Smasputra 		 * endpoints which will only be used during/after unfuse.
1759994SAnders.Persson@Sun.COM 		 * The mblk might already exist if we are doing a re-fuse.
176741Smasputra 		 */
1778348SEric.Yu@Sun.COM 		if (!IPCL_IS_NONSTR(tcp->tcp_connp)) {
1789992SAnders.Persson@Sun.COM 			ASSERT(!IPCL_IS_NONSTR(peer_tcp->tcp_connp));
1798348SEric.Yu@Sun.COM 
1809994SAnders.Persson@Sun.COM 			if (tcp->tcp_fused_sigurg_mp == NULL) {
1819994SAnders.Persson@Sun.COM 				if ((mp = allocb(1, BPRI_HI)) == NULL)
1829994SAnders.Persson@Sun.COM 					goto failed;
1839994SAnders.Persson@Sun.COM 				tcp->tcp_fused_sigurg_mp = mp;
1849994SAnders.Persson@Sun.COM 			}
1858348SEric.Yu@Sun.COM 
1869994SAnders.Persson@Sun.COM 			if (peer_tcp->tcp_fused_sigurg_mp == NULL) {
1879994SAnders.Persson@Sun.COM 				if ((mp = allocb(1, BPRI_HI)) == NULL)
1889994SAnders.Persson@Sun.COM 					goto failed;
1899994SAnders.Persson@Sun.COM 				peer_tcp->tcp_fused_sigurg_mp = mp;
1909994SAnders.Persson@Sun.COM 			}
1919992SAnders.Persson@Sun.COM 
1929992SAnders.Persson@Sun.COM 			if ((mp = allocb(sizeof (struct stroptions),
1939992SAnders.Persson@Sun.COM 			    BPRI_HI)) == NULL)
1949992SAnders.Persson@Sun.COM 				goto failed;
1958348SEric.Yu@Sun.COM 		}
196741Smasputra 
197741Smasputra 		/* Fuse both endpoints */
198741Smasputra 		peer_tcp->tcp_loopback_peer = tcp;
199741Smasputra 		tcp->tcp_loopback_peer = peer_tcp;
200741Smasputra 		peer_tcp->tcp_fused = tcp->tcp_fused = B_TRUE;
201741Smasputra 
202741Smasputra 		/*
203741Smasputra 		 * We never use regular tcp paths in fusion and should
204741Smasputra 		 * therefore clear tcp_unsent on both endpoints.  Having
205741Smasputra 		 * them set to non-zero values means asking for trouble
206741Smasputra 		 * especially after unfuse, where we may end up sending
207741Smasputra 		 * through regular tcp paths which expect xmit_list and
208741Smasputra 		 * friends to be correctly setup.
209741Smasputra 		 */
210741Smasputra 		peer_tcp->tcp_unsent = tcp->tcp_unsent = 0;
211741Smasputra 
212741Smasputra 		tcp_timers_stop(tcp);
213741Smasputra 		tcp_timers_stop(peer_tcp);
214741Smasputra 
21511042SErik.Nordmark@Sun.COM 		/*
21611042SErik.Nordmark@Sun.COM 		 * Set receive buffer and max packet size for the
21711042SErik.Nordmark@Sun.COM 		 * active open tcp.
21811042SErik.Nordmark@Sun.COM 		 * eager's values will be set in tcp_accept_finish.
21911042SErik.Nordmark@Sun.COM 		 */
22011042SErik.Nordmark@Sun.COM 		(void) tcp_rwnd_set(peer_tcp, peer_tcp->tcp_connp->conn_rcvbuf);
221741Smasputra 
22211042SErik.Nordmark@Sun.COM 		/*
22311042SErik.Nordmark@Sun.COM 		 * Set the write offset value to zero since we won't
22411042SErik.Nordmark@Sun.COM 		 * be needing any room for TCP/IP headers.
22511042SErik.Nordmark@Sun.COM 		 */
22611042SErik.Nordmark@Sun.COM 		if (!IPCL_IS_NONSTR(peer_tcp->tcp_connp)) {
22711042SErik.Nordmark@Sun.COM 			struct stroptions *stropt;
2288348SEric.Yu@Sun.COM 
22911042SErik.Nordmark@Sun.COM 			DB_TYPE(mp) = M_SETOPTS;
23011042SErik.Nordmark@Sun.COM 			mp->b_wptr += sizeof (*stropt);
2318348SEric.Yu@Sun.COM 
23211042SErik.Nordmark@Sun.COM 			stropt = (struct stroptions *)mp->b_rptr;
23312643SAnders.Persson@Sun.COM 			stropt->so_flags = SO_WROFF | SO_MAXBLK;
23411042SErik.Nordmark@Sun.COM 			stropt->so_wroff = 0;
23512643SAnders.Persson@Sun.COM 			stropt->so_maxblk = INFPSZ;
2368348SEric.Yu@Sun.COM 
23711042SErik.Nordmark@Sun.COM 			/* Send the options up */
23811042SErik.Nordmark@Sun.COM 			putnext(peer_rq, mp);
23911042SErik.Nordmark@Sun.COM 		} else {
24011042SErik.Nordmark@Sun.COM 			struct sock_proto_props sopp;
241741Smasputra 
24211042SErik.Nordmark@Sun.COM 			/* The peer is a non-STREAMS end point */
24311042SErik.Nordmark@Sun.COM 			ASSERT(IPCL_IS_TCP(peer_connp));
24411042SErik.Nordmark@Sun.COM 
24512643SAnders.Persson@Sun.COM 			sopp.sopp_flags = SOCKOPT_WROFF | SOCKOPT_MAXBLK;
24611042SErik.Nordmark@Sun.COM 			sopp.sopp_wroff = 0;
24712643SAnders.Persson@Sun.COM 			sopp.sopp_maxblk = INFPSZ;
24811042SErik.Nordmark@Sun.COM 			(*peer_connp->conn_upcalls->su_set_proto_props)
24911042SErik.Nordmark@Sun.COM 			    (peer_connp->conn_upper_handle, &sopp);
2508023SPhil.Kirk@Sun.COM 		}
251741Smasputra 	} else {
2523448Sdh155122 		TCP_STAT(tcps, tcp_fusion_unqualified);
253741Smasputra 	}
254741Smasputra 	CONN_DEC_REF(peer_connp);
255741Smasputra 	return;
256741Smasputra 
257741Smasputra failed:
258741Smasputra 	if (tcp->tcp_fused_sigurg_mp != NULL) {
259741Smasputra 		freeb(tcp->tcp_fused_sigurg_mp);
260741Smasputra 		tcp->tcp_fused_sigurg_mp = NULL;
261741Smasputra 	}
262741Smasputra 	if (peer_tcp->tcp_fused_sigurg_mp != NULL) {
263741Smasputra 		freeb(peer_tcp->tcp_fused_sigurg_mp);
264741Smasputra 		peer_tcp->tcp_fused_sigurg_mp = NULL;
265741Smasputra 	}
266741Smasputra 	CONN_DEC_REF(peer_connp);
267741Smasputra }
268741Smasputra 
269741Smasputra /*
270741Smasputra  * Unfuse a previously-fused pair of tcp loopback endpoints.
271741Smasputra  */
272741Smasputra void
tcp_unfuse(tcp_t * tcp)273741Smasputra tcp_unfuse(tcp_t *tcp)
274741Smasputra {
275741Smasputra 	tcp_t *peer_tcp = tcp->tcp_loopback_peer;
2769993SAnders.Persson@Sun.COM 	tcp_stack_t *tcps = tcp->tcp_tcps;
277741Smasputra 
278741Smasputra 	ASSERT(tcp->tcp_fused && peer_tcp != NULL);
279741Smasputra 	ASSERT(peer_tcp->tcp_fused && peer_tcp->tcp_loopback_peer == tcp);
280741Smasputra 	ASSERT(tcp->tcp_connp->conn_sqp == peer_tcp->tcp_connp->conn_sqp);
281741Smasputra 	ASSERT(tcp->tcp_unsent == 0 && peer_tcp->tcp_unsent == 0);
282741Smasputra 
283741Smasputra 	/*
2849993SAnders.Persson@Sun.COM 	 * Cancel any pending push timers.
2859993SAnders.Persson@Sun.COM 	 */
2869993SAnders.Persson@Sun.COM 	if (tcp->tcp_push_tid != 0) {
2879993SAnders.Persson@Sun.COM 		(void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid);
2889993SAnders.Persson@Sun.COM 		tcp->tcp_push_tid = 0;
2899993SAnders.Persson@Sun.COM 	}
2909993SAnders.Persson@Sun.COM 	if (peer_tcp->tcp_push_tid != 0) {
2919993SAnders.Persson@Sun.COM 		(void) TCP_TIMER_CANCEL(peer_tcp, peer_tcp->tcp_push_tid);
2929993SAnders.Persson@Sun.COM 		peer_tcp->tcp_push_tid = 0;
2939993SAnders.Persson@Sun.COM 	}
2949993SAnders.Persson@Sun.COM 
2959993SAnders.Persson@Sun.COM 	/*
2969993SAnders.Persson@Sun.COM 	 * Drain any pending data; Note that in case of a detached tcp, the
2979993SAnders.Persson@Sun.COM 	 * draining will happen later after the tcp is unfused.  For non-
2989993SAnders.Persson@Sun.COM 	 * urgent data, this can be handled by the regular tcp_rcv_drain().
2999993SAnders.Persson@Sun.COM 	 * If we have urgent data sitting in the receive list, we will
3009993SAnders.Persson@Sun.COM 	 * need to send up a SIGURG signal first before draining the data.
3019993SAnders.Persson@Sun.COM 	 * All of these will be handled by the code in tcp_fuse_rcv_drain()
3029993SAnders.Persson@Sun.COM 	 * when called from tcp_rcv_drain().
303741Smasputra 	 */
3049993SAnders.Persson@Sun.COM 	if (!TCP_IS_DETACHED(tcp)) {
30511042SErik.Nordmark@Sun.COM 		(void) tcp_fuse_rcv_drain(tcp->tcp_connp->conn_rq, tcp,
3069993SAnders.Persson@Sun.COM 		    &tcp->tcp_fused_sigurg_mp);
3079993SAnders.Persson@Sun.COM 	}
3089993SAnders.Persson@Sun.COM 	if (!TCP_IS_DETACHED(peer_tcp)) {
30911042SErik.Nordmark@Sun.COM 		(void) tcp_fuse_rcv_drain(peer_tcp->tcp_connp->conn_rq,
31011042SErik.Nordmark@Sun.COM 		    peer_tcp,  &peer_tcp->tcp_fused_sigurg_mp);
3119993SAnders.Persson@Sun.COM 	}
3129993SAnders.Persson@Sun.COM 
3139993SAnders.Persson@Sun.COM 	/* Lift up any flow-control conditions */
3149993SAnders.Persson@Sun.COM 	mutex_enter(&tcp->tcp_non_sq_lock);
3159993SAnders.Persson@Sun.COM 	if (tcp->tcp_flow_stopped) {
3169993SAnders.Persson@Sun.COM 		tcp_clrqfull(tcp);
3179993SAnders.Persson@Sun.COM 		TCP_STAT(tcps, tcp_fusion_backenabled);
3189993SAnders.Persson@Sun.COM 	}
3199993SAnders.Persson@Sun.COM 	mutex_exit(&tcp->tcp_non_sq_lock);
3209993SAnders.Persson@Sun.COM 
3219993SAnders.Persson@Sun.COM 	mutex_enter(&peer_tcp->tcp_non_sq_lock);
3229993SAnders.Persson@Sun.COM 	if (peer_tcp->tcp_flow_stopped) {
3239993SAnders.Persson@Sun.COM 		tcp_clrqfull(peer_tcp);
3249993SAnders.Persson@Sun.COM 		TCP_STAT(tcps, tcp_fusion_backenabled);
3259993SAnders.Persson@Sun.COM 	}
3269993SAnders.Persson@Sun.COM 	mutex_exit(&peer_tcp->tcp_non_sq_lock);
327741Smasputra 
328741Smasputra 	/*
32911042SErik.Nordmark@Sun.COM 	 * Update tha_seq and tha_ack in the header template
330741Smasputra 	 */
33111042SErik.Nordmark@Sun.COM 	tcp->tcp_tcpha->tha_seq = htonl(tcp->tcp_snxt);
33211042SErik.Nordmark@Sun.COM 	tcp->tcp_tcpha->tha_ack = htonl(tcp->tcp_rnxt);
33311042SErik.Nordmark@Sun.COM 	peer_tcp->tcp_tcpha->tha_seq = htonl(peer_tcp->tcp_snxt);
33411042SErik.Nordmark@Sun.COM 	peer_tcp->tcp_tcpha->tha_ack = htonl(peer_tcp->tcp_rnxt);
335741Smasputra 
336741Smasputra 	/* Unfuse the endpoints */
337741Smasputra 	peer_tcp->tcp_fused = tcp->tcp_fused = B_FALSE;
338741Smasputra 	peer_tcp->tcp_loopback_peer = tcp->tcp_loopback_peer = NULL;
339741Smasputra }
340741Smasputra 
341741Smasputra /*
3429992SAnders.Persson@Sun.COM  * Fusion output routine used to handle urgent data sent by STREAMS based
3439992SAnders.Persson@Sun.COM  * endpoints. This routine is called by tcp_fuse_output() for handling
3449992SAnders.Persson@Sun.COM  * non-M_DATA mblks.
345741Smasputra  */
346741Smasputra void
tcp_fuse_output_urg(tcp_t * tcp,mblk_t * mp)347741Smasputra tcp_fuse_output_urg(tcp_t *tcp, mblk_t *mp)
348741Smasputra {
349741Smasputra 	mblk_t *mp1;
350741Smasputra 	struct T_exdata_ind *tei;
351741Smasputra 	tcp_t *peer_tcp = tcp->tcp_loopback_peer;
352741Smasputra 	mblk_t *head, *prev_head = NULL;
3533448Sdh155122 	tcp_stack_t	*tcps = tcp->tcp_tcps;
354741Smasputra 
355741Smasputra 	ASSERT(tcp->tcp_fused);
3569993SAnders.Persson@Sun.COM 	ASSERT(peer_tcp != NULL && peer_tcp->tcp_loopback_peer == tcp);
3579992SAnders.Persson@Sun.COM 	ASSERT(!IPCL_IS_NONSTR(tcp->tcp_connp));
358741Smasputra 	ASSERT(DB_TYPE(mp) == M_PROTO || DB_TYPE(mp) == M_PCPROTO);
359741Smasputra 	ASSERT(mp->b_cont != NULL && DB_TYPE(mp->b_cont) == M_DATA);
360741Smasputra 	ASSERT(MBLKL(mp) >= sizeof (*tei) && MBLKL(mp->b_cont) > 0);
361741Smasputra 
362741Smasputra 	/*
363741Smasputra 	 * Urgent data arrives in the form of T_EXDATA_REQ from above.
364741Smasputra 	 * Each occurence denotes a new urgent pointer.  For each new
365741Smasputra 	 * urgent pointer we signal (SIGURG) the receiving app to indicate
366741Smasputra 	 * that it needs to go into urgent mode.  This is similar to the
367741Smasputra 	 * urgent data handling in the regular tcp.  We don't need to keep
368741Smasputra 	 * track of where the urgent pointer is, because each T_EXDATA_REQ
369741Smasputra 	 * "advances" the urgent pointer for us.
370741Smasputra 	 *
371741Smasputra 	 * The actual urgent data carried by T_EXDATA_REQ is then prepended
372741Smasputra 	 * by a T_EXDATA_IND before being enqueued behind any existing data
373741Smasputra 	 * destined for the receiving app.  There is only a single urgent
374741Smasputra 	 * pointer (out-of-band mark) for a given tcp.  If the new urgent
375741Smasputra 	 * data arrives before the receiving app reads some existing urgent
376741Smasputra 	 * data, the previous marker is lost.  This behavior is emulated
377741Smasputra 	 * accordingly below, by removing any existing T_EXDATA_IND messages
378741Smasputra 	 * and essentially converting old urgent data into non-urgent.
379741Smasputra 	 */
380741Smasputra 	ASSERT(tcp->tcp_valid_bits & TCP_URG_VALID);
381741Smasputra 	/* Let sender get out of urgent mode */
382741Smasputra 	tcp->tcp_valid_bits &= ~TCP_URG_VALID;
383741Smasputra 
384741Smasputra 	/*
385741Smasputra 	 * This flag indicates that a signal needs to be sent up.
386741Smasputra 	 * This flag will only get cleared once SIGURG is delivered and
387741Smasputra 	 * is not affected by the tcp_fused flag -- delivery will still
388741Smasputra 	 * happen even after an endpoint is unfused, to handle the case
389741Smasputra 	 * where the sending endpoint immediately closes/unfuses after
390741Smasputra 	 * sending urgent data and the accept is not yet finished.
391741Smasputra 	 */
392741Smasputra 	peer_tcp->tcp_fused_sigurg = B_TRUE;
393741Smasputra 
394741Smasputra 	/* Reuse T_EXDATA_REQ mblk for T_EXDATA_IND */
395741Smasputra 	DB_TYPE(mp) = M_PROTO;
396741Smasputra 	tei = (struct T_exdata_ind *)mp->b_rptr;
397741Smasputra 	tei->PRIM_type = T_EXDATA_IND;
398741Smasputra 	tei->MORE_flag = 0;
399741Smasputra 	mp->b_wptr = (uchar_t *)&tei[1];
400741Smasputra 
4013448Sdh155122 	TCP_STAT(tcps, tcp_fusion_urg);
40211754SKacheong.Poon@Sun.COM 	TCPS_BUMP_MIB(tcps, tcpOutUrg);
403741Smasputra 
404741Smasputra 	head = peer_tcp->tcp_rcv_list;
405741Smasputra 	while (head != NULL) {
406741Smasputra 		/*
407741Smasputra 		 * Remove existing T_EXDATA_IND, keep the data which follows
408741Smasputra 		 * it and relink our list.  Note that we don't modify the
409741Smasputra 		 * tcp_rcv_last_tail since it never points to T_EXDATA_IND.
410741Smasputra 		 */
411741Smasputra 		if (DB_TYPE(head) != M_DATA) {
412741Smasputra 			mp1 = head;
413741Smasputra 
414741Smasputra 			ASSERT(DB_TYPE(mp1->b_cont) == M_DATA);
415741Smasputra 			head = mp1->b_cont;
416741Smasputra 			mp1->b_cont = NULL;
417741Smasputra 			head->b_next = mp1->b_next;
418741Smasputra 			mp1->b_next = NULL;
419741Smasputra 			if (prev_head != NULL)
420741Smasputra 				prev_head->b_next = head;
421741Smasputra 			if (peer_tcp->tcp_rcv_list == mp1)
422741Smasputra 				peer_tcp->tcp_rcv_list = head;
423741Smasputra 			if (peer_tcp->tcp_rcv_last_head == mp1)
424741Smasputra 				peer_tcp->tcp_rcv_last_head = head;
425741Smasputra 			freeb(mp1);
426741Smasputra 		}
427741Smasputra 		prev_head = head;
428741Smasputra 		head = head->b_next;
429741Smasputra 	}
430741Smasputra }
431741Smasputra 
432741Smasputra /*
433741Smasputra  * Fusion output routine, called by tcp_output() and tcp_wput_proto().
4343429Svi117747  * If we are modifying any member that can be changed outside the squeue,
4353429Svi117747  * like tcp_flow_stopped, we need to take tcp_non_sq_lock.
436741Smasputra  */
437741Smasputra boolean_t
tcp_fuse_output(tcp_t * tcp,mblk_t * mp,uint32_t send_size)438741Smasputra tcp_fuse_output(tcp_t *tcp, mblk_t *mp, uint32_t send_size)
439741Smasputra {
44011042SErik.Nordmark@Sun.COM 	conn_t		*connp = tcp->tcp_connp;
44111042SErik.Nordmark@Sun.COM 	tcp_t		*peer_tcp = tcp->tcp_loopback_peer;
44211042SErik.Nordmark@Sun.COM 	conn_t		*peer_connp = peer_tcp->tcp_connp;
44311042SErik.Nordmark@Sun.COM 	boolean_t	flow_stopped, peer_data_queued = B_FALSE;
44411042SErik.Nordmark@Sun.COM 	boolean_t	urgent = (DB_TYPE(mp) != M_DATA);
44511042SErik.Nordmark@Sun.COM 	boolean_t	push = B_TRUE;
44611042SErik.Nordmark@Sun.COM 	mblk_t		*mp1 = mp;
44711042SErik.Nordmark@Sun.COM 	uint_t		ip_hdr_len;
44811042SErik.Nordmark@Sun.COM 	uint32_t	recv_size = send_size;
4493448Sdh155122 	tcp_stack_t	*tcps = tcp->tcp_tcps;
4503448Sdh155122 	netstack_t	*ns = tcps->tcps_netstack;
4513448Sdh155122 	ip_stack_t	*ipst = ns->netstack_ip;
45211042SErik.Nordmark@Sun.COM 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
45311042SErik.Nordmark@Sun.COM 	iaflags_t	ixaflags = connp->conn_ixa->ixa_flags;
45411042SErik.Nordmark@Sun.COM 	boolean_t	do_ipsec, hooks_out, hooks_in, ipobs_enabled;
455741Smasputra 
456741Smasputra 	ASSERT(tcp->tcp_fused);
457741Smasputra 	ASSERT(peer_tcp != NULL && peer_tcp->tcp_loopback_peer == tcp);
45811042SErik.Nordmark@Sun.COM 	ASSERT(connp->conn_sqp == peer_connp->conn_sqp);
459741Smasputra 	ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_PROTO ||
460741Smasputra 	    DB_TYPE(mp) == M_PCPROTO);
461741Smasputra 
462741Smasputra 	if (send_size == 0) {
463741Smasputra 		freemsg(mp);
464741Smasputra 		return (B_TRUE);
465741Smasputra 	}
466741Smasputra 
467741Smasputra 	/*
468741Smasputra 	 * Handle urgent data; we either send up SIGURG to the peer now
469741Smasputra 	 * or do it later when we drain, in case the peer is detached
470741Smasputra 	 * or if we're short of memory for M_PCSIG mblk.
471741Smasputra 	 */
472741Smasputra 	if (urgent) {
473741Smasputra 		tcp_fuse_output_urg(tcp, mp);
4742958Sdr146992 
4752958Sdr146992 		mp1 = mp->b_cont;
4762958Sdr146992 	}
4772958Sdr146992 
47811042SErik.Nordmark@Sun.COM 	/*
47911042SErik.Nordmark@Sun.COM 	 * Check that we are still using an IRE_LOCAL or IRE_LOOPBACK before
48011042SErik.Nordmark@Sun.COM 	 * further processes.
48111042SErik.Nordmark@Sun.COM 	 */
48211042SErik.Nordmark@Sun.COM 	if (!ip_output_verify_local(connp->conn_ixa))
48311042SErik.Nordmark@Sun.COM 		goto unfuse;
48411042SErik.Nordmark@Sun.COM 
48511042SErik.Nordmark@Sun.COM 	/*
48611042SErik.Nordmark@Sun.COM 	 * Build IP and TCP header in case we have something that needs the
48711042SErik.Nordmark@Sun.COM 	 * headers. Those cases are:
48811042SErik.Nordmark@Sun.COM 	 * 1. IPsec
48911042SErik.Nordmark@Sun.COM 	 * 2. IPobs
49011042SErik.Nordmark@Sun.COM 	 * 3. FW_HOOKS
49111042SErik.Nordmark@Sun.COM 	 *
49211042SErik.Nordmark@Sun.COM 	 * If tcp_xmit_mp() fails to dupb() the message, unfuse the connection
49311042SErik.Nordmark@Sun.COM 	 * and back to regular path.
49411042SErik.Nordmark@Sun.COM 	 */
49511042SErik.Nordmark@Sun.COM 	if (ixaflags & IXAF_IS_IPV4) {
49611042SErik.Nordmark@Sun.COM 		do_ipsec = (ixaflags & IXAF_IPSEC_SECURE) ||
49711042SErik.Nordmark@Sun.COM 		    CONN_INBOUND_POLICY_PRESENT(peer_connp, ipss);
49811042SErik.Nordmark@Sun.COM 
49911042SErik.Nordmark@Sun.COM 		hooks_out = HOOKS4_INTERESTED_LOOPBACK_OUT(ipst);
50011042SErik.Nordmark@Sun.COM 		hooks_in = HOOKS4_INTERESTED_LOOPBACK_IN(ipst);
50111042SErik.Nordmark@Sun.COM 		ipobs_enabled = (ipst->ips_ip4_observe.he_interested != 0);
50211042SErik.Nordmark@Sun.COM 	} else {
50311042SErik.Nordmark@Sun.COM 		do_ipsec = (ixaflags & IXAF_IPSEC_SECURE) ||
50411042SErik.Nordmark@Sun.COM 		    CONN_INBOUND_POLICY_PRESENT_V6(peer_connp, ipss);
50511042SErik.Nordmark@Sun.COM 
50611042SErik.Nordmark@Sun.COM 		hooks_out = HOOKS6_INTERESTED_LOOPBACK_OUT(ipst);
50711042SErik.Nordmark@Sun.COM 		hooks_in = HOOKS6_INTERESTED_LOOPBACK_IN(ipst);
50811042SErik.Nordmark@Sun.COM 		ipobs_enabled = (ipst->ips_ip6_observe.he_interested != 0);
50911042SErik.Nordmark@Sun.COM 	}
51011042SErik.Nordmark@Sun.COM 
51111042SErik.Nordmark@Sun.COM 	/* We do logical 'or' for efficiency */
51211042SErik.Nordmark@Sun.COM 	if (ipobs_enabled | do_ipsec | hooks_in | hooks_out) {
5132958Sdr146992 		if ((mp1 = tcp_xmit_mp(tcp, mp1, tcp->tcp_mss, NULL, NULL,
5142958Sdr146992 		    tcp->tcp_snxt, B_TRUE, NULL, B_FALSE)) == NULL)
5152958Sdr146992 			/* If tcp_xmit_mp fails, use regular path */
5162958Sdr146992 			goto unfuse;
5172958Sdr146992 
5187828SBrian.Ruthven@Sun.COM 		/*
51911042SErik.Nordmark@Sun.COM 		 * Leave all IP relevant processes to ip_output_process_local(),
52011042SErik.Nordmark@Sun.COM 		 * which handles IPsec, IPobs, and FW_HOOKS.
5217828SBrian.Ruthven@Sun.COM 		 */
52211042SErik.Nordmark@Sun.COM 		mp1 = ip_output_process_local(mp1, connp->conn_ixa, hooks_out,
52311042SErik.Nordmark@Sun.COM 		    hooks_in, do_ipsec ? peer_connp : NULL);
5247828SBrian.Ruthven@Sun.COM 
52511042SErik.Nordmark@Sun.COM 		/* If the message is dropped for any reason. */
5262958Sdr146992 		if (mp1 == NULL)
5272958Sdr146992 			goto unfuse;
5282958Sdr146992 
5297828SBrian.Ruthven@Sun.COM 		/*
53011042SErik.Nordmark@Sun.COM 		 * Data length might have been changed by FW_HOOKS.
53111042SErik.Nordmark@Sun.COM 		 * We assume that the first mblk contains the TCP/IP headers.
5327828SBrian.Ruthven@Sun.COM 		 */
53311042SErik.Nordmark@Sun.COM 		if (hooks_in || hooks_out) {
53411042SErik.Nordmark@Sun.COM 			tcpha_t *tcpha;
5352958Sdr146992 
53611042SErik.Nordmark@Sun.COM 			ip_hdr_len = (ixaflags & IXAF_IS_IPV4) ?
53711042SErik.Nordmark@Sun.COM 			    IPH_HDR_LENGTH((ipha_t *)mp1->b_rptr) :
53811042SErik.Nordmark@Sun.COM 			    ip_hdr_length_v6(mp1, (ip6_t *)mp1->b_rptr);
5392958Sdr146992 
54011042SErik.Nordmark@Sun.COM 			tcpha = (tcpha_t *)&mp1->b_rptr[ip_hdr_len];
54111042SErik.Nordmark@Sun.COM 			ASSERT((uchar_t *)tcpha + sizeof (tcpha_t) <=
54211042SErik.Nordmark@Sun.COM 			    mp1->b_wptr);
54311042SErik.Nordmark@Sun.COM 			recv_size += htonl(tcpha->tha_seq) - tcp->tcp_snxt;
5442958Sdr146992 
5452958Sdr146992 		}
5462958Sdr146992 
5472958Sdr146992 		/*
5482958Sdr146992 		 * The message duplicated by tcp_xmit_mp is freed.
5492958Sdr146992 		 * Note: the original message passed in remains unchanged.
5502958Sdr146992 		 */
5512958Sdr146992 		freemsg(mp1);
552741Smasputra 	}
553741Smasputra 
554741Smasputra 	/*
555741Smasputra 	 * Enqueue data into the peer's receive list; we may or may not
556741Smasputra 	 * drain the contents depending on the conditions below.
5578682SAnders.Persson@Sun.COM 	 *
5589993SAnders.Persson@Sun.COM 	 * For non-STREAMS sockets we normally queue data directly in the
5599993SAnders.Persson@Sun.COM 	 * socket by calling the su_recv upcall. However, if the peer is
5609993SAnders.Persson@Sun.COM 	 * detached we use tcp_rcv_enqueue() instead. Queued data will be
5619993SAnders.Persson@Sun.COM 	 * drained when the accept completes (in tcp_accept_finish()).
562741Smasputra 	 */
56311042SErik.Nordmark@Sun.COM 	if (IPCL_IS_NONSTR(peer_connp) &&
5649993SAnders.Persson@Sun.COM 	    !TCP_IS_DETACHED(peer_tcp)) {
5658348SEric.Yu@Sun.COM 		int error;
5668348SEric.Yu@Sun.COM 		int flags = 0;
5678348SEric.Yu@Sun.COM 
5688348SEric.Yu@Sun.COM 		if ((tcp->tcp_valid_bits & TCP_URG_VALID) &&
5698348SEric.Yu@Sun.COM 		    (tcp->tcp_urg == tcp->tcp_snxt)) {
5708348SEric.Yu@Sun.COM 			flags = MSG_OOB;
57111042SErik.Nordmark@Sun.COM 			(*peer_connp->conn_upcalls->su_signal_oob)
57211042SErik.Nordmark@Sun.COM 			    (peer_connp->conn_upper_handle, 0);
5738348SEric.Yu@Sun.COM 			tcp->tcp_valid_bits &= ~TCP_URG_VALID;
5748348SEric.Yu@Sun.COM 		}
57511042SErik.Nordmark@Sun.COM 		if ((*peer_connp->conn_upcalls->su_recv)(
57611042SErik.Nordmark@Sun.COM 		    peer_connp->conn_upper_handle, mp, recv_size,
5779534SAnders.Persson@Sun.COM 		    flags, &error, &push) < 0) {
5789534SAnders.Persson@Sun.COM 			ASSERT(error != EOPNOTSUPP);
5799534SAnders.Persson@Sun.COM 			peer_data_queued = B_TRUE;
5809534SAnders.Persson@Sun.COM 		}
5818348SEric.Yu@Sun.COM 	} else {
58211042SErik.Nordmark@Sun.COM 		if (IPCL_IS_NONSTR(peer_connp) &&
5838348SEric.Yu@Sun.COM 		    (tcp->tcp_valid_bits & TCP_URG_VALID) &&
5848348SEric.Yu@Sun.COM 		    (tcp->tcp_urg == tcp->tcp_snxt)) {
5858348SEric.Yu@Sun.COM 			/*
5868348SEric.Yu@Sun.COM 			 * Can not deal with urgent pointers
5878348SEric.Yu@Sun.COM 			 * that arrive before the connection has been
5888348SEric.Yu@Sun.COM 			 * accept()ed.
5898348SEric.Yu@Sun.COM 			 */
5908348SEric.Yu@Sun.COM 			tcp->tcp_valid_bits &= ~TCP_URG_VALID;
5918348SEric.Yu@Sun.COM 			freemsg(mp);
5928348SEric.Yu@Sun.COM 			return (B_TRUE);
5938348SEric.Yu@Sun.COM 		}
5948348SEric.Yu@Sun.COM 
59511042SErik.Nordmark@Sun.COM 		tcp_rcv_enqueue(peer_tcp, mp, recv_size,
59611042SErik.Nordmark@Sun.COM 		    tcp->tcp_connp->conn_cred);
5979993SAnders.Persson@Sun.COM 
5989993SAnders.Persson@Sun.COM 		/* In case it wrapped around and also to keep it constant */
5999993SAnders.Persson@Sun.COM 		peer_tcp->tcp_rwnd += recv_size;
6008348SEric.Yu@Sun.COM 	}
601741Smasputra 
602741Smasputra 	/*
603741Smasputra 	 * Exercise flow-control when needed; we will get back-enabled
6049993SAnders.Persson@Sun.COM 	 * in either tcp_accept_finish(), tcp_unfuse(), or when data is
6059993SAnders.Persson@Sun.COM 	 * consumed. If peer endpoint is detached, we emulate streams flow
6069993SAnders.Persson@Sun.COM 	 * control by checking the peer's queue size and high water mark;
6079993SAnders.Persson@Sun.COM 	 * otherwise we simply use canputnext() to decide if we need to stop
6089993SAnders.Persson@Sun.COM 	 * our flow.
609741Smasputra 	 *
6103429Svi117747 	 * Since we are accessing our tcp_flow_stopped and might modify it,
6119993SAnders.Persson@Sun.COM 	 * we need to take tcp->tcp_non_sq_lock.
6123429Svi117747 	 */
6139993SAnders.Persson@Sun.COM 	mutex_enter(&tcp->tcp_non_sq_lock);
614741Smasputra 	flow_stopped = tcp->tcp_flow_stopped;
6159993SAnders.Persson@Sun.COM 	if ((TCP_IS_DETACHED(peer_tcp) &&
61611042SErik.Nordmark@Sun.COM 	    (peer_tcp->tcp_rcv_cnt >= peer_connp->conn_rcvbuf)) ||
6179993SAnders.Persson@Sun.COM 	    (!TCP_IS_DETACHED(peer_tcp) &&
61811042SErik.Nordmark@Sun.COM 	    !IPCL_IS_NONSTR(peer_connp) && !canputnext(peer_connp->conn_rq))) {
6194011Sudpa 		peer_data_queued = B_TRUE;
6204011Sudpa 	}
6214011Sudpa 
6224011Sudpa 	if (!flow_stopped && (peer_data_queued ||
62311042SErik.Nordmark@Sun.COM 	    (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf))) {
624741Smasputra 		tcp_setqfull(tcp);
625741Smasputra 		flow_stopped = B_TRUE;
6263448Sdh155122 		TCP_STAT(tcps, tcp_fusion_flowctl);
6279993SAnders.Persson@Sun.COM 		DTRACE_PROBE3(tcp__fuse__output__flowctl, tcp_t *, tcp,
6289993SAnders.Persson@Sun.COM 		    uint_t, send_size, uint_t, peer_tcp->tcp_rcv_cnt);
6294011Sudpa 	} else if (flow_stopped && !peer_data_queued &&
63011042SErik.Nordmark@Sun.COM 	    (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat)) {
631741Smasputra 		tcp_clrqfull(tcp);
6326970Sja97890 		TCP_STAT(tcps, tcp_fusion_backenabled);
6332578Smeem 		flow_stopped = B_FALSE;
634741Smasputra 	}
6353429Svi117747 	mutex_exit(&tcp->tcp_non_sq_lock);
6366970Sja97890 
6373448Sdh155122 	ipst->ips_loopback_packets++;
638741Smasputra 	tcp->tcp_last_sent_len = send_size;
639741Smasputra 
640741Smasputra 	/* Need to adjust the following SNMP MIB-related variables */
641741Smasputra 	tcp->tcp_snxt += send_size;
642741Smasputra 	tcp->tcp_suna = tcp->tcp_snxt;
6432958Sdr146992 	peer_tcp->tcp_rnxt += recv_size;
64412507SAlan.Maguire@Sun.COM 	peer_tcp->tcp_last_recv_len = recv_size;
645741Smasputra 	peer_tcp->tcp_rack = peer_tcp->tcp_rnxt;
646741Smasputra 
64711754SKacheong.Poon@Sun.COM 	TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
64811754SKacheong.Poon@Sun.COM 	TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, send_size);
649741Smasputra 
65011754SKacheong.Poon@Sun.COM 	TCPS_BUMP_MIB(tcps, tcpHCInSegs);
65111754SKacheong.Poon@Sun.COM 	TCPS_BUMP_MIB(tcps, tcpInDataInorderSegs);
65211754SKacheong.Poon@Sun.COM 	TCPS_UPDATE_MIB(tcps, tcpInDataInorderBytes, send_size);
653741Smasputra 
654741Smasputra 	BUMP_LOCAL(tcp->tcp_obsegs);
655741Smasputra 	BUMP_LOCAL(peer_tcp->tcp_ibsegs);
656741Smasputra 
65712507SAlan.Maguire@Sun.COM 	DTRACE_TCP5(send, void, NULL, ip_xmit_attr_t *, connp->conn_ixa,
65812507SAlan.Maguire@Sun.COM 	    __dtrace_tcp_void_ip_t *, NULL, tcp_t *, tcp,
65912507SAlan.Maguire@Sun.COM 	    __dtrace_tcp_tcph_t *, NULL);
66012507SAlan.Maguire@Sun.COM 	DTRACE_TCP5(receive, void, NULL, ip_xmit_attr_t *,
66112507SAlan.Maguire@Sun.COM 	    peer_connp->conn_ixa, __dtrace_tcp_void_ip_t *, NULL,
66212507SAlan.Maguire@Sun.COM 	    tcp_t *, peer_tcp, __dtrace_tcp_tcph_t *, NULL);
663741Smasputra 
6649992SAnders.Persson@Sun.COM 	if (!IPCL_IS_NONSTR(peer_tcp->tcp_connp) &&
6659992SAnders.Persson@Sun.COM 	    !TCP_IS_DETACHED(peer_tcp)) {
666741Smasputra 		/*
667741Smasputra 		 * Drain the peer's receive queue it has urgent data or if
6689993SAnders.Persson@Sun.COM 		 * we're not flow-controlled.
669741Smasputra 		 */
6709993SAnders.Persson@Sun.COM 		if (urgent || !flow_stopped) {
6719992SAnders.Persson@Sun.COM 			ASSERT(peer_tcp->tcp_rcv_list != NULL);
6722504Smeem 			/*
6732504Smeem 			 * For TLI-based streams, a thread in tcp_accept_swap()
6742504Smeem 			 * can race with us.  That thread will ensure that the
67511042SErik.Nordmark@Sun.COM 			 * correct peer_connp->conn_rq is globally visible
67611042SErik.Nordmark@Sun.COM 			 * before peer_tcp->tcp_detached is visible as clear,
67711042SErik.Nordmark@Sun.COM 			 * but we must also ensure that the load of conn_rq
67811042SErik.Nordmark@Sun.COM 			 * cannot be reordered to be before the tcp_detached
67911042SErik.Nordmark@Sun.COM 			 * check.
6802504Smeem 			 */
6812504Smeem 			membar_consumer();
68211042SErik.Nordmark@Sun.COM 			(void) tcp_fuse_rcv_drain(peer_connp->conn_rq, peer_tcp,
6832504Smeem 			    NULL);
684741Smasputra 		}
685741Smasputra 	}
686741Smasputra 	return (B_TRUE);
6872958Sdr146992 unfuse:
6882958Sdr146992 	tcp_unfuse(tcp);
6892958Sdr146992 	return (B_FALSE);
690741Smasputra }
691741Smasputra 
692741Smasputra /*
693741Smasputra  * This routine gets called to deliver data upstream on a fused or
694741Smasputra  * previously fused tcp loopback endpoint; the latter happens only
695741Smasputra  * when there is a pending SIGURG signal plus urgent data that can't
696741Smasputra  * be sent upstream in the past.
697741Smasputra  */
698741Smasputra boolean_t
tcp_fuse_rcv_drain(queue_t * q,tcp_t * tcp,mblk_t ** sigurg_mpp)699741Smasputra tcp_fuse_rcv_drain(queue_t *q, tcp_t *tcp, mblk_t **sigurg_mpp)
700741Smasputra {
701741Smasputra 	mblk_t *mp;
7028348SEric.Yu@Sun.COM 	conn_t	*connp = tcp->tcp_connp;
7038348SEric.Yu@Sun.COM 
704741Smasputra #ifdef DEBUG
705741Smasputra 	uint_t cnt = 0;
706741Smasputra #endif
7073448Sdh155122 	tcp_stack_t	*tcps = tcp->tcp_tcps;
7086970Sja97890 	tcp_t		*peer_tcp = tcp->tcp_loopback_peer;
709741Smasputra 
710741Smasputra 	ASSERT(tcp->tcp_loopback);
711741Smasputra 	ASSERT(tcp->tcp_fused || tcp->tcp_fused_sigurg);
712741Smasputra 	ASSERT(!tcp->tcp_fused || tcp->tcp_loopback_peer != NULL);
7138348SEric.Yu@Sun.COM 	ASSERT(IPCL_IS_NONSTR(connp) || sigurg_mpp != NULL || tcp->tcp_fused);
714741Smasputra 
715741Smasputra 	/* No need for the push timer now, in case it was scheduled */
716741Smasputra 	if (tcp->tcp_push_tid != 0) {
717741Smasputra 		(void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid);
718741Smasputra 		tcp->tcp_push_tid = 0;
719741Smasputra 	}
720741Smasputra 	/*
721741Smasputra 	 * If there's urgent data sitting in receive list and we didn't
722741Smasputra 	 * get a chance to send up a SIGURG signal, make sure we send
723741Smasputra 	 * it first before draining in order to ensure that SIOCATMARK
724741Smasputra 	 * works properly.
725741Smasputra 	 */
726741Smasputra 	if (tcp->tcp_fused_sigurg) {
7279992SAnders.Persson@Sun.COM 		ASSERT(!IPCL_IS_NONSTR(tcp->tcp_connp));
7289992SAnders.Persson@Sun.COM 
7298348SEric.Yu@Sun.COM 		tcp->tcp_fused_sigurg = B_FALSE;
7309992SAnders.Persson@Sun.COM 		/*
7319992SAnders.Persson@Sun.COM 		 * sigurg_mpp is normally NULL, i.e. when we're still
7329992SAnders.Persson@Sun.COM 		 * fused and didn't get here because of tcp_unfuse().
7339992SAnders.Persson@Sun.COM 		 * In this case try hard to allocate the M_PCSIG mblk.
7349992SAnders.Persson@Sun.COM 		 */
7359992SAnders.Persson@Sun.COM 		if (sigurg_mpp == NULL &&
7369992SAnders.Persson@Sun.COM 		    (mp = allocb(1, BPRI_HI)) == NULL &&
7379992SAnders.Persson@Sun.COM 		    (mp = allocb_tryhard(1)) == NULL) {
7389992SAnders.Persson@Sun.COM 			/* Alloc failed; try again next time */
7399992SAnders.Persson@Sun.COM 			tcp->tcp_push_tid = TCP_TIMER(tcp,
74012056SKacheong.Poon@Sun.COM 			    tcp_push_timer, tcps->tcps_push_timer_interval);
7419992SAnders.Persson@Sun.COM 			return (B_TRUE);
7429992SAnders.Persson@Sun.COM 		} else if (sigurg_mpp != NULL) {
743741Smasputra 			/*
7449992SAnders.Persson@Sun.COM 			 * Use the supplied M_PCSIG mblk; it means we're
7459992SAnders.Persson@Sun.COM 			 * either unfused or in the process of unfusing,
7469992SAnders.Persson@Sun.COM 			 * and the drain must happen now.
747741Smasputra 			 */
7489992SAnders.Persson@Sun.COM 			mp = *sigurg_mpp;
7499992SAnders.Persson@Sun.COM 			*sigurg_mpp = NULL;
7509992SAnders.Persson@Sun.COM 		}
7519992SAnders.Persson@Sun.COM 		ASSERT(mp != NULL);
7528348SEric.Yu@Sun.COM 
7539992SAnders.Persson@Sun.COM 		/* Send up the signal */
7549992SAnders.Persson@Sun.COM 		DB_TYPE(mp) = M_PCSIG;
7559992SAnders.Persson@Sun.COM 		*mp->b_wptr++ = (uchar_t)SIGURG;
7569992SAnders.Persson@Sun.COM 		putnext(q, mp);
7579992SAnders.Persson@Sun.COM 
758741Smasputra 		/*
759741Smasputra 		 * Let the regular tcp_rcv_drain() path handle
760741Smasputra 		 * draining the data if we're no longer fused.
761741Smasputra 		 */
762741Smasputra 		if (!tcp->tcp_fused)
763741Smasputra 			return (B_FALSE);
764741Smasputra 	}
765741Smasputra 
766741Smasputra 	/* Drain the data */
767741Smasputra 	while ((mp = tcp->tcp_rcv_list) != NULL) {
768741Smasputra 		tcp->tcp_rcv_list = mp->b_next;
769741Smasputra 		mp->b_next = NULL;
770741Smasputra #ifdef DEBUG
771741Smasputra 		cnt += msgdsize(mp);
772741Smasputra #endif
7738348SEric.Yu@Sun.COM 		ASSERT(!IPCL_IS_NONSTR(connp));
7749993SAnders.Persson@Sun.COM 		putnext(q, mp);
7759993SAnders.Persson@Sun.COM 		TCP_STAT(tcps, tcp_fusion_putnext);
776741Smasputra 	}
777741Smasputra 
7788348SEric.Yu@Sun.COM #ifdef DEBUG
779741Smasputra 	ASSERT(cnt == tcp->tcp_rcv_cnt);
7808348SEric.Yu@Sun.COM #endif
781741Smasputra 	tcp->tcp_rcv_last_head = NULL;
782741Smasputra 	tcp->tcp_rcv_last_tail = NULL;
783741Smasputra 	tcp->tcp_rcv_cnt = 0;
78411042SErik.Nordmark@Sun.COM 	tcp->tcp_rwnd = tcp->tcp_connp->conn_rcvbuf;
785741Smasputra 
7869993SAnders.Persson@Sun.COM 	mutex_enter(&peer_tcp->tcp_non_sq_lock);
7876970Sja97890 	if (peer_tcp->tcp_flow_stopped && (TCP_UNSENT_BYTES(peer_tcp) <=
78811042SErik.Nordmark@Sun.COM 	    peer_tcp->tcp_connp->conn_sndlowat)) {
7896970Sja97890 		tcp_clrqfull(peer_tcp);
7906970Sja97890 		TCP_STAT(tcps, tcp_fusion_backenabled);
7916970Sja97890 	}
7929993SAnders.Persson@Sun.COM 	mutex_exit(&peer_tcp->tcp_non_sq_lock);
7936970Sja97890 
794741Smasputra 	return (B_TRUE);
795741Smasputra }
796741Smasputra 
797741Smasputra /*
798741Smasputra  * Calculate the size of receive buffer for a fused tcp endpoint.
799741Smasputra  */
800741Smasputra size_t
tcp_fuse_set_rcv_hiwat(tcp_t * tcp,size_t rwnd)801741Smasputra tcp_fuse_set_rcv_hiwat(tcp_t *tcp, size_t rwnd)
802741Smasputra {
8033448Sdh155122 	tcp_stack_t	*tcps = tcp->tcp_tcps;
80411303SKacheong.Poon@Sun.COM 	uint32_t	max_win;
8053448Sdh155122 
806741Smasputra 	ASSERT(tcp->tcp_fused);
807741Smasputra 
808741Smasputra 	/* Ensure that value is within the maximum upper bound */
8093448Sdh155122 	if (rwnd > tcps->tcps_max_buf)
8103448Sdh155122 		rwnd = tcps->tcps_max_buf;
811741Smasputra 	/*
812741Smasputra 	 * Round up to system page size in case SO_RCVBUF is modified
813741Smasputra 	 * after SO_SNDBUF; the latter is also similarly rounded up.
814741Smasputra 	 */
815741Smasputra 	rwnd = P2ROUNDUP_TYPED(rwnd, PAGESIZE, size_t);
81611303SKacheong.Poon@Sun.COM 	max_win = TCP_MAXWIN << tcp->tcp_rcv_ws;
81711303SKacheong.Poon@Sun.COM 	if (rwnd > max_win) {
81811303SKacheong.Poon@Sun.COM 		rwnd = max_win - (max_win % tcp->tcp_mss);
81911303SKacheong.Poon@Sun.COM 		if (rwnd < tcp->tcp_mss)
82011303SKacheong.Poon@Sun.COM 			rwnd = max_win;
82111303SKacheong.Poon@Sun.COM 	}
82210312SRao.Shoaib@Sun.COM 
82310312SRao.Shoaib@Sun.COM 	/*
82410312SRao.Shoaib@Sun.COM 	 * Record high water mark, this is used for flow-control
82510312SRao.Shoaib@Sun.COM 	 * purposes in tcp_fuse_output().
82610312SRao.Shoaib@Sun.COM 	 */
82711042SErik.Nordmark@Sun.COM 	tcp->tcp_connp->conn_rcvbuf = rwnd;
82811042SErik.Nordmark@Sun.COM 	tcp->tcp_rwnd = rwnd;
829741Smasputra 	return (rwnd);
830741Smasputra }
831741Smasputra 
832741Smasputra /*
833741Smasputra  * Calculate the maximum outstanding unread data block for a fused tcp endpoint.
834741Smasputra  */
835741Smasputra int
tcp_fuse_maxpsz(tcp_t * tcp)83610312SRao.Shoaib@Sun.COM tcp_fuse_maxpsz(tcp_t *tcp)
837741Smasputra {
838741Smasputra 	tcp_t *peer_tcp = tcp->tcp_loopback_peer;
83911042SErik.Nordmark@Sun.COM 	conn_t *connp = tcp->tcp_connp;
84011042SErik.Nordmark@Sun.COM 	uint_t sndbuf = connp->conn_sndbuf;
841741Smasputra 	uint_t maxpsz = sndbuf;
842741Smasputra 
843741Smasputra 	ASSERT(tcp->tcp_fused);
844741Smasputra 	ASSERT(peer_tcp != NULL);
84511042SErik.Nordmark@Sun.COM 	ASSERT(peer_tcp->tcp_connp->conn_rcvbuf != 0);
846741Smasputra 	/*
847741Smasputra 	 * In the fused loopback case, we want the stream head to split
848741Smasputra 	 * up larger writes into smaller chunks for a more accurate flow-
849741Smasputra 	 * control accounting.  Our maxpsz is half of the sender's send
850741Smasputra 	 * buffer or the receiver's receive buffer, whichever is smaller.
851741Smasputra 	 * We round up the buffer to system page size due to the lack of
852741Smasputra 	 * TCP MSS concept in Fusion.
853741Smasputra 	 */
85411042SErik.Nordmark@Sun.COM 	if (maxpsz > peer_tcp->tcp_connp->conn_rcvbuf)
85511042SErik.Nordmark@Sun.COM 		maxpsz = peer_tcp->tcp_connp->conn_rcvbuf;
856741Smasputra 	maxpsz = P2ROUNDUP_TYPED(maxpsz, PAGESIZE, uint_t) >> 1;
857741Smasputra 
858741Smasputra 	return (maxpsz);
859741Smasputra }
8609534SAnders.Persson@Sun.COM 
8619534SAnders.Persson@Sun.COM /*
8629534SAnders.Persson@Sun.COM  * Called to release flow control.
8639534SAnders.Persson@Sun.COM  */
8649534SAnders.Persson@Sun.COM void
tcp_fuse_backenable(tcp_t * tcp)8659534SAnders.Persson@Sun.COM tcp_fuse_backenable(tcp_t *tcp)
8669534SAnders.Persson@Sun.COM {
8679534SAnders.Persson@Sun.COM 	tcp_t *peer_tcp = tcp->tcp_loopback_peer;
8689534SAnders.Persson@Sun.COM 
8699534SAnders.Persson@Sun.COM 	ASSERT(tcp->tcp_fused);
8709534SAnders.Persson@Sun.COM 	ASSERT(peer_tcp != NULL && peer_tcp->tcp_fused);
8719534SAnders.Persson@Sun.COM 	ASSERT(peer_tcp->tcp_loopback_peer == tcp);
8729534SAnders.Persson@Sun.COM 	ASSERT(!TCP_IS_DETACHED(tcp));
8739534SAnders.Persson@Sun.COM 	ASSERT(tcp->tcp_connp->conn_sqp ==
8749534SAnders.Persson@Sun.COM 	    peer_tcp->tcp_connp->conn_sqp);
8759534SAnders.Persson@Sun.COM 
8769534SAnders.Persson@Sun.COM 	if (tcp->tcp_rcv_list != NULL)
87711042SErik.Nordmark@Sun.COM 		(void) tcp_fuse_rcv_drain(tcp->tcp_connp->conn_rq, tcp, NULL);
8789534SAnders.Persson@Sun.COM 
8799993SAnders.Persson@Sun.COM 	mutex_enter(&peer_tcp->tcp_non_sq_lock);
8809534SAnders.Persson@Sun.COM 	if (peer_tcp->tcp_flow_stopped &&
8819534SAnders.Persson@Sun.COM 	    (TCP_UNSENT_BYTES(peer_tcp) <=
88211042SErik.Nordmark@Sun.COM 	    peer_tcp->tcp_connp->conn_sndlowat)) {
8839534SAnders.Persson@Sun.COM 		tcp_clrqfull(peer_tcp);
8849534SAnders.Persson@Sun.COM 	}
8859534SAnders.Persson@Sun.COM 	mutex_exit(&peer_tcp->tcp_non_sq_lock);
8869534SAnders.Persson@Sun.COM 
8879534SAnders.Persson@Sun.COM 	TCP_STAT(tcp->tcp_tcps, tcp_fusion_backenabled);
8889534SAnders.Persson@Sun.COM }
889