xref: /onnv-gate/usr/src/cmd/cmd-inet/usr.lib/in.mpathd/mpd_probe.c (revision 8700:54ffe244faa6)

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 1987 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms are permitted
 * provided that the above copyright notice and this paragraph are
 * duplicated in all such forms and that any documentation,
 * advertising materials, and other materials related to such
 * distribution and use acknowledge that the software was developed
 * by the University of California, Berkeley. The name of the
 * University may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include "mpd_defs.h"
#include "mpd_tables.h"

/*
 * Probe types for probe()
 */
#define	PROBE_UNI	0x1234		/* Unicast probe packet */
#define	PROBE_MULTI	0x5678		/* Multicast probe packet */
#define	PROBE_RTT	0x9abc		/* RTT only probe packet */

#define	MSEC_PERMIN	(60 * MILLISEC)	/* Number of milliseconds in a minute */

/*
 * Format of probe / probe response packets. This is an ICMP Echo request
 * or ICMP Echo reply. Packet format is same for both IPv4 and IPv6
 */
struct pr_icmp
{
	uint8_t  pr_icmp_type;		/* type field */
	uint8_t  pr_icmp_code;		/* code field */
	uint16_t pr_icmp_cksum;		/* checksum field */
	uint16_t pr_icmp_id;		/* Identification */
	uint16_t pr_icmp_seq;		/* sequence number */
	uint64_t pr_icmp_timestamp;	/* Time stamp (in ns) */
	uint32_t pr_icmp_mtype;		/* Message type */
};

static struct in6_addr all_nodes_mcast_v6 = { { 0xff, 0x2, 0x0, 0x0,
				    0x0, 0x0, 0x0, 0x0,
				    0x0, 0x0, 0x0, 0x0,
				    0x0, 0x0, 0x0, 0x1 } };

static struct in_addr all_nodes_mcast_v4 = { { { 0xe0, 0x0, 0x0, 0x1 } } };

static hrtime_t	last_fdt_bumpup_time;	/* When FDT was bumped up last */

static void		*find_ancillary(struct msghdr *msg, int cmsg_level,
    int cmsg_type);
static void		pi_set_crtt(struct target *tg, int64_t m,
    boolean_t is_probe_uni);
static void		incoming_echo_reply(struct phyint_instance *pii,
    struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp);
static void		incoming_rtt_reply(struct phyint_instance *pii,
    struct pr_icmp *reply, struct in6_addr fromaddr);
static void		incoming_mcast_reply(struct phyint_instance *pii,
    struct pr_icmp *reply, struct in6_addr fromaddr);

static boolean_t	check_pg_crtt_improved(struct phyint_group *pg);
static boolean_t	check_pii_crtt_improved(struct phyint_instance *pii);
static boolean_t	check_exception_target(struct phyint_instance *pii,
    struct target *target);
static void		probe_fail_info(struct phyint_instance *pii,
    struct target *cur_tg, struct probe_fail_count *pfinfo);
static void		probe_success_info(struct phyint_instance *pii,
    struct target *cur_tg, struct probe_success_count *psinfo);
static boolean_t	phyint_repaired(struct phyint *pi);

static boolean_t	highest_ack_tg(uint16_t seq, struct target *tg);
static int 		in_cksum(ushort_t *addr, int len);
static void		reset_snxt_basetimes(void);
static int		ns2ms(int64_t ns);
static int64_t		tv2ns(struct timeval *);

/*
 * CRTT - Conservative Round Trip Time Estimate
 * Probe success - A matching probe reply received before CRTT ms has elapsed
 *	after sending the probe.
 * Probe failure - No probe reply received and more than CRTT ms has elapsed
 *	after sending the probe.
 *
 * TLS - Time last success. Most recent probe ack received at this time.
 * TFF - Time first fail. The time of the earliest probe failure in
 *	a consecutive series of probe failures.
 * NUM_PROBE_REPAIRS  - Number of consecutive successful probes required
 * 	before declaring phyint repair.
 * NUM_PROBE_FAILS - Number of consecutive probe failures required to
 *	declare a phyint failure.
 *
 * 			Phyint state diagram
 *
 * The state of a phyint that is capable of being probed, is completely
 * specified by the 3-tuple <pi_state, pg_state, I>.
 *
 * A phyint starts in either PI_RUNNING or PI_FAILED, depending on the state
 * of the link (according to the driver).  If the phyint is also configured
 * with a test address (the common case) and probe targets, then a phyint must
 * also successfully be able to send and receive probes in order to remain in
 * the PI_RUNNING state (otherwise, it transitions to PI_FAILED).
 *
 * Further, if a PI_RUNNING phyint is configured with a test address but is
 * unable to find any probe targets, it will transition to the PI_NOTARGETS
 * state, which indicates that the link is apparently functional but that
 * in.mpathd is unable to send probes to verify functionality (in this case,
 * in.mpathd makes the optimistic assumption that the interface is working
 * correctly and thus does not mark the interface FAILED, but reports it as
 * IPMP_IF_UNKNOWN through the async events and query interfaces).
 *
 * At any point, a phyint may be administratively marked offline via if_mpadm.
 * In this case, the interface always transitions to PI_OFFLINE, regardless
 * of its previous state.  When the interface is later brought back online,
 * in.mpathd acts as if the interface is new (and thus it transitions to
 * PI_RUNNING or PI_FAILED based on the status of the link and the result of
 * its probes, if probes are sent).
 *
 * pi_state -  PI_RUNNING or PI_FAILED
 *	PI_RUNNING: The failure detection logic says the phyint is good.
 *	PI_FAILED: The failure detection logic says the phyint has failed.
 *
 * pg_state  - PG_OK, PG_DEGRADED, or PG_FAILED.
 *	PG_OK: All interfaces in the group are OK.
 *	PG_DEGRADED: Some interfaces in the group are unusable.
 *	PG_FAILED: All interfaces in the group are unusable.
 *
 *	In the case of router targets, we assume that the current list of
 *	targets obtained from the routing table, is still valid, so the
 *	phyint stat is PI_FAILED. In the case of host targets, we delete the
 *	list of targets, and multicast to the all hosts, to reconstruct the
 *	target list. So the phyints are in the PI_NOTARGETS state.
 *
 * I -	value of (pi_flags & IFF_INACTIVE)
 *	IFF_INACTIVE: This phyint will not send or receive packets.
 *	Usually, inactive is tied to standby interfaces that are not yet
 *	needed (e.g., no non-standby interfaces in the group have failed).
 *	When failback has been disabled (FAILBACK=no configured), phyint can
 *	also be a non-STANDBY. In this case IFF_INACTIVE is set when phyint
 *	subsequently recovers after a failure.
 *
 * Not all 9 possible combinations of the above 3-tuple are possible.
 *
 * I is tracked by IP. pi_state is tracked by mpathd.
 *
 *			pi_state state machine
 * ---------------------------------------------------------------------------
 *	Event			State			New State
 *				Action:
 * ---------------------------------------------------------------------------
 *	IP interface failure	(PI_RUNNING, I == 0) -> (PI_FAILED, I == 0)
 *	detection		: set IFF_FAILED on this phyint
 *
 *	IP interface failure	(PI_RUNNING, I == 1) -> (PI_FAILED, I == 0)
 *	detection		: set IFF_FAILED on this phyint
 *
 *	IP interface repair 	(PI_FAILED, I == 0, FAILBACK=yes)
 *	detection				     -> (PI_RUNNING, I == 0)
 *				: clear IFF_FAILED on this phyint
 *
 *	IP interface repair 	(PI_FAILED, I == 0, FAILBACK=no)
 *	detection				     ->	(PI_RUNNING, I == 1)
 *				: clear IFF_FAILED on this phyint
 *				: if failback is disabled set I == 1
 *
 *	Group failure		(perform on all phyints in the group)
 *	detection 		PI_RUNNING		PI_FAILED
 *	(Router targets)	: set IFF_FAILED
 *
 *	Group failure		(perform on all phyints in the group)
 *	detection 		PI_RUNNING		PI_NOTARGETS
 *	(Host targets)		: set IFF_FAILED
 *				: delete the target list on all phyints
 * ---------------------------------------------------------------------------
 */

struct probes_missed probes_missed;

/*
 * Compose and transmit an ICMP ECHO REQUEST packet.  The IP header
 * will be added on by the kernel.  The id field identifies this phyint.
 * and the sequence number is an increasing (modulo 2^^16) integer. The data
 * portion holds the time value when the packet is sent. On echo this is
 * extracted to compute the round-trip time. Three different types of
 * probe packets are used.
 *
 * PROBE_UNI: This type is used to do failure detection / failure recovery
 *	and RTT calculation. PROBE_UNI probes are spaced apart in time,
 *	not less than the current CRTT. pii_probes[] stores data
 *	about these probes. These packets consume sequence number space.
 *
 * PROBE_RTT: This type is used to make only rtt measurements. Normally these
 * 	are not used. Under heavy network load, the rtt may go up very high,
 *	due to a spike, or may appear to go high, due to extreme scheduling
 * 	delays. Once the network stress is removed, mpathd takes long time to
 *	recover, because the probe_interval is already high, and it takes
 *	a long time to send out sufficient number of probes to bring down the
 *	rtt. To avoid this problem, PROBE_RTT probes are sent out every
 *	user_probe_interval ms. and will cause only rtt updates. These packets
 *	do not consume sequence number space nor is information about these
 *	packets stored in the pii_probes[]
 *
 * PROBE_MULTI: This type is only used to construct a list of targets, when
 *	no targets are known. The packet is multicast to the all hosts addr.
 */
static void
probe(struct phyint_instance *pii, uint_t probe_type, hrtime_t start_hrtime)
{
	hrtime_t sent_hrtime;
	struct timeval sent_tv;
	struct pr_icmp probe_pkt;	/* Probe packet */
	struct sockaddr_storage targ;	/* target address */
	uint_t	targaddrlen;		/* targed address length */
	int	pr_ndx;			/* probe index in pii->pii_probes[] */
	boolean_t sent = _B_TRUE;

	if (debug & D_TARGET) {
		logdebug("probe(%s %s %d %lld)\n", AF_STR(pii->pii_af),
		    pii->pii_name, probe_type, start_hrtime);
	}

	assert(pii->pii_probe_sock != -1);
	assert(probe_type == PROBE_UNI || probe_type == PROBE_MULTI ||
	    probe_type == PROBE_RTT);

	probe_pkt.pr_icmp_type = (pii->pii_af == AF_INET) ?
	    ICMP_ECHO_REQUEST : ICMP6_ECHO_REQUEST;
	probe_pkt.pr_icmp_code = 0;
	probe_pkt.pr_icmp_cksum = 0;
	probe_pkt.pr_icmp_seq = htons(pii->pii_snxt);

	/*
	 * Since there is no need to do arithmetic on the icmpid,
	 * (only equality check is done) pii_icmpid is stored in
	 * network byte order at initialization itself.
	 */
	probe_pkt.pr_icmp_id = pii->pii_icmpid;
	probe_pkt.pr_icmp_timestamp = htonll(start_hrtime);
	probe_pkt.pr_icmp_mtype = htonl(probe_type);

	/*
	 * If probe_type is PROBE_MULTI, this packet will be multicast to
	 * the all hosts address. Otherwise it is unicast to the next target.
	 */
	assert(probe_type == PROBE_MULTI || ((pii->pii_target_next != NULL) &&
	    pii->pii_rtt_target_next != NULL));

	bzero(&targ, sizeof (targ));
	targ.ss_family = pii->pii_af;

	if (pii->pii_af == AF_INET6) {
		struct in6_addr *addr6;

		addr6 = &((struct sockaddr_in6 *)&targ)->sin6_addr;
		targaddrlen = sizeof (struct sockaddr_in6);
		if (probe_type == PROBE_MULTI) {
			*addr6 = all_nodes_mcast_v6;
		} else if (probe_type == PROBE_UNI) {
			*addr6 = pii->pii_target_next->tg_address;
		} else { /* type is PROBE_RTT */
			*addr6 = pii->pii_rtt_target_next->tg_address;
		}
	} else {
		struct in_addr *addr4;

		addr4 = &((struct sockaddr_in *)&targ)->sin_addr;
		targaddrlen = sizeof (struct sockaddr_in);
		if (probe_type == PROBE_MULTI) {
			*addr4 = all_nodes_mcast_v4;
		} else if (probe_type == PROBE_UNI) {
			IN6_V4MAPPED_TO_INADDR(
			    &pii->pii_target_next->tg_address, addr4);
		} else { /* type is PROBE_RTT */
			IN6_V4MAPPED_TO_INADDR(
			    &pii->pii_rtt_target_next->tg_address, addr4);
		}

		/*
		 * Compute the IPv4 icmp checksum. Does not cover the IP header.
		 */
		probe_pkt.pr_icmp_cksum =
		    in_cksum((ushort_t *)&probe_pkt, (int)sizeof (probe_pkt));
	}

	/*
	 * Use the current time as the time we sent.  Not atomic, but the best
	 * we can do from here.
	 */
	sent_hrtime = gethrtime();
	(void) gettimeofday(&sent_tv, NULL);
	if (sendto(pii->pii_probe_sock, &probe_pkt, sizeof (probe_pkt), 0,
	    (struct sockaddr *)&targ, targaddrlen) != sizeof (probe_pkt)) {
		logperror_pii(pii, "probe: probe sendto");
		sent = _B_FALSE;
	}

	/*
	 * If this is a PROBE_UNI probe packet being unicast to a target, then
	 * update our tables. We will need this info in processing the probe
	 * response. PROBE_MULTI and PROBE_RTT packets are not used for
	 * the purpose of failure or recovery detection. PROBE_MULTI packets
	 * are only used to construct a list of targets. PROBE_RTT packets are
	 * used only for updating the rtt and not for failure detection.
	 */
	if (probe_type == PROBE_UNI && sent) {
		pr_ndx = pii->pii_probe_next;
		assert(pr_ndx >= 0 && pr_ndx < PROBE_STATS_COUNT);

		/* Collect statistics, before we reuse the last slot. */
		if (pii->pii_probes[pr_ndx].pr_status == PR_LOST)
			pii->pii_cum_stats.lost++;
		else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED)
			pii->pii_cum_stats.acked++;
		pii->pii_cum_stats.sent++;

		pii->pii_probes[pr_ndx].pr_id = pii->pii_snxt;
		pii->pii_probes[pr_ndx].pr_tv_sent = sent_tv;
		pii->pii_probes[pr_ndx].pr_hrtime_sent = sent_hrtime;
		pii->pii_probes[pr_ndx].pr_hrtime_start = start_hrtime;
		pii->pii_probes[pr_ndx].pr_target = pii->pii_target_next;
		probe_chstate(&pii->pii_probes[pr_ndx], pii, PR_UNACKED);

		pii->pii_probe_next = PROBE_INDEX_NEXT(pii->pii_probe_next);
		pii->pii_target_next = target_next(pii->pii_target_next);
		assert(pii->pii_target_next != NULL);
		/*
		 * If we have a single variable to denote the next target to
		 * probe for both rtt probes and failure detection probes, we
		 * could end up with a situation where the failure detection
		 * probe targets become disjoint from the rtt probe targets.
		 * Eg. if 2 targets and the actual fdt is double the user
		 * specified fdt. So we have 2 variables. In this scheme
		 * we also reset pii_rtt_target_next for every fdt probe,
		 * though that may not be necessary.
		 */
		pii->pii_rtt_target_next = pii->pii_target_next;
		pii->pii_snxt++;
	} else if (probe_type == PROBE_RTT) {
		pii->pii_rtt_target_next =
		    target_next(pii->pii_rtt_target_next);
		assert(pii->pii_rtt_target_next != NULL);
	}
}

/*
 * Incoming IPv4 data from wire, is received here. Called from main.
 */
void
in_data(struct phyint_instance *pii)
{
	struct	sockaddr_in 	from;
	struct	in6_addr	fromaddr;
	static uint64_t in_packet[(IP_MAXPACKET + 1)/8];
	static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8];
	struct ip *ip;
	int 	iphlen;
	int 	len;
	char 	abuf[INET_ADDRSTRLEN];
	struct msghdr msg;
	struct iovec iov;
	struct pr_icmp *reply;
	struct timeval *recv_tvp;

	if (debug & D_PROBE) {
		logdebug("in_data(%s %s)\n",
		    AF_STR(pii->pii_af), pii->pii_name);
	}

	iov.iov_base = (char *)in_packet;
	iov.iov_len = sizeof (in_packet);
	msg.msg_iov = &iov;
	msg.msg_iovlen = 1;
	msg.msg_name = (struct sockaddr *)&from;
	msg.msg_namelen = sizeof (from);
	msg.msg_control = ancillary_data;
	msg.msg_controllen = sizeof (ancillary_data);

	/*
	 * Poll has already told us that a message is waiting,
	 * on this socket. Read it now. We should not block.
	 */
	if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) {
		logperror_pii(pii, "in_data: recvmsg");
		return;
	}

	/*
	 * If the datalink has indicated the link is down, don't go
	 * any further.
	 */
	if (LINK_DOWN(pii->pii_phyint))
		return;

	/* Get the printable address for error reporting */
	(void) inet_ntop(AF_INET, &from.sin_addr, abuf, sizeof (abuf));

	/* Ignore packets > 64k or control buffers that don't fit */
	if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
		if (debug & D_PKTBAD) {
			logdebug("Truncated message: msg_flags 0x%x from %s\n",
			    msg.msg_flags, abuf);
		}
		return;
	}

	/* Make sure packet contains at least minimum ICMP header */
	ip = (struct ip *)in_packet;
	iphlen = ip->ip_hl << 2;
	if (len < iphlen + ICMP_MINLEN) {
		if (debug & D_PKTBAD) {
			logdebug("in_data: packet too short (%d bytes)"
			    " from %s\n", len, abuf);
		}
		return;
	}

	/*
	 * Subtract the IP hdr length, 'len' will be length of the probe
	 * reply, starting from the icmp hdr.
	 */
	len -= iphlen;
	/* LINTED */
	reply = (struct pr_icmp *)((char *)in_packet + iphlen);

	/* Probe replies are icmp echo replies. Ignore anything else */
	if (reply->pr_icmp_type != ICMP_ECHO_REPLY)
		return;

	/*
	 * The icmp id should match what we sent, which is stored
	 * in pi_icmpid. The icmp code for reply must be 0.
	 * The reply content must be a struct pr_icmp
	 */
	if (reply->pr_icmp_id != pii->pii_icmpid) {
		/* Not in response to our probe */
		return;
	}

	if (reply->pr_icmp_code != 0) {
		logtrace("probe reply code %d from %s on %s\n",
		    reply->pr_icmp_code, abuf, pii->pii_name);
		return;
	}

	if (len < sizeof (struct pr_icmp)) {
		logtrace("probe reply too short: %d bytes from %s on %s\n",
		    len, abuf, pii->pii_name);
		return;
	}

	recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP);
	if (recv_tvp == NULL) {
		logtrace("message without timestamp from %s on %s\n",
		    abuf, pii->pii_name);
		return;
	}

	IN6_INADDR_TO_V4MAPPED(&from.sin_addr, &fromaddr);
	if (reply->pr_icmp_mtype == htonl(PROBE_UNI))
		/* Unicast probe reply */
		incoming_echo_reply(pii, reply, fromaddr, recv_tvp);
	else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) {
		/* Multicast reply */
		incoming_mcast_reply(pii, reply, fromaddr);
	} else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) {
		incoming_rtt_reply(pii, reply, fromaddr);
	} else {
		/* Probably not in response to our probe */
		logtrace("probe reply type: %d from %s on %s\n",
		    reply->pr_icmp_mtype, abuf, pii->pii_name);
		return;
	}
}

/*
 * Incoming IPv6 data from wire is received here. Called from main.
 */
void
in6_data(struct phyint_instance *pii)
{
	struct sockaddr_in6 from;
	static uint64_t in_packet[(IP_MAXPACKET + 1)/8];
	static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8];
	int len;
	char abuf[INET6_ADDRSTRLEN];
	struct msghdr msg;
	struct iovec iov;
	void	*opt;
	struct	pr_icmp *reply;
	struct	timeval *recv_tvp;

	if (debug & D_PROBE) {
		logdebug("in6_data(%s %s)\n",
		    AF_STR(pii->pii_af), pii->pii_name);
	}

	iov.iov_base = (char *)in_packet;
	iov.iov_len = sizeof (in_packet);
	msg.msg_iov = &iov;
	msg.msg_iovlen = 1;
	msg.msg_name = (struct sockaddr *)&from;
	msg.msg_namelen = sizeof (from);
	msg.msg_control = ancillary_data;
	msg.msg_controllen = sizeof (ancillary_data);

	if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) {
		logperror_pii(pii, "in6_data: recvmsg");
		return;
	}

	/*
	 * If the datalink has indicated that the link is down, don't go
	 * any further.
	 */
	if (LINK_DOWN(pii->pii_phyint))
		return;

	/* Get the printable address for error reporting */
	(void) inet_ntop(AF_INET6, &from.sin6_addr, abuf, sizeof (abuf));
	if (len < ICMP_MINLEN) {
		if (debug & D_PKTBAD) {
			logdebug("Truncated message: msg_flags 0x%x from %s\n",
			    msg.msg_flags, abuf);
		}
		return;
	}
	/* Ignore packets > 64k or control buffers that don't fit */
	if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
		if (debug & D_PKTBAD) {
			logdebug("Truncated message: msg_flags 0x%x from %s\n",
			    msg.msg_flags, abuf);
		}
		return;
	}

	reply = (struct pr_icmp *)in_packet;
	if (reply->pr_icmp_type != ICMP6_ECHO_REPLY)
		return;

	if (reply->pr_icmp_id != pii->pii_icmpid) {
		/* Not in response to our probe */
		return;
	}

	/*
	 * The kernel has already verified the the ICMP checksum.
	 */
	if (!IN6_IS_ADDR_LINKLOCAL(&from.sin6_addr)) {
		logtrace("ICMPv6 echo reply source address not linklocal from "
		    "%s on %s\n", abuf, pii->pii_name);
		return;
	}
	opt = find_ancillary(&msg, IPPROTO_IPV6, IPV6_RTHDR);
	if (opt != NULL) {
		/* Can't allow routing headers in probe replies  */
		logtrace("message with routing header from %s on %s\n",
		    abuf, pii->pii_name);
		return;
	}

	if (reply->pr_icmp_code != 0) {
		logtrace("probe reply code: %d from %s on %s\n",
		    reply->pr_icmp_code, abuf, pii->pii_name);
		return;
	}
	if (len < (sizeof (struct pr_icmp))) {
		logtrace("probe reply too short: %d bytes from %s on %s\n",
		    len, abuf, pii->pii_name);
		return;
	}

	recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP);
	if (recv_tvp == NULL) {
		logtrace("message without timestamp from %s on %s\n",
		    abuf, pii->pii_name);
		return;
	}

	if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) {
		incoming_echo_reply(pii, reply, from.sin6_addr, recv_tvp);
	} else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) {
		incoming_mcast_reply(pii, reply, from.sin6_addr);
	} else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) {
		incoming_rtt_reply(pii, reply, from.sin6_addr);
	} else  {
		/* Probably not in response to our probe */
		logtrace("probe reply type: %d from %s on %s\n",
		    reply->pr_icmp_mtype, abuf, pii->pii_name);
	}
}

/*
 * Process the incoming rtt reply, in response to our rtt probe.
 * Common for both IPv4 and IPv6. Unlike incoming_echo_reply() we don't
 * have any stored information about the probe we sent. So we don't log
 * any errors if we receive bad replies.
 */
static void
incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply,
    struct in6_addr fromaddr)
{
	int64_t	m;		/* rtt measurement in ns */
	char	abuf[INET6_ADDRSTRLEN];
	struct	target	*target;
	struct 	phyint_group *pg;

	/* Get the printable address for error reporting */
	(void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf));

	if (debug & D_PROBE) {
		logdebug("incoming_rtt_reply: %s %s %s\n",
		    AF_STR(pii->pii_af), pii->pii_name, abuf);
	}

	/* Do we know this target ? */
	target = target_lookup(pii, fromaddr);
	if (target == NULL)
		return;

	m = (int64_t)(gethrtime() - ntohll(reply->pr_icmp_timestamp));
	/* Invalid rtt. It has wrapped around */
	if (m < 0)
		return;

	/*
	 * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses
	 * The initial few responses after the interface is repaired may
	 * contain high rtt's because they could have been queued up waiting
	 * for ARP/NDP resolution on a failed interface.
	 */
	pg = pii->pii_phyint->pi_group;
	if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg))
		return;

	/*
	 * Update rtt only if the new rtt is lower than the current rtt.
	 * (specified by the 3rd parameter to pi_set_crtt).
	 * If a spike has caused the current probe_interval to be >
	 * user_probe_interval, then this mechanism is used to bring down
	 * the rtt rapidly once the network stress is removed.
	 * If the new rtt is higher than the current rtt, we don't want to
	 * update the rtt. We are having more than 1 outstanding probe and
	 * the increase in rtt we are seeing is being unnecessarily weighted
	 * many times. The regular rtt update will be handled by
	 * incoming_echo_reply() and will take care of any rtt increase.
	 */
	pi_set_crtt(target, m, _B_FALSE);
	if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) &&
	    (user_failure_detection_time < pg->pg_fdt) &&
	    (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) {
		/*
		 * If the crtt has now dropped by a factor of LOWER_FT_TRIGGER,
		 * investigate if we can improve the failure detection time to
		 * meet whatever the user specified.
		 */
		if (check_pg_crtt_improved(pg)) {
			pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE,
			    user_failure_detection_time);
			pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2);
			if (pii->pii_phyint->pi_group != phyint_anongroup) {
				logerr("Improved failure detection time %d ms "
				    "on (%s %s) for group \"%s\"\n",
				    pg->pg_fdt, AF_STR(pii->pii_af),
				    pii->pii_name,
				    pii->pii_phyint->pi_group->pg_name);
			}
			if (user_failure_detection_time == pg->pg_fdt) {
				/* Avoid any truncation or rounding errors */
				pg->pg_probeint = user_probe_interval;
				/*
				 * No more rtt probes will be sent. The actual
				 * fdt has dropped to the user specified value.
				 * pii_fd_snxt_basetime and pii_snxt_basetime
				 * will be in sync henceforth.
				 */
				reset_snxt_basetimes();
			}
		}
	}
}

/*
 * Process the incoming echo reply, in response to our unicast probe.
 * Common for both IPv4 and IPv6
 */
static void
incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply,
    struct in6_addr fromaddr, struct timeval *recv_tvp)
{
	int64_t	m;		/* rtt measurement in ns */
	hrtime_t cur_hrtime;	/* in ns from some arbitrary point */
	char	abuf[INET6_ADDRSTRLEN];
	int	pr_ndx;
	struct	target	*target;
	boolean_t exception;
	uint64_t pr_icmp_timestamp;
	uint16_t pr_icmp_seq;
	struct	probe_stats *pr_statp;
	struct 	phyint_group *pg = pii->pii_phyint->pi_group;

	/* Get the printable address for error reporting */
	(void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf));

	if (debug & D_PROBE) {
		logdebug("incoming_echo_reply: %s %s %s seq %u recv_tvp %lld\n",
		    AF_STR(pii->pii_af), pii->pii_name, abuf,
		    ntohs(reply->pr_icmp_seq), tv2ns(recv_tvp));
	}

	pr_icmp_timestamp = ntohll(reply->pr_icmp_timestamp);
	pr_icmp_seq = ntohs(reply->pr_icmp_seq);

	/* Reject out of window probe replies */
	if (SEQ_GE(pr_icmp_seq, pii->pii_snxt) ||
	    SEQ_LT(pr_icmp_seq, pii->pii_snxt - PROBE_STATS_COUNT)) {
		logtrace("out of window probe seq %u snxt %u on %s from %s\n",
		    pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
		pii->pii_cum_stats.unknown++;
		return;
	}

	cur_hrtime = gethrtime();
	m = (int64_t)(cur_hrtime - pr_icmp_timestamp);
	if (m < 0) {
		/*
		 * This is a ridiculously high value of rtt. rtt has wrapped
		 * around. Log a message, and ignore the rtt.
		 */
		logerr("incoming_echo_reply: rtt wraparound cur_hrtime %lld "
		    "reply timestamp %lld\n", cur_hrtime, pr_icmp_timestamp);
	}

	/*
	 * Get the probe index pr_ndx corresponding to the received icmp seq.
	 * number in our pii->pii_probes[] array. The icmp sequence number
	 * pii_snxt corresponds to the probe index pii->pii_probe_next
	 */
	pr_ndx = MOD_SUB(pii->pii_probe_next,
	    (uint16_t)(pii->pii_snxt - pr_icmp_seq), PROBE_STATS_COUNT);

	assert(PR_STATUS_VALID(pii->pii_probes[pr_ndx].pr_status));

	target = pii->pii_probes[pr_ndx].pr_target;

	/*
	 * Perform sanity checks, whether this probe reply that we
	 * have received is genuine
	 */
	if (target != NULL) {
		/*
		 * Compare the src. addr of the received ICMP or ICMPv6
		 * probe reply with the target address in our tables.
		 */
		if (!IN6_ARE_ADDR_EQUAL(&target->tg_address, &fromaddr)) {
			/*
			 * We don't have any record of having sent a probe to
			 * this target. This is a fake probe reply. Log an error
			 */
			logtrace("probe status %d Fake probe reply seq %u "
			    "snxt %u on %s from %s\n",
			    pii->pii_probes[pr_ndx].pr_status,
			    pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
			pii->pii_cum_stats.unknown++;
			return;
		} else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) {
			/*
			 * The address matches, but our tables indicate that
			 * this probe reply has been acked already. So this
			 * is a duplicate probe reply. Log an error
			 */
			logtrace("probe status %d Duplicate probe reply seq %u "
			    "snxt %u on %s from %s\n",
			    pii->pii_probes[pr_ndx].pr_status,
			    pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
			pii->pii_cum_stats.unknown++;
			return;
		}
	} else {
		/*
		 * Target must not be NULL in the PR_UNACKED state
		 */
		assert(pii->pii_probes[pr_ndx].pr_status != PR_UNACKED);
		if (pii->pii_probes[pr_ndx].pr_status == PR_UNUSED) {
			/*
			 * The probe stats slot is unused. So we didn't
			 * send out any probe to this target. This is a fake.
			 * Log an error.
			 */
			logtrace("probe status %d Fake probe reply seq %u "
			    "snxt %u on %s from %s\n",
			    pii->pii_probes[pr_ndx].pr_status,
			    pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
		}
		pii->pii_cum_stats.unknown++;
		return;
	}

	/*
	 * If the rtt does not appear to be right, don't update the
	 * rtt stats. This can happen if the system dropped into the
	 * debugger, or the system was hung or too busy for a
	 * substantial time that we didn't get a chance to run.
	 */
	if ((m < 0) || (ns2ms(m) > PROBE_STATS_COUNT * pg->pg_probeint)) {
		/*
		 * If the probe corresponding to this received response
		 * was truly sent 'm' ns. ago, then this response must
		 * have been rejected by the sequence number checks. The
		 * fact that it has passed the sequence number checks
		 * means that the measured rtt is wrong. We were probably
		 * scheduled long after the packet was received.
		 */
		goto out;
	}

	/*
	 * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses
	 * The initial few responses after the interface is repaired may
	 * contain high rtt's because they could have been queued up waiting
	 * for ARP/NDP resolution on a failed interface.
	 */
	if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg))
		goto out;

	/*
	 * Don't update the Conservative Round Trip Time estimate for this
	 * (phint, target) pair if this is the not the highest ack seq seen
	 * thus far on this target.
	 */
	if (!highest_ack_tg(pr_icmp_seq, target))
		goto out;

	/*
	 * Always update the rtt. This is a failure detection probe
	 * and we want to measure both increase / decrease in rtt.
	 */
	pi_set_crtt(target, m, _B_TRUE);

	/*
	 * If the crtt exceeds the average time between probes,
	 * investigate if this slow target is an exception. If so we
	 * can avoid this target and still meet the failure detection
	 * time. Otherwise we can't meet the failure detection time.
	 */
	if (target->tg_crtt > pg->pg_probeint) {
		exception = check_exception_target(pii, target);
		if (exception) {
			/*
			 * This target is exceptionally slow. Don't use it
			 * for future probes. check_exception_target() has
			 * made sure that we have at least MIN_PROBE_TARGETS
			 * other active targets
			 */
			if (pii->pii_targets_are_routers) {
				/*
				 * This is a slow router, mark it as slow
				 * and don't use it for further probes. We
				 * don't delete it, since it will be populated
				 * again when we do a router scan. Hence we
				 * need to maintain extra state (unlike the
				 * host case below).  Mark it as TG_SLOW.
				 */
				if (target->tg_status == TG_ACTIVE)
					pii->pii_ntargets--;
				target->tg_status = TG_SLOW;
				target->tg_latime = gethrtime();
				target->tg_rtt_sa = -1;
				target->tg_crtt = 0;
				target->tg_rtt_sd = 0;
				if (pii->pii_target_next == target) {
					pii->pii_target_next =
					    target_next(target);
				}
			} else {
				/*
				 * the slow target is not a router, we can
				 * just delete it. Send an icmp multicast and
				 * pick the fastest responder that is not
				 * already an active target. target_delete()
				 * adjusts pii->pii_target_next
				 */
				target_delete(target);
				probe(pii, PROBE_MULTI, cur_hrtime);
			}
		} else {
			/*
			 * We can't meet the failure detection time.
			 * Log a message, and update the detection time to
			 * whatever we can achieve.
			 */
			pg->pg_probeint = target->tg_crtt * NEXT_FDT_MULTIPLE;
			pg->pg_fdt = pg->pg_probeint * (NUM_PROBE_FAILS + 2);
			last_fdt_bumpup_time = gethrtime();
			if (pg != phyint_anongroup) {
				logerr("Cannot meet requested failure detection"
				    " time of %d ms on (%s %s) new failure"
				    " detection time for group \"%s\" is %d"
				    " ms\n", user_failure_detection_time,
				    AF_STR(pii->pii_af), pii->pii_name,
				    pg->pg_name, pg->pg_fdt);
			}
		}
	} else if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) &&
	    (user_failure_detection_time < pg->pg_fdt) &&
	    (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) {
		/*
		 * If the crtt has now dropped by a factor of LOWER_FDT_TRIGGER
		 * investigate if we can improve the failure detection time to
		 * meet whatever the user specified.
		 */
		if (check_pg_crtt_improved(pg)) {
			pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE,
			    user_failure_detection_time);
			pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2);
			if (pg != phyint_anongroup) {
				logerr("Improved failure detection time %d ms "
				    "on (%s %s) for group \"%s\"\n", pg->pg_fdt,
				    AF_STR(pii->pii_af), pii->pii_name,
				    pg->pg_name);
			}
			if (user_failure_detection_time == pg->pg_fdt) {
				/* Avoid any truncation or rounding errors */
				pg->pg_probeint = user_probe_interval;
				/*
				 * No more rtt probes will be sent. The actual
				 * fdt has dropped to the user specified value.
				 * pii_fd_snxt_basetime and pii_snxt_basetime
				 * will be in sync henceforth.
				 */
				reset_snxt_basetimes();
			}
		}
	}
out:
	pr_statp = &pii->pii_probes[pr_ndx];
	pr_statp->pr_hrtime_ackproc = cur_hrtime;
	pr_statp->pr_hrtime_ackrecv = pr_statp->pr_hrtime_sent +
	    (tv2ns(recv_tvp) - tv2ns(&pr_statp->pr_tv_sent));

	probe_chstate(pr_statp, pii, PR_ACKED);

	/*
	 * Update pii->pii_rack, i.e. the sequence number of the last received
	 * probe response, based on the echo reply we have received now, if
	 * either of the following conditions are satisfied.
	 * a. pii_rack is outside the current receive window of
	 *    [pii->pii_snxt - PROBE_STATS_COUNT, pii->pii_snxt).
	 *    This means we have not received probe responses for a
	 *    long time, and the sequence number has wrapped around.
	 * b. pii_rack is within the current receive window and this echo
	 *    reply corresponds to the highest sequence number we have seen
	 *    so far.
	 */
	if (SEQ_GE(pii->pii_rack, pii->pii_snxt) ||
	    SEQ_LT(pii->pii_rack, pii->pii_snxt - PROBE_STATS_COUNT) ||
	    SEQ_GT(pr_icmp_seq, pii->pii_rack)) {
		pii->pii_rack = pr_icmp_seq;
	}
}

/*
 * Returns true if seq is the highest unacknowledged seq for target tg
 * else returns false
 */
static boolean_t
highest_ack_tg(uint16_t seq, struct target *tg)
{
	struct phyint_instance *pii;
	int	 pr_ndx;
	uint16_t pr_seq;

	pii = tg->tg_phyint_inst;

	/*
	 * Get the seq number of the most recent probe sent so far,
	 * and also get the corresponding probe index in the probe stats
	 * array.
	 */
	pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
	pr_seq = pii->pii_snxt;
	pr_seq--;

	/*
	 * Start from the most recent probe and walk back, trying to find
	 * an acked probe corresponding to target tg.
	 */
	for (; pr_ndx != pii->pii_probe_next;
	    pr_ndx = PROBE_INDEX_PREV(pr_ndx), pr_seq--) {
		if (pii->pii_probes[pr_ndx].pr_target == tg &&
		    pii->pii_probes[pr_ndx].pr_status == PR_ACKED) {
			if (SEQ_GT(pr_seq, seq))
				return (_B_FALSE);
		}
	}
	return (_B_TRUE);
}

/*
 * Check whether the crtt for the group has improved by a factor of
 * LOWER_FDT_TRIGGER.  Small crtt improvements are ignored to avoid failure
 * detection time flapping in the face of small crtt changes.
 */
static boolean_t
check_pg_crtt_improved(struct phyint_group *pg)
{
	struct	phyint *pi;

	if (debug & D_PROBE)
		logdebug("check_pg_crtt_improved()\n");

	/*
	 * The crtt for the group is only improved if each phyint_instance
	 * for both ipv4 and ipv6 is improved.
	 */
	for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) {
		if (!check_pii_crtt_improved(pi->pi_v4) ||
		    !check_pii_crtt_improved(pi->pi_v6))
			return (_B_FALSE);
	}

	return (_B_TRUE);
}

/*
 * Check whether the crtt has improved substantially on this phyint_instance.
 * Returns _B_TRUE if there's no crtt information available, because pii
 * is NULL or the phyint_instance is not capable of probing.
 */
boolean_t
check_pii_crtt_improved(struct phyint_instance *pii) {
	struct 	target *tg;

	if (pii == NULL)
		return (_B_TRUE);

	if (!PROBE_CAPABLE(pii) ||
	    pii->pii_phyint->pi_state == PI_FAILED)
		return (_B_TRUE);

	for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
		if (tg->tg_status != TG_ACTIVE)
			continue;
		if (tg->tg_crtt > (pii->pii_phyint->pi_group->pg_probeint /
		    LOWER_FDT_TRIGGER)) {
			return (_B_FALSE);
		}
	}

	return (_B_TRUE);
}

/*
 * This target responds very slowly to probes. The target's crtt exceeds
 * the probe interval of its group. Compare against other targets
 * and determine if this target is an exception, if so return true, else false
 */
static boolean_t
check_exception_target(struct phyint_instance *pii, struct target *target)
{
	struct	target *tg;
	char abuf[INET6_ADDRSTRLEN];

	if (debug & D_PROBE) {
		logdebug("check_exception_target(%s %s target %s)\n",
		    AF_STR(pii->pii_af), pii->pii_name,
		    pr_addr(pii->pii_af, target->tg_address,
		    abuf, sizeof (abuf)));
	}

	/*
	 * We should have at least MIN_PROBE_TARGETS + 1 good targets now,
	 * to make a good judgement. Otherwise don't drop this target.
	 */
	if (pii->pii_ntargets <  MIN_PROBE_TARGETS + 1)
		return (_B_FALSE);

	/*
	 * Determine whether only this particular target is slow.
	 * We know that this target's crtt exceeds the group's probe interval.
	 * If all other active targets have a
	 * crtt < (this group's probe interval) / EXCEPTION_FACTOR,
	 * then this target is considered slow.
	 */
	for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
		if (tg != target && tg->tg_status == TG_ACTIVE) {
			if (tg->tg_crtt >
			    pii->pii_phyint->pi_group->pg_probeint /
			    EXCEPTION_FACTOR) {
				return (_B_FALSE);
			}
		}
	}

	return (_B_TRUE);
}

/*
 * Update the target list. The icmp all hosts multicast has given us
 * some host to which we can send probes. If we already have sufficient
 * targets, discard it.
 */
static void
incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply,
    struct in6_addr fromaddr)
/* ARGSUSED */
{
	int af;
	char abuf[INET6_ADDRSTRLEN];
	struct phyint *pi;

	if (debug & D_PROBE) {
		logdebug("incoming_mcast_reply(%s %s %s)\n",
		    AF_STR(pii->pii_af), pii->pii_name,
		    pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)));
	}

	/*
	 * Using host targets is a fallback mechanism. If we have
	 * found a router, don't add this host target. If we already
	 * know MAX_PROBE_TARGETS, don't add another target.
	 */
	assert(pii->pii_ntargets <= MAX_PROBE_TARGETS);
	if (pii->pii_targets != NULL) {
		if (pii->pii_targets_are_routers ||
		    (pii->pii_ntargets == MAX_PROBE_TARGETS)) {
			return;
		}
	}

	if (IN6_IS_ADDR_UNSPECIFIED(&fromaddr) ||
	    IN6_IS_ADDR_V4MAPPED_ANY(&fromaddr)) {
		/*
		 * Guard against response from 0.0.0.0
		 * and ::. Log a trace message
		 */
		logtrace("probe response from %s on %s\n",
		    pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)),
		    pii->pii_name);
		return;
	}

	/*
	 * This address is one of our own, so reject this address as a
	 * valid probe target.
	 */
	af = pii->pii_af;
	if (own_address(fromaddr))
		return;

	/*
	 * If the phyint is part a named group, then add the address to all
	 * members of the group.  Otherwise, add the address only to the
	 * phyint itself, since other phyints in the anongroup may not be on
	 * the same subnet.
	 */
	pi = pii->pii_phyint;
	if (pi->pi_group == phyint_anongroup) {
		target_add(pii, fromaddr, _B_FALSE);
	} else {
		pi = pi->pi_group->pg_phyint;
		for (; pi != NULL; pi = pi->pi_pgnext)
			target_add(PHYINT_INSTANCE(pi, af), fromaddr, _B_FALSE);
	}
}

/*
 * Compute CRTT given an existing scaled average, scaled deviation estimate
 * and a new rtt time.  The formula is from Jacobson and Karels'
 * "Congestion Avoidance and Control" in SIGCOMM '88.  The variable names
 * are the same as those in Appendix A.2 of that paper.
 *
 * m = new measurement
 * sa = scaled RTT average (8 * average estimates)
 * sv = scaled mean deviation (mdev) of RTT (4 * deviation estimates).
 * crtt = Conservative round trip time. Used to determine whether probe
 * has timed out.
 *
 * New scaled average and deviation are passed back via sap and svp
 */
static int64_t
compute_crtt(int64_t *sap, int64_t *svp, int64_t m)
{
	int64_t sa = *sap;
	int64_t sv = *svp;
	int64_t crtt;
	int64_t saved_m = m;

	assert(*sap >= -1);
	assert(*svp >= 0);

	if (sa != -1) {
		/*
		 * Update average estimator:
		 *	new rtt = old rtt + 1/8 Error
		 *	    where Error = m - old rtt
		 *	i.e. 8 * new rtt = 8 * old rtt + Error
		 *	i.e. new sa =  old sa + Error
		 */
		m -= sa >> 3;		/* m is now Error in estimate. */
		if ((sa += m) < 0) {
			/* Don't allow the smoothed average to be negative. */
			sa = 0;
		}

		/*
		 * Update deviation estimator:
		 *	new mdev =  old mdev + 1/4 (abs(Error) - old mdev)
		 *	i.e. 4 * new mdev = 4 * old mdev +
		 *		(abs(Error) - old mdev)
		 * 	i.e. new sv = old sv + (abs(Error) - old mdev)
		 */
		if (m < 0)
			m = -m;
		m -= sv >> 2;
		sv += m;
	} else {
		/* Initialization. This is the first response received. */
		sa = (m << 3);
		sv = (m << 1);
	}

	crtt = (sa >> 3) + sv;

	if (debug & D_PROBE) {
		logerr("compute_crtt: m = %lld sa = %lld, sv = %lld -> "
		    "crtt = %lld\n", saved_m, sa, sv, crtt);
	}

	*sap = sa;
	*svp = sv;

	/*
	 * CRTT = average estimates  + 4 * deviation estimates
	 *	= sa / 8 + sv
	 */
	return (crtt);
}

static void
pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni)
{
	struct phyint_instance *pii = tg->tg_phyint_inst;
	int probe_interval = pii->pii_phyint->pi_group->pg_probeint;
	int64_t sa = tg->tg_rtt_sa;
	int64_t sv = tg->tg_rtt_sd;
	int new_crtt;
	int i;

	if (debug & D_PROBE)
		logdebug("pi_set_crtt: target -  m %lld\n", m);

	/* store the round trip time, in case we need to defer computation */
	tg->tg_deferred[tg->tg_num_deferred] = m;

	new_crtt = ns2ms(compute_crtt(&sa, &sv, m));

	/*
	 * If this probe's round trip time would singlehandedly cause an
	 * increase in the group's probe interval consider it suspect.
	 */
	if ((new_crtt > probe_interval) && is_probe_uni) {
		if (debug & D_PROBE) {
			logdebug("Received a suspect probe on %s, new_crtt ="
			    " %d, probe_interval = %d, num_deferred = %d\n",
			    pii->pii_probe_logint->li_name, new_crtt,
			    probe_interval, tg->tg_num_deferred);
		}

		/*
		 * If we've deferred as many rtts as we plan on deferring, then
		 * assume the link really did slow down and process all queued
		 * rtts
		 */
		if (tg->tg_num_deferred == MAXDEFERREDRTT) {
			if (debug & D_PROBE) {
				logdebug("Received MAXDEFERREDRTT probes which "
				    "would cause an increased probe_interval.  "
				    "Integrating queued rtt data points.\n");
			}

			for (i = 0; i <= tg->tg_num_deferred; i++) {
				tg->tg_crtt = ns2ms(compute_crtt(&tg->tg_rtt_sa,
				    &tg->tg_rtt_sd, tg->tg_deferred[i]));
			}

			tg->tg_num_deferred = 0;
		} else {
			tg->tg_num_deferred++;
		}
		return;
	}

	/*
	 * If this is a normal probe, or an RTT probe that would lead to a
	 * reduced CRTT, then update our CRTT data.  Further, if this was
	 * a normal probe, pitch any deferred probes since our probes are
	 * again being answered within our CRTT estimates.
	 */
	if (is_probe_uni || new_crtt < tg->tg_crtt) {
		tg->tg_rtt_sa = sa;
		tg->tg_rtt_sd = sv;
		tg->tg_crtt = new_crtt;
		if (is_probe_uni)
			tg->tg_num_deferred = 0;
	}
}

/*
 * Return a pointer to the specified option buffer.
 * If not found return NULL.
 */
static void *
find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type)
{
	struct cmsghdr *cmsg;

	for (cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
	    cmsg = CMSG_NXTHDR(msg, cmsg)) {
		if (cmsg->cmsg_level == cmsg_level &&
		    cmsg->cmsg_type == cmsg_type) {
			return (CMSG_DATA(cmsg));
		}
	}
	return (NULL);
}

/*
 * Try to activate another INACTIVE interface in the same group as `pi'.
 * Prefer STANDBY INACTIVE to just INACTIVE.
 */
void
phyint_activate_another(struct phyint *pi)
{
	struct phyint *pi2;
	struct phyint *inactivepi = NULL;

	if (pi->pi_group == phyint_anongroup)
		return;

	for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
		if (pi == pi2 || pi2->pi_state != PI_RUNNING ||
		    !(pi2->pi_flags & IFF_INACTIVE))
			continue;

		inactivepi = pi2;
		if (pi2->pi_flags & IFF_STANDBY)
			break;
	}

	if (inactivepi != NULL)
		(void) change_pif_flags(inactivepi, 0, IFF_INACTIVE);
}

/*
 * Transition a phyint back to PI_RUNNING (from PI_FAILED or PI_OFFLINE).  The
 * caller must ensure that the transition is appropriate.  Clears IFF_OFFLINE
 * or IFF_FAILED, as appropriate.  Also sets IFF_INACTIVE on this or other
 * interfaces as appropriate (see comment below).  Finally, also updates the
 * phyint's group state to account for the change.
 */
void
phyint_transition_to_running(struct phyint *pi)
{
	struct phyint *pi2;
	struct phyint *actstandbypi = NULL;
	uint_t nactive = 0, nnonstandby = 0;
	boolean_t onlining = (pi->pi_state == PI_OFFLINE);
	uint64_t set, clear;

	/*
	 * The interface is running again, but should it or another interface
	 * in the group end up INACTIVE?  There are three cases:
	 *
	 * 1. If it's a STANDBY interface, it should be end up INACTIVE if
	 *    the group is operating at capacity (i.e., there are at least as
	 *    many active interfaces as non-STANDBY interfaces in the group).
	 *    No other interfaces should be changed.
	 *
	 * 2. If it's a non-STANDBY interface and we're onlining it or
	 *    FAILBACK is enabled, then it should *not* end up INACTIVE.
	 *    Further, if the group is above capacity as a result of this
	 *    interface, then an active STANDBY interface in the group should
	 *    end up INACTIVE.
	 *
	 * 3. If it's a non-STANDBY interface, we're repairing it, and
	 *    FAILBACK is disabled, then it should end up INACTIVE *unless*
	 *    the group was failed (in which case we have no choice but to
	 *    use it).  No other interfaces should be changed.
	 */
	if (pi->pi_group != phyint_anongroup) {
		pi2 = pi->pi_group->pg_phyint;
		for (; pi2 != NULL; pi2 = pi2->pi_pgnext) {
			if (!(pi2->pi_flags & IFF_STANDBY))
				nnonstandby++;

			if (pi2->pi_state == PI_RUNNING) {
				if (!(pi2->pi_flags & IFF_INACTIVE)) {
					nactive++;
					if (pi2->pi_flags & IFF_STANDBY)
						actstandbypi = pi2;
				}
			}
		}
	}

	set = 0;
	clear = (onlining ? IFF_OFFLINE : IFF_FAILED);

	if (pi->pi_flags & IFF_STANDBY) {			/* case 1 */
		if (nactive >= nnonstandby)
			set |= IFF_INACTIVE;
		else
			clear |= IFF_INACTIVE;
	} else if (onlining || failback_enabled) {		/* case 2 */
		if (nactive >= nnonstandby && actstandbypi != NULL)
			(void) change_pif_flags(actstandbypi, IFF_INACTIVE, 0);
	} else if (!GROUP_FAILED(pi->pi_group)) {		/* case 3 */
		set |= IFF_INACTIVE;
	}
	(void) change_pif_flags(pi, set, clear);

	phyint_chstate(pi, PI_RUNNING);

	/*
	 * Update the group state to account for the change.
	 */
	phyint_group_refresh_state(pi->pi_group);
}

/*
 * See if a previously failed interface has started working again.
 */
void
phyint_check_for_repair(struct phyint *pi)
{
	if (!phyint_repaired(pi))
		return;

	if (pi->pi_group == phyint_anongroup) {
		logerr("IP interface repair detected on %s\n", pi->pi_name);
	} else {
		logerr("IP interface repair detected on %s of group %s\n",
		    pi->pi_name, pi->pi_group->pg_name);
	}

	/*
	 * If the interface is PI_OFFLINE, it can't be made PI_RUNNING yet.
	 * So just clear IFF_OFFLINE and defer phyint_transition_to_running()
	 * until it is brought back online.
	 */
	if (pi->pi_state == PI_OFFLINE) {
		(void) change_pif_flags(pi, 0, IFF_FAILED);
		return;
	}

	phyint_transition_to_running(pi);	/* calls phyint_chstate() */
}

/*
 * See if an interface has failed, or if the whole group of interfaces has
 * failed.
 */
static void
phyint_inst_check_for_failure(struct phyint_instance *pii)
{
	struct phyint	*pi = pii->pii_phyint;
	struct phyint	*pi2;
	boolean_t	was_active;

	switch (failure_state(pii)) {
	case PHYINT_FAILURE:
		was_active = ((pi->pi_flags & IFF_INACTIVE) == 0);

		(void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE);
		if (pi->pi_group == phyint_anongroup) {
			logerr("IP interface failure detected on %s\n",
			    pii->pii_name);
		} else {
			logerr("IP interface failure detected on %s of group"
			    " %s\n", pii->pii_name, pi->pi_group->pg_name);
		}

		/*
		 * If the failed interface was active, activate another
		 * INACTIVE interface in the group if possible.
		 */
		if (was_active)
			phyint_activate_another(pi);

		/*
		 * If the interface is offline, the state change will be
		 * noted when it comes back online.
		 */
		if (pi->pi_state != PI_OFFLINE) {
			phyint_chstate(pi, PI_FAILED);
			reset_crtt_all(pi);
		}
		break;

	case GROUP_FAILURE:
		pi2 = pi->pi_group->pg_phyint;
		for (; pi2 != NULL; pi2 = pi2->pi_pgnext) {
			(void) change_pif_flags(pi2, IFF_FAILED, IFF_INACTIVE);
			if (pi2->pi_state == PI_OFFLINE) /* see comment above */
				continue;

			reset_crtt_all(pi2);
			/*
			 * In the case of host targets, we would have flushed
			 * the targets, and gone to PI_NOTARGETS state.
			 */
			if (pi2->pi_state == PI_RUNNING)
				phyint_chstate(pi2, PI_FAILED);
		}
		break;

	default:
		break;
	}
}

/*
 * Determines if any timeout event has occurred and returns the number of
 * milliseconds until the next timeout event for the phyint. Returns
 * TIMER_INFINITY for "never".
 */
uint_t
phyint_inst_timer(struct phyint_instance *pii)
{
	int 	pr_ndx;
	uint_t	timeout;
	struct	target	*cur_tg;
	struct	probe_stats *pr_statp;
	struct	phyint_instance *pii_other;
	struct	phyint *pi;
	int	valid_unack_count;
	int	i;
	int	interval;
	uint_t	check_time;
	uint_t	cur_time;
	hrtime_t cur_hrtime;
	int	probe_interval = pii->pii_phyint->pi_group->pg_probeint;

	cur_hrtime = gethrtime();
	cur_time = ns2ms(cur_hrtime);

	if (debug & D_TIMER) {
		logdebug("phyint_inst_timer(%s %s)\n",
		    AF_STR(pii->pii_af), pii->pii_name);
	}

	pii_other = phyint_inst_other(pii);
	if (!PROBE_ENABLED(pii) && !PROBE_ENABLED(pii_other)) {
		/*
		 * Check to see if we're here due to link up/down flapping; If
		 * enough time has passed, then try to bring the interface
		 * back up; otherwise, schedule a timer to bring it back up
		 * when enough time *has* elapsed.
		 */
		pi = pii->pii_phyint;
		if (pi->pi_state == PI_FAILED && LINK_UP(pi)) {
			check_time = pi->pi_whenup[pi->pi_whendx] + MSEC_PERMIN;
			if (check_time > cur_time)
				return (check_time - cur_time);

			phyint_check_for_repair(pi);
		}
	}

	/*
	 * If probing is not enabled on this phyint instance, don't proceed.
	 */
	if (!PROBE_ENABLED(pii))
		return (TIMER_INFINITY);

	/*
	 * If the timer has fired too soon, probably triggered
	 * by some other phyint instance, return the remaining
	 * time
	 */
	if (TIME_LT(cur_time, pii->pii_snxt_time))
		return (pii->pii_snxt_time - cur_time);

	/*
	 * If the link is down, don't send any probes for now.
	 */
	if (LINK_DOWN(pii->pii_phyint))
		return (TIMER_INFINITY);

	/*
	 * Randomize the next probe time, between MIN_RANDOM_FACTOR
	 * and MAX_RANDOM_FACTOR with respect to the base probe time.
	 * Base probe time is strictly periodic.
	 */
	interval = GET_RANDOM(
	    (int)(MIN_RANDOM_FACTOR * user_probe_interval),
	    (int)(MAX_RANDOM_FACTOR * user_probe_interval));
	pii->pii_snxt_time = pii->pii_snxt_basetime + interval;

	/*
	 * Check if the current time > next time to probe. If so, we missed
	 * sending 1 or more probes, probably due to heavy system load. At least
	 * 'MIN_RANDOM_FACTOR * user_probe_interval' ms has elapsed since we
	 * were scheduled. Make adjustments to the times, in multiples of
	 * user_probe_interval.
	 */
	if (TIME_GT(cur_time, pii->pii_snxt_time)) {
		int n;

		n = (cur_time - pii->pii_snxt_time) / user_probe_interval;
		pii->pii_snxt_time 	+= (n + 1) * user_probe_interval;
		pii->pii_snxt_basetime 	+= (n + 1) * user_probe_interval;
		logtrace("missed sending %d probes cur_time %u snxt_time %u"
		    " snxt_basetime %u\n", n + 1, cur_time, pii->pii_snxt_time,
		    pii->pii_snxt_basetime);

		/* Collect statistics about missed probes */
		probes_missed.pm_nprobes += n + 1;
		probes_missed.pm_ntimes++;
	}
	pii->pii_snxt_basetime += user_probe_interval;
	interval = pii->pii_snxt_time - cur_time;
	if (debug & D_TARGET) {
		logdebug("cur_time %u snxt_time %u snxt_basetime %u"
		    " interval %u\n", cur_time, pii->pii_snxt_time,
		    pii->pii_snxt_basetime, interval);
	}

	/*
	 * If no targets are known, we need to send an ICMP multicast. The
	 * probe type is PROBE_MULTI.  We'll check back in 'interval' msec
	 * to see if we found a target.
	 */
	if (pii->pii_target_next == NULL) {
		assert(pii->pii_ntargets == 0);
		pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
		probe(pii, PROBE_MULTI, cur_time);
		return (interval);
	}

	if ((user_probe_interval != probe_interval) &&
	    TIME_LT(pii->pii_snxt_time, pii->pii_fd_snxt_basetime)) {
		/*
		 * the failure detection (fd) probe timer has not yet fired.
		 * Need to send only an rtt probe. The probe type is PROBE_RTT.
		 */
		probe(pii, PROBE_RTT, cur_hrtime);
		return (interval);
	}
	/*
	 * the fd probe timer has fired. Need to do all failure
	 * detection / recovery calculations, and then send an fd probe
	 * of type PROBE_UNI.
	 */
	if (user_probe_interval == probe_interval) {
		/*
		 * We could have missed some probes, and then adjusted
		 * pii_snxt_basetime above. Otherwise we could have
		 * blindly added probe_interval to pii_fd_snxt_basetime.
		 */
		pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
	} else {
		pii->pii_fd_snxt_basetime += probe_interval;
		if (TIME_GT(cur_time, pii->pii_fd_snxt_basetime)) {
			int n;

			n = (cur_time - pii->pii_fd_snxt_basetime) /
			    probe_interval;
			pii->pii_fd_snxt_basetime += (n + 1) * probe_interval;
		}
	}

	/*
	 * We can have at most, the latest 2 probes that we sent, in
	 * the PR_UNACKED state. All previous probes sent, are either
	 * PR_LOST or PR_ACKED. An unacknowledged probe is considered
	 * timed out if the probe's time_start + the CRTT < currenttime.
	 * For each of the last 2 probes, examine whether it has timed
	 * out. If so, mark it PR_LOST. The probe stats is a circular array.
	 */
	pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
	valid_unack_count = 0;

	for (i = 0; i < 2; i++) {
		pr_statp = &pii->pii_probes[pr_ndx];
		cur_tg = pii->pii_probes[pr_ndx].pr_target;
		switch (pr_statp->pr_status) {
		case PR_ACKED:
			/*
			 * We received back an ACK, so the switch clearly
			 * is not dropping our traffic, and thus we can
			 * enable failure detection immediately.
			 */
			if (pii->pii_fd_hrtime > gethrtime()) {
				if (debug & D_PROBE) {
					logdebug("successful probe on %s; "
					    "ending quiet period\n",
					    pii->pii_phyint->pi_name);
				}
				pii->pii_fd_hrtime = gethrtime();
			}
			break;

		case PR_UNACKED:
			assert(cur_tg != NULL);
			/*
			 * The crtt could be zero for some reason,
			 * Eg. the phyint could be failed. If the crtt is
			 * not available use group's probe interval,
			 * which is a worst case estimate.
			 */
			timeout = ns2ms(pr_statp->pr_hrtime_start);
			if (cur_tg->tg_crtt != 0) {
				timeout += cur_tg->tg_crtt;
			} else {
				timeout += probe_interval;
			}
			if (TIME_LT(timeout, cur_time)) {
				pr_statp->pr_time_lost = timeout;
				probe_chstate(pr_statp, pii, PR_LOST);
			} else if (i == 1) {
				/*
				 * We are forced to consider this probe
				 * lost, as we can have at most 2 unack.
				 * probes any time, and we will be sending a
				 * probe at the end of this function.
				 * Normally, we should not be here, but
				 * this can happen if an incoming response
				 * that was considered lost has increased
				 * the crtt for this target, and also bumped
				 * up the FDT. Note that we never cancel or
				 * increase the current pii_time_left, so
				 * when the timer fires, we find 2 valid
				 * unacked probes, and they are yet to timeout
				 */
				pr_statp->pr_time_lost = cur_time;
				probe_chstate(pr_statp, pii, PR_LOST);
			} else {
				/*
				 * Only the most recent probe can enter
				 * this 'else' arm. The second most recent
				 * probe must take either of the above arms,
				 * if it is unacked.
				 */
				valid_unack_count++;
			}
			break;
		}
		pr_ndx = PROBE_INDEX_PREV(pr_ndx);
	}

	/*
	 * We send out 1 probe randomly in the interval between one half
	 * and one probe interval for the group. Given that the CRTT is always
	 * less than the group's probe interval, we can have at most 1
	 * unacknowledged probe now.  All previous probes are either lost or
	 * acked.
	 */
	assert(valid_unack_count == 0 || valid_unack_count == 1);

	/*
	 * The timer has fired. Take appropriate action depending
	 * on the current state of the phyint.
	 *
	 * PI_RUNNING state 	- Failure detection
	 * PI_FAILED state 	- Repair detection
	 */
	switch (pii->pii_phyint->pi_state) {
	case PI_FAILED:
		/*
		 * If the most recent probe (excluding unacked probes that
		 * are yet to time out) has been acked, check whether the
		 * phyint is now repaired.
		 */
		if (pii->pii_rack + valid_unack_count + 1 == pii->pii_snxt) {
			phyint_check_for_repair(pii->pii_phyint);
		}
		break;

	case PI_RUNNING:
		/*
		 * It's possible our probes have been lost because of a
		 * spanning-tree mandated quiet period on the switch.  If so,
		 * ignore the lost probes.
		 */
		if (pii->pii_fd_hrtime - cur_hrtime > 0)
			break;

		if (pii->pii_rack + valid_unack_count + 1 != pii->pii_snxt) {
			/*
			 * We have 1 or more failed probes (excluding unacked
			 * probes that are yet to time out). Determine if the
			 * phyint has failed.
			 */
			phyint_inst_check_for_failure(pii);
		}
		break;

	default:
		logerr("phyint_inst_timer: invalid state %d\n",
		    pii->pii_phyint->pi_state);
		abort();
	}

	/*
	 * Start the next probe. probe() will also set pii->pii_probe_time_left
	 * to the group's probe interval. If phyint_failed -> target_flush_hosts
	 * was called, the target list may be empty.
	 */
	if (pii->pii_target_next != NULL) {
		probe(pii, PROBE_UNI, cur_hrtime);
		/*
		 * If we have just the one probe target, and we're not using
		 * router targets, try to find another as we presently have
		 * no resilience.
		 */
		if (!pii->pii_targets_are_routers && pii->pii_ntargets == 1)
			probe(pii, PROBE_MULTI, cur_hrtime);
	} else {
		probe(pii, PROBE_MULTI, cur_hrtime);
	}
	return (interval);
}

/*
 * Start the probe timer for an interface instance.
 */
void
start_timer(struct phyint_instance *pii)
{
	uint32_t interval;

	/*
	 * Spread the base probe times (pi_snxt_basetime) across phyints
	 * uniformly over the (curtime..curtime + the group's probe_interval).
	 * pi_snxt_basetime is strictly periodic with a frequency of
	 * the group's probe interval. The actual probe time pi_snxt_time
	 * adds some randomness to pi_snxt_basetime and happens in probe().
	 * For the 1st probe on each phyint after the timer is started,
	 * pi_snxt_time and pi_snxt_basetime are the same.
	 */
	interval = GET_RANDOM(0,
	    (int)pii->pii_phyint->pi_group->pg_probeint);

	pii->pii_snxt_basetime = getcurrenttime() + interval;
	pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
	pii->pii_snxt_time = pii->pii_snxt_basetime;
	timer_schedule(interval);
}

/*
 * Restart the probe timer on an interface instance.
 */
static void
restart_timer(struct phyint_instance *pii)
{
	/*
	 * We don't need to restart the timer if it was never started in
	 * the first place (pii->pii_basetime_inited not set), as the timer
	 * won't have gone off yet.
	 */
	if (pii->pii_basetime_inited != 0) {

		if (debug & D_LINKNOTE)
			logdebug("restart timer: restarting timer on %s, "
			    "address family %s\n", pii->pii_phyint->pi_name,
			    AF_STR(pii->pii_af));

		start_timer(pii);
	}
}

static void
process_link_state_down(struct phyint *pi)
{
	logerr("The link has gone down on %s\n", pi->pi_name);

	/*
	 * Clear the probe statistics arrays, we don't want the repair
	 * detection logic relying on probes that were successful prior
	 * to the link going down.
	 */
	if (PROBE_CAPABLE(pi->pi_v4))
		clear_pii_probe_stats(pi->pi_v4);
	if (PROBE_CAPABLE(pi->pi_v6))
		clear_pii_probe_stats(pi->pi_v6);
	/*
	 * Check for interface failure.  Although we know the interface
	 * has failed, we don't know if all the other interfaces in the
	 * group have failed as well.
	 */
	if ((pi->pi_state == PI_RUNNING) ||
	    (pi->pi_state != PI_FAILED && !GROUP_FAILED(pi->pi_group))) {
		if (debug & D_LINKNOTE) {
			logdebug("process_link_state_down:"
			    " checking for failure on %s\n", pi->pi_name);
		}

		if (pi->pi_v4 != NULL)
			phyint_inst_check_for_failure(pi->pi_v4);
		else if (pi->pi_v6 != NULL)
			phyint_inst_check_for_failure(pi->pi_v6);
	}
}

static void
process_link_state_up(struct phyint *pi)
{
	logerr("The link has come up on %s\n", pi->pi_name);

	/*
	 * We stopped any running timers on each instance when the link
	 * went down, so restart them.
	 */
	if (pi->pi_v4)
		restart_timer(pi->pi_v4);
	if (pi->pi_v6)
		restart_timer(pi->pi_v6);

	phyint_check_for_repair(pi);

	pi->pi_whenup[pi->pi_whendx++] = getcurrenttime();
	if (pi->pi_whendx == LINK_UP_PERMIN)
		pi->pi_whendx = 0;
}

/*
 * Process any changes in link state passed up from the interfaces.
 */
void
process_link_state_changes(void)
{
	struct phyint *pi;

	/* Look for interfaces where the link state has just changed */

	for (pi = phyints; pi != NULL; pi = pi->pi_next) {
		boolean_t old_link_state_up = LINK_UP(pi);

		/*
		 * Except when the "phyint" structure is created, this is
		 * the only place the link state is updated.  This allows
		 * this routine to detect changes in link state, rather
		 * than just the current state.
		 */
		UPDATE_LINK_STATE(pi);

		if (LINK_DOWN(pi)) {
			/*
			 * Has link just gone down?
			 */
			if (old_link_state_up)
				process_link_state_down(pi);
		} else {
			/*
			 * Has link just gone back up?
			 */
			if (!old_link_state_up)
				process_link_state_up(pi);
		}
	}
}

void
reset_crtt_all(struct phyint *pi)
{
	struct phyint_instance *pii;
	struct target *tg;

	pii = pi->pi_v4;
	if (pii != NULL) {
		for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
			tg->tg_crtt = 0;
			tg->tg_rtt_sa = -1;
			tg->tg_rtt_sd = 0;
		}
	}

	pii = pi->pi_v6;
	if (pii != NULL) {
		for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
			tg->tg_crtt = 0;
			tg->tg_rtt_sa = -1;
			tg->tg_rtt_sd = 0;
		}
	}
}

/*
 * Check if the phyint has failed the last NUM_PROBE_FAILS consecutive
 * probes on both instances IPv4 and IPv6.
 * If the interface has failed, return the time of the first probe failure
 * in "tff".
 */
static int
phyint_inst_probe_failure_state(struct phyint_instance *pii, uint_t *tff)
{
	uint_t	pi_tff;
	struct	target *cur_tg;
	struct	probe_fail_count pfinfo;
	struct	phyint_instance *pii_other;
	int	pr_ndx;

	/*
	 * Get the number of consecutive failed probes on
	 * this phyint across all targets. Also get the number
	 * of consecutive failed probes on this target only
	 */
	pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
	cur_tg = pii->pii_probes[pr_ndx].pr_target;
	probe_fail_info(pii, cur_tg, &pfinfo);

	/* Get the time of first failure, for later use */
	pi_tff = pfinfo.pf_tff;

	/*
	 * If the current target has not responded to the
	 * last NUM_PROBE_FAILS probes, and other targets are
	 * responding delete this target. Dead gateway detection
	 * will eventually remove this target (if router) from the
	 * routing tables. If that does not occur, we may end
	 * up adding this to our list again.
	 */
	if (pfinfo.pf_nfail < NUM_PROBE_FAILS &&
	    pfinfo.pf_nfail_tg >= NUM_PROBE_FAILS) {
		if (pii->pii_targets_are_routers) {
			if (cur_tg->tg_status == TG_ACTIVE)
				pii->pii_ntargets--;
			cur_tg->tg_status = TG_DEAD;
			cur_tg->tg_crtt = 0;
			cur_tg->tg_rtt_sa = -1;
			cur_tg->tg_rtt_sd = 0;
			if (pii->pii_target_next == cur_tg)
				pii->pii_target_next = target_next(cur_tg);
		} else {
			target_delete(cur_tg);
			probe(pii, PROBE_MULTI, gethrtime());
		}
		return (PHYINT_OK);
	}

	/*
	 * If the phyint has lost NUM_PROBE_FAILS or more
	 * consecutive probes, on both IPv4 and IPv6 protocol
	 * instances of the phyint, then trigger failure
	 * detection, else return false
	 */
	if (pfinfo.pf_nfail < NUM_PROBE_FAILS)
		return (PHYINT_OK);

	pii_other = phyint_inst_other(pii);
	if (PROBE_CAPABLE(pii_other)) {
		probe_fail_info(pii_other, NULL, &pfinfo);
		if (pfinfo.pf_nfail >= NUM_PROBE_FAILS) {
			/*
			 * We have NUM_PROBE_FAILS or more failures
			 * on both IPv4 and IPv6. Get the earliest
			 * time when failure was detected on this
			 * phyint across IPv4 and IPv6.
			 */
			if (TIME_LT(pfinfo.pf_tff, pi_tff))
				pi_tff = pfinfo.pf_tff;
		} else {
			/*
			 * This instance has < NUM_PROBE_FAILS failure.
			 * So return false
			 */
			return (PHYINT_OK);
		}
	}
	*tff = pi_tff;
	return (PHYINT_FAILURE);
}

/*
 * Check if the link has gone down on this phyint, or it has failed the
 * last NUM_PROBE_FAILS consecutive probes on both instances IPv4 and IPv6.
 * Also look at other phyints of this group, for group failures.
 */
int
failure_state(struct phyint_instance *pii)
{
	struct	probe_success_count psinfo;
	uint_t	pi2_tls;		/* time last success */
	uint_t	pi_tff;			/* time first fail */
	struct	phyint *pi2;
	struct	phyint *pi;
	struct	phyint_instance *pii2;
	struct  phyint_group *pg;
	int	retval;

	if (debug & D_FAILREP)
		logdebug("phyint_failed(%s)\n", pii->pii_name);

	pi = pii->pii_phyint;
	pg = pi->pi_group;

	if (LINK_UP(pi) && phyint_inst_probe_failure_state(pii, &pi_tff) ==
	    PHYINT_OK)
		return (PHYINT_OK);

	/*
	 * At this point, the link is down, or the phyint is suspect, as it
	 * has lost NUM_PROBE_FAILS or more probes. If the phyint does not
	 * belong to any group, this is a PHYINT_FAILURE.  Otherwise, continue
	 * on to determine whether this should be considered a PHYINT_FAILURE
	 * or GROUP_FAILURE.
	 */
	if (pg == phyint_anongroup)
		return (PHYINT_FAILURE);

	/*
	 * Need to compare against other phyints of the same group
	 * to exclude group failures. If the failure was detected via
	 * probing, then if the time of last success (tls) of any
	 * phyint is more recent than the time of first fail (tff) of the
	 * phyint in question, and the link is up on the phyint,
	 * then it is a phyint failure. Otherwise it is a group failure.
	 * If failure was detected via a link down notification sent from
	 * the driver to IP, we see if any phyints in the group are still
	 * running and haven't received a link down notification.  We
	 * will usually be processing the link down notification shortly
	 * after it was received, so there is no point looking at the tls
	 * of other phyints.
	 */
	retval = GROUP_FAILURE;
	for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
		/* Exclude ourself from comparison */
		if (pi2 == pi)
			continue;

		if (LINK_DOWN(pi)) {
			/*
			 * We use FLAGS_TO_LINK_STATE() to test the flags
			 * directly, rather then LINK_UP() or LINK_DOWN(), as
			 * we may not have got round to processing the link
			 * state for the other phyints in the group yet.
			 *
			 * The check for PI_RUNNING and group failure handles
			 * the case when the group begins to recover.
			 * PI_RUNNING will be set, and group failure cleared
			 * only after receipt of NUM_PROBE_REPAIRS, by which
			 * time the other phyints should have received at
			 * least 1 packet, and so will not have NUM_PROBE_FAILS.
			 */
			if ((pi2->pi_state == PI_RUNNING) &&
			    !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) {
				retval = PHYINT_FAILURE;
				break;
			}
			continue;
		}

		if (LINK_DOWN(pi2))
			continue;

		/*
		 * If there's no probe-based failure detection on this
		 * interface, and its link is still up, then it's still
		 * working and thus the group has not failed.
		 */
		if (!PROBE_ENABLED(pi2->pi_v4) && !PROBE_ENABLED(pi2->pi_v6)) {
			retval = PHYINT_FAILURE;
			break;
		}

		/*
		 * Need to compare against both IPv4 and IPv6 instances.
		 */
		pii2 = pi2->pi_v4;
		if (pii2 != NULL) {
			probe_success_info(pii2, NULL, &psinfo);
			if (psinfo.ps_tls_valid) {
				pi2_tls = psinfo.ps_tls;
				/*
				 * See comment above regarding check
				 * for PI_RUNNING and group failure.
				 */
				if (TIME_GT(pi2_tls, pi_tff) &&
				    (pi2->pi_state == PI_RUNNING) &&
				    !GROUP_FAILED(pg) &&
				    FLAGS_TO_LINK_STATE(pi2)) {
					retval = PHYINT_FAILURE;
					break;
				}
			}
		}

		pii2 = pi2->pi_v6;
		if (pii2 != NULL) {
			probe_success_info(pii2, NULL, &psinfo);
			if (psinfo.ps_tls_valid) {
				pi2_tls = psinfo.ps_tls;
				/*
				 * See comment above regarding check
				 * for PI_RUNNING and group failure.
				 */
				if (TIME_GT(pi2_tls, pi_tff) &&
				    (pi2->pi_state == PI_RUNNING) &&
				    !GROUP_FAILED(pg) &&
				    FLAGS_TO_LINK_STATE(pi2)) {
					retval = PHYINT_FAILURE;
					break;
				}
			}
		}
	}

	/*
	 * Update the group state to account for the changes.
	 */
	phyint_group_refresh_state(pg);
	return (retval);
}

/*
 * Return the information associated with consecutive probe successes
 * starting with the most recent probe. At most the last 2 probes can be
 * in the unacknowledged state. All previous probes have either failed
 * or succeeded.
 */
static void
probe_success_info(struct phyint_instance *pii, struct target *cur_tg,
    struct probe_success_count *psinfo)
{
	uint_t	i;
	struct probe_stats *pr_statp;
	uint_t most_recent;
	uint_t second_most_recent;
	boolean_t pi_found_failure = _B_FALSE;
	boolean_t tg_found_failure = _B_FALSE;
	uint_t now;
	uint_t timeout;
	struct target *tg;

	if (debug & D_FAILREP)
		logdebug("probe_success_info(%s)\n", pii->pii_name);

	bzero(psinfo, sizeof (*psinfo));
	now = getcurrenttime();

	/*
	 * Start with the most recent probe, and count the number
	 * of consecutive probe successes. Latch the number of successes
	 * on hitting a failure.
	 */
	most_recent = PROBE_INDEX_PREV(pii->pii_probe_next);
	second_most_recent = PROBE_INDEX_PREV(most_recent);

	for (i = most_recent; i != pii->pii_probe_next;
	    i = PROBE_INDEX_PREV(i)) {
		pr_statp = &pii->pii_probes[i];

		switch (pr_statp->pr_status) {
		case PR_UNACKED:
			/*
			 * Only the most recent 2 probes can be unacknowledged
			 */
			assert(i == most_recent || i == second_most_recent);

			tg = pr_statp->pr_target;
			assert(tg != NULL);
			/*
			 * The crtt could be zero for some reason,
			 * Eg. the phyint could be failed. If the crtt is
			 * not available use the value of the group's probe
			 * interval which is a worst case estimate.
			 */
			timeout = ns2ms(pr_statp->pr_hrtime_start);
			if (tg->tg_crtt != 0) {
				timeout += tg->tg_crtt;
			} else {
				timeout +=
				    pii->pii_phyint->pi_group->pg_probeint;
			}

			if (TIME_LT(timeout, now)) {
				/*
				 * We hit a failure. Latch the total number of
				 * recent consecutive successes.
				 */
				pr_statp->pr_time_lost = timeout;
				probe_chstate(pr_statp, pii, PR_LOST);
				pi_found_failure = _B_TRUE;
				if (cur_tg != NULL && tg == cur_tg) {
					/*
					 * We hit a failure for the desired
					 * target. Latch the number of recent
					 * consecutive successes for this target
					 */
					tg_found_failure = _B_TRUE;
				}
			}
			break;

		case PR_ACKED:
			/*
			 * Bump up the count of probe successes, if we
			 * have not seen any failure so far.
			 */
			if (!pi_found_failure)
				psinfo->ps_nsucc++;

			if (cur_tg != NULL && pr_statp->pr_target == cur_tg &&
			    !tg_found_failure) {
				psinfo->ps_nsucc_tg++;
			}

			/*
			 * Record the time of last success, if this is
			 * the most recent probe success.
			 */
			if (!psinfo->ps_tls_valid) {
				psinfo->ps_tls =
				    ns2ms(pr_statp->pr_hrtime_ackproc);
				psinfo->ps_tls_valid = _B_TRUE;
			}
			break;

		case PR_LOST:
			/*
			 * We hit a failure. Latch the total number of
			 * recent consecutive successes.
			 */
			pi_found_failure = _B_TRUE;
			if (cur_tg != NULL && pr_statp->pr_target == cur_tg) {
				/*
				 * We hit a failure for the desired target.
				 * Latch the number of recent consecutive
				 * successes for this target
				 */
				tg_found_failure = _B_TRUE;
			}
			break;

		default:
			return;

		}
	}
}

/*
 * Return the information associated with consecutive probe failures
 * starting with the most recent probe. Only the last 2 probes can be in the
 * unacknowledged state. All previous probes have either failed or succeeded.
 */
static void
probe_fail_info(struct phyint_instance *pii, struct target *cur_tg,
    struct probe_fail_count *pfinfo)
{
	int	i;
	struct probe_stats *pr_statp;
	boolean_t	tg_found_success = _B_FALSE;
	boolean_t	pi_found_success = _B_FALSE;
	int	most_recent;
	int	second_most_recent;
	uint_t	now;
	uint_t	timeout;
	struct	target *tg;

	if (debug & D_FAILREP)
		logdebug("probe_fail_info(%s)\n", pii->pii_name);

	bzero(pfinfo, sizeof (*pfinfo));
	now = getcurrenttime();

	/*
	 * Start with the most recent probe, and count the number
	 * of consecutive probe failures. Latch the number of failures
	 * on hitting a probe success.
	 */
	most_recent = PROBE_INDEX_PREV(pii->pii_probe_next);
	second_most_recent = PROBE_INDEX_PREV(most_recent);

	for (i = most_recent; i != pii->pii_probe_next;
	    i = PROBE_INDEX_PREV(i)) {
		pr_statp = &pii->pii_probes[i];

		assert(PR_STATUS_VALID(pr_statp->pr_status));

		switch (pr_statp->pr_status) {
		case PR_UNACKED:
			/*
			 * Only the most recent 2 probes can be unacknowledged
			 */
			assert(i == most_recent || i == second_most_recent);

			tg = pr_statp->pr_target;
			/*
			 * Target is guaranteed to exist in the unack. state
			 */
			assert(tg != NULL);
			/*
			 * The crtt could be zero for some reason,
			 * Eg. the phyint could be failed. If the crtt is
			 * not available use the group's probe interval,
			 * which is a worst case estimate.
			 */
			timeout = ns2ms(pr_statp->pr_hrtime_start);
			if (tg->tg_crtt != 0) {
				timeout += tg->tg_crtt;
			} else {
				timeout +=
				    pii->pii_phyint->pi_group->pg_probeint;
			}

			if (TIME_GT(timeout, now))
				break;

			pr_statp->pr_time_lost = timeout;
			probe_chstate(pr_statp, pii, PR_LOST);
			/* FALLTHRU */

		case PR_LOST:
			if (!pi_found_success) {
				pfinfo->pf_nfail++;
				pfinfo->pf_tff = pr_statp->pr_time_lost;
			}
			if (cur_tg != NULL && pr_statp->pr_target == cur_tg &&
			    !tg_found_success)  {
				pfinfo->pf_nfail_tg++;
			}
			break;

		default:
			/*
			 * We hit a success or unused slot. Latch the
			 * total number of recent consecutive failures.
			 */
			pi_found_success = _B_TRUE;
			if (cur_tg != NULL && pr_statp->pr_target == cur_tg) {
				/*
				 * We hit a success for the desired target.
				 * Latch the number of recent consecutive
				 * failures for this target
				 */
				tg_found_success = _B_TRUE;
			}
		}
	}
}

/*
 * Change the state of probe `pr' on phyint_instance `pii' to state `state'.
 */
void
probe_chstate(struct probe_stats *pr, struct phyint_instance *pii, int state)
{
	if (pr->pr_status == state)
		return;

	pr->pr_status = state;
	(void) probe_state_event(pr, pii);
}

/*
 * Check if the phyint has been repaired.  If no test address has been
 * configured, then consider the interface repaired if the link is up (unless
 * the link is flapping; see below).  Otherwise, look for proof of probes
 * being sent and received. If last NUM_PROBE_REPAIRS probes are fine on
 * either IPv4 or IPv6 instance, the phyint can be considered repaired.
 */
static boolean_t
phyint_repaired(struct phyint *pi)
{
	struct	probe_success_count psinfo;
	struct	phyint_instance *pii;
	struct	target *cur_tg;
	int	pr_ndx;
	uint_t	cur_time;

	if (debug & D_FAILREP)
		logdebug("phyint_repaired(%s)\n", pi->pi_name);

	if (LINK_DOWN(pi))
		return (_B_FALSE);

	/*
	 * If we don't have any test addresses and the link is up, then
	 * consider the interface repaired, unless we've received more than
	 * LINK_UP_PERMIN link up notifications in the last minute, in
	 * which case we keep the link down until we drop back below
	 * the threshold.
	 */
	if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) {
		cur_time = getcurrenttime();
		if ((pi->pi_whenup[pi->pi_whendx] == 0 ||
		    (cur_time - pi->pi_whenup[pi->pi_whendx]) > MSEC_PERMIN)) {
			pi->pi_lfmsg_printed = 0;
			return (_B_TRUE);
		}
		if (!pi->pi_lfmsg_printed) {
			logerr("The link has come up on %s more than %d times "
			    "in the last minute; disabling repair until it "
			    "stabilizes\n", pi->pi_name, LINK_UP_PERMIN);
			pi->pi_lfmsg_printed = 1;
		}

		return (_B_FALSE);
	}

	pii = pi->pi_v4;
	if (PROBE_CAPABLE(pii)) {
		pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
		cur_tg = pii->pii_probes[pr_ndx].pr_target;
		probe_success_info(pii, cur_tg, &psinfo);
		if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS ||
		    psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS)
			return (_B_TRUE);
	}

	pii = pi->pi_v6;
	if (PROBE_CAPABLE(pii)) {
		pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
		cur_tg = pii->pii_probes[pr_ndx].pr_target;
		probe_success_info(pii, cur_tg, &psinfo);
		if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS ||
		    psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS)
			return (_B_TRUE);
	}

	return (_B_FALSE);
}

/*
 * Used to set/clear phyint flags, by making a SIOCSLIFFLAGS call.
 */
boolean_t
change_pif_flags(struct phyint *pi, uint64_t set, uint64_t clear)
{
	int ifsock;
	struct lifreq lifr;
	uint64_t old_flags;

	if (debug & D_FAILREP) {
		logdebug("change_pif_flags(%s): set %llx clear %llx\n",
		    pi->pi_name, set, clear);
	}

	if (pi->pi_v4 != NULL)
		ifsock = ifsock_v4;
	else
		ifsock = ifsock_v6;

	/*
	 * Get the current flags from the kernel, and set/clear the
	 * desired phyint flags. Since we set only phyint flags, we can
	 * do it on either IPv4 or IPv6 instance.
	 */
	(void) strlcpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name));

	if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) {
		if (errno != ENXIO)
			logperror("change_pif_flags: ioctl (get flags)");
		return (_B_FALSE);
	}

	old_flags = lifr.lifr_flags;
	lifr.lifr_flags |= set;
	lifr.lifr_flags &= ~clear;

	if (old_flags == lifr.lifr_flags) {
		/* No change in the flags. No need to send ioctl */
		return (_B_TRUE);
	}

	if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) {
		if (errno != ENXIO)
			logperror("change_pif_flags: ioctl (set flags)");
		return (_B_FALSE);
	}

	/*
	 * Keep pi_flags in synch. with actual flags. Assumes flags are
	 * phyint flags.
	 */
	pi->pi_flags |= set;
	pi->pi_flags &= ~clear;

	if (pi->pi_v4 != NULL)
		pi->pi_v4->pii_flags = pi->pi_flags;

	if (pi->pi_v6 != NULL)
		pi->pi_v6->pii_flags = pi->pi_flags;

	return (_B_TRUE);
}

/*
 * icmp cksum computation for IPv4.
 */
static int
in_cksum(ushort_t *addr, int len)
{
	register int nleft = len;
	register ushort_t *w = addr;
	register ushort_t answer;
	ushort_t odd_byte = 0;
	register int sum = 0;

	/*
	 *  Our algorithm is simple, using a 32 bit accumulator (sum),
	 *  we add sequential 16 bit words to it, and at the end, fold
	 *  back all the carry bits from the top 16 bits into the lower
	 *  16 bits.
	 */
	while (nleft > 1)  {
		sum += *w++;
		nleft -= 2;
	}

	/* mop up an odd byte, if necessary */
	if (nleft == 1) {
		*(uchar_t *)(&odd_byte) = *(uchar_t *)w;
		sum += odd_byte;
	}

	/*
	 * add back carry outs from top 16 bits to low 16 bits
	 */
	sum = (sum >> 16) + (sum & 0xffff);	/* add hi 16 to low 16 */
	sum += (sum >> 16);			/* add carry */
	answer = ~sum;				/* truncate to 16 bits */
	return (answer);
}

static void
reset_snxt_basetimes(void)
{
	struct phyint_instance *pii;

	for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) {
		pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
	}
}

/*
 * Is the address one of our own addresses? Unfortunately,
 * we cannot check our phyint tables to determine if the address
 * is our own. This is because, we don't track interfaces that
 * are not part of any group. We have to either use a 'bind' or
 * get the complete list of all interfaces using SIOCGLIFCONF,
 * to do this check. We could also use SIOCTMYADDR.
 * Bind fails for the local zone address, so we might include local zone
 * address as target address. If local zone address is a target address
 * and it is up, it is not possible to detect the interface failure.
 * SIOCTMYADDR also doesn't consider local zone address as own address.
 * So, we choose to use SIOCGLIFCONF to collect the local addresses, and they
 * are stored in `localaddrs'
 */
boolean_t
own_address(struct in6_addr addr)
{
	addrlist_t *addrp;
	struct sockaddr_storage ss;
	int af = IN6_IS_ADDR_V4MAPPED(&addr) ? AF_INET : AF_INET6;

	addr2storage(af, &addr, &ss);
	for (addrp = localaddrs; addrp != NULL; addrp = addrp->al_next) {
		if (sockaddrcmp(&ss, &addrp->al_addr))
			return (_B_TRUE);
	}
	return (_B_FALSE);
}

static int
ns2ms(int64_t ns)
{
	return (ns / (NANOSEC / MILLISEC));
}

static int64_t
tv2ns(struct timeval *tvp)
{
	return (tvp->tv_sec * NANOSEC + tvp->tv_usec * 1000);
}