xref: /onnv-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 12618:0e5eaf4bf546)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/types.h>
#include <fmadm.h>
#include <errno.h>
#include <limits.h>
#include <strings.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <fm/fmd_log.h>
#include <sys/fm/protocol.h>
#include <fm/libtopo.h>
#include <fm/fmd_adm.h>
#include <fm/fmd_msg.h>
#include <dlfcn.h>
#include <sys/systeminfo.h>
#include <sys/utsname.h>
#include <libintl.h>
#include <locale.h>
#include <sys/smbios.h>
#include <libdevinfo.h>
#include <stdlib.h>

#define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))

/*
 * Fault records are added to catalog by calling add_fault_record_to_catalog()
 * records are stored in order of importance to the system.
 * If -g flag is set or not_suppressed is not set and the class fru, fault,
 * type are the same then details are merged into an existing record, with uuid
 * records are stored in time order.
 * For each record information is extracted from nvlist and merged into linked
 * list each is checked for identical records for which percentage certainty are
 * added together.
 * print_catalog() is called to print out catalog and release external resources
 *
 *                         /---------------\
 *	status_rec_list -> |               | -|
 *                         \---------------/
 *                                \/
 *                         /---------------\    /-------\    /-------\
 *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
 *            \/           |               | |- |       | <- |       |
 *      /-------------\    |               |    \-------/    \-------/
 *      |             | -> |               |       \/           \/
 *      \-------------/    |               |    /-------\    /-------\
 *            \/           |               | -> | asru  | -> | asru  |
 *            ---          |               |    |       | <- |       |
 *                         |               |    \-------/    \-------/
 *      status_asru_list   |  class        |
 *            \/           |  resource     |    /-------\    /-------\
 *      /-------------\    |  fru          | -> | list  | -> | list  |
 *      |             | -> |  serial       |    |       | <- |       |
 *      \-------------/    |               |    \-------/    \-------/
 *            \/           \---------------/
 *            ---               \/    /\
 *                         /---------------\
 *                         | status_record |
 *                         \---------------/
 *
 * Fmadm faulty takes a number of options which affect the format of the
 * output displayed. By default, the display reports the FRU and ASRU along
 * with other information on per-case basis as in the example below.
 *
 * --------------- ------------------------------------  -------------- -------
 * TIME            EVENT-ID                              MSG-ID         SEVERITY
 * --------------- ------------------------------------  -------------- -------
 * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
 *
 * Fault class	: fault.memory.dimm_sb
 * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
 *		    faulted but still in service
 * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
 *		    faulty
 *
 * Description	: The number of errors associated with this memory module has
 *		exceeded acceptable levels.  Refer to
 *		http://sun.com/msg/AMD-8000-2F for more information.
 *
 * Response	: Pages of memory associated with this memory module are being
 *		removed from service as errors are reported.
 *
 * Impact	: Total system memory capacity will be reduced as pages are
 *		retired.
 *
 * Action	: Schedule a repair procedure to replace the affected memory
 *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
 *
 * The -v flag is similar, but adds some additonal information such as the
 * resource. The -s flag is also similar but just gives the top line summary.
 * All these options (ie without the -f or -r flags) use the print_catalog()
 * function to do the display.
 *
 * The -f flag changes the output so that it appears sorted on a per-fru basis.
 * The output is somewhat cut down compared to the default output. If -f is
 * used, then print_fru() is used to print the output.
 *
 * -----------------------------------------------------------------------------
 * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
 * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
 *
 * Description	: A problem was detected for a PCI device.
 *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
 *
 * Response	: One or more device instances may be disabled
 *
 * Impact	: Possible loss of services provided by the device instances
 *		associated with this fault
 *
 * Action	: Schedule a repair procedure to replace the affected device.
 * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
 *		Sun for support.
 *
 * The -r flag changes the output so that it appears sorted on a per-asru basis.
 * The output is very much cut down compared to the default output, just giving
 * the asru fmri and state. Here print_asru() is used to print the output.
 *
 * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
 *
 * For all fmadm faulty options, the sequence of events is
 *
 * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
 * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
 * This will extract the data from the nvlist and call catalog_new_record() to
 * save the data away in various linked lists in the catalogue.
 *
 * 2) Once this is done, the data can be supplemented by using
 * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
 *
 * 3) Finally print_catalog(), print_fru() or print_asru() are called as
 * appropriate to display the information from the catalogue sorted in the
 * requested way.
 *
 */

typedef struct name_list {
	struct name_list *next;
	struct name_list *prev;
	char *name;
	uint8_t pct;
	uint8_t max_pct;
	ushort_t count;
	int status;
	char *label;
} name_list_t;

typedef struct ari_list {
	char *ari_uuid;
	struct ari_list *next;
} ari_list_t;

typedef struct uurec {
	struct uurec *next;
	struct uurec *prev;
	char *uuid;
	ari_list_t *ari_uuid_list;
	name_list_t *asru;
	uint64_t sec;
	nvlist_t *event;
} uurec_t;

typedef struct uurec_select {
	struct uurec_select *next;
	char *uuid;
} uurec_select_t;

typedef struct host_id {
	char *chassis;
	char *server;
	char *platform;
	char *domain;
	char *product_sn;
} hostid_t;

typedef struct host_id_list {
	hostid_t hostid;
	struct host_id_list *next;
} host_id_list_t;

typedef struct status_record {
	hostid_t *host;
	int nrecs;
	uurec_t *uurec;
	char *severity;			/* in C locale */
	char *msgid;
	name_list_t *class;
	name_list_t *resource;
	name_list_t *asru;
	name_list_t *fru;
	name_list_t *serial;
	uint8_t not_suppressed;
	uint8_t injected;
} status_record_t;

typedef struct sr_list {
	struct sr_list *next;
	struct sr_list *prev;
	struct status_record *status_record;
} sr_list_t;

typedef struct resource_list {
	struct resource_list *next;
	struct resource_list *prev;
	sr_list_t *status_rec_list;
	char *resource;
	uint8_t not_suppressed;
	uint8_t injected;
	uint8_t max_pct;
} resource_list_t;

sr_list_t *status_rec_list;
resource_list_t *status_fru_list;
resource_list_t *status_asru_list;

static int max_display;
static int max_fault = 0;
static topo_hdl_t *topo_handle;
static host_id_list_t *host_list;
static int n_server;
static int opt_g;
static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */

static char *
format_date(char *buf, size_t len, uint64_t sec)
{
	if (sec > LONG_MAX) {
		(void) fprintf(stderr,
		    "record time is too large for 32-bit utility\n");
		(void) snprintf(buf, len, "0x%llx", sec);
	} else {
		time_t tod = (time_t)sec;
		time_t now = time(NULL);
		if (tod > now+60 ||
		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
			(void) strftime(buf, len, "%b %d %Y    ",
			    localtime(&tod));
		} else {
			(void) strftime(buf, len, "%b %d %T", localtime(&tod));
		}
	}

	return (buf);
}

static hostid_t *
find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
    char *product_sn)
{
	hostid_t *rt = NULL;
	host_id_list_t *hostp;

	if (platform == NULL)
		platform = "-";
	if (server == NULL)
		server = "-";
	hostp = host_list;
	while (hostp) {
		if (hostp->hostid.platform &&
		    strcmp(hostp->hostid.platform, platform) == 0 &&
		    hostp->hostid.server &&
		    strcmp(hostp->hostid.server, server) == 0 &&
		    (chassis == NULL || hostp->hostid.chassis == NULL ||
		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
		    (domain == NULL || hostp->hostid.domain == NULL ||
		    strcmp(domain, hostp->hostid.domain) == 0)) {
			rt = &hostp->hostid;
			break;
		}
		hostp = hostp->next;
	}
	if (rt == NULL) {
		hostp = malloc(sizeof (host_id_list_t));
		hostp->hostid.platform = strdup(platform);
		hostp->hostid.product_sn =
		    product_sn ? strdup(product_sn) : NULL;
		hostp->hostid.server = strdup(server);
		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
		hostp->hostid.domain = domain ? strdup(domain) : NULL;
		hostp->next = host_list;
		host_list = hostp;
		rt = &hostp->hostid;
		n_server++;
	}
	return (rt);
}

static hostid_t *
find_hostid(nvlist_t *nvl)
{
	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
	char *product_sn = NULL;
	nvlist_t *auth, *fmri;
	hostid_t *rt = NULL;

	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
		    &platform);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
		    &product_sn);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
		    &chassis);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
		rt = find_hostid_in_list(platform, chassis, server,
		    domain, product_sn);
	}
	return (rt);
}

static char *
get_nvl2str_topo(nvlist_t *nvl)
{
	char *name = NULL;
	char *tname;
	int err;
	char *scheme = NULL;
	char *mod_name = NULL;
	char buf[128];

	if (topo_handle == NULL)
		topo_handle = topo_open(TOPO_VERSION, 0, &err);
	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
		name = strdup(tname);
		topo_hdl_strfree(topo_handle, tname);
	} else {
		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
		    mod_name) {
			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
			    scheme, mod_name);
			name = strdup(buf);
		}
	}
	return (name);
}

static int
set_priority(char *s)
{
	int rt = 0;

	if (s) {
		if (strcmp(s, "Minor") == 0)
			rt = 1;
		else if (strcmp(s, "Major") == 0)
			rt = 10;
		else if (strcmp(s, "Critical") == 0)
			rt = 100;
	}
	return (rt);
}

static int
cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
    uint8_t p2)
{
	int r1, r2;
	int rt;

	r1 = set_priority(s1);
	r2 = set_priority(s2);
	rt = r1 - r2;
	if (rt == 0) {
		if (t1 > t2)
			rt = 1;
		else if (t1 < t2)
			rt = -1;
		else
			rt = p1 - p2;
	}
	return (rt);
}

/*
 * merge two lists into one, by comparing enties in new and moving into list if
 * name is not there or free off memory for names which are already there
 * add_pct indicates if pct is the sum or highest pct
 */
static name_list_t *
merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
{
	name_list_t *lp, *np, *sp, *rt = NULL;
	int max_pct;

	rt = *list;
	np = new;
	while (np) {
		lp = *list;
		while (lp) {
			if (strcmp(lp->name, np->name) == 0)
				break;
			lp = lp->next;
			if (lp == *list)
				lp = NULL;
		}
		if (np->next == new)
			sp = NULL;
		else
			sp = np->next;
		if (lp) {
			lp->status |= (np->status & FM_SUSPECT_FAULTY);
			if (add_pct) {
				lp->pct += np->pct;
				lp->count += np->count;
			} else if (np->pct > lp->pct) {
				lp->pct = np->pct;
			}
			max_pct = np->max_pct;
			if (np->label)
				free(np->label);
			free(np->name);
			free(np);
			np = NULL;
			if (max_pct > lp->max_pct) {
				lp->max_pct = max_pct;
				if (lp->max_pct > lp->prev->max_pct &&
				    lp != *list) {
					lp->prev->next = lp->next;
					lp->next->prev = lp->prev;
					np = lp;
				}
			}
		}
		if (np) {
			lp = *list;
			if (lp) {
				if (np->max_pct > lp->max_pct) {
					np->next = lp;
					np->prev = lp->prev;
					lp->prev->next = np;
					lp->prev = np;
					*list = np;
					rt = np;
				} else {
					lp = lp->next;
					while (lp != *list &&
					    np->max_pct < lp->max_pct) {
						lp = lp->next;
					}
					np->next = lp;
					np->prev = lp->prev;
					lp->prev->next = np;
					lp->prev = np;
				}
			} else {
				*list = np;
				np->next = np;
				np->prev = np;
				rt = np;
			}
		}
		np = sp;
	}
	return (rt);
}

static name_list_t *
alloc_name_list(char *name, uint8_t pct)
{
	name_list_t *nlp;

	nlp = malloc(sizeof (*nlp));
	nlp->name = strdup(name);
	nlp->pct = pct;
	nlp->max_pct = pct;
	nlp->count = 1;
	nlp->next = nlp;
	nlp->prev = nlp;
	nlp->status = 0;
	nlp->label = NULL;
	return (nlp);
}

static status_record_t *
new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
    name_list_t *fru, name_list_t *asru, name_list_t *resource,
    name_list_t *serial, boolean_t not_suppressed,
    hostid_t *hostid, boolean_t injected)
{
	status_record_t *status_rec_p;

	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
	status_rec_p->nrecs = 1;
	status_rec_p->host = hostid;
	status_rec_p->uurec = uurec_p;
	uurec_p->next = NULL;
	uurec_p->prev = NULL;
	uurec_p->asru = asru;
	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
		status_rec_p->severity = strdup("unknown");
	status_rec_p->class = class;
	status_rec_p->fru = fru;
	status_rec_p->asru = asru;
	status_rec_p->resource = resource;
	status_rec_p->serial = serial;
	status_rec_p->msgid = strdup(msgid);
	status_rec_p->not_suppressed = not_suppressed;
	status_rec_p->injected = injected;
	return (status_rec_p);
}

/*
 * add record to given list maintaining order higher priority first.
 */
static void
add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
{
	sr_list_t *tp, *np, *sp;
	int order;
	uint64_t sec;

	np = malloc(sizeof (sr_list_t));
	np->status_record = status_rec_p;
	sec = status_rec_p->uurec->sec;
	if ((sp = *list_pp) == NULL) {
		*list_pp = np;
		np->next = np;
		np->prev = np;
	} else {
		/* insert new record in front of lower priority */
		tp = sp;
		order = cmp_priority(status_rec_p->severity,
		    sp->status_record->severity, sec,
		    tp->status_record->uurec->sec, 0, 0);
		if (order > 0) {
			*list_pp = np;
		} else {
			tp = sp->next;
			while (tp != sp &&
			    cmp_priority(status_rec_p->severity,
			    tp->status_record->severity, sec,
			    tp->status_record->uurec->sec, 0, 0)) {
				tp = tp->next;
			}
		}
		np->next = tp;
		np->prev = tp->prev;
		tp->prev->next = np;
		tp->prev = np;
	}
}

static void
add_resource(status_record_t *status_rec_p, resource_list_t **rp,
    resource_list_t *np)
{
	int order;
	uint64_t sec;
	resource_list_t *sp, *tp;
	status_record_t *srp;
	char *severity = status_rec_p->severity;

	add_rec_list(status_rec_p, &np->status_rec_list);
	if ((sp = *rp) == NULL) {
		np->next = np;
		np->prev = np;
		*rp = np;
	} else {
		/*
		 * insert new record in front of lower priority
		 */
		tp = sp->next;
		srp = sp->status_rec_list->status_record;
		sec = status_rec_p->uurec->sec;
		order = cmp_priority(severity, srp->severity, sec,
		    srp->uurec->sec, np->max_pct, sp->max_pct);
		if (order > 0) {
			*rp = np;
		} else {
			srp = tp->status_rec_list->status_record;
			while (tp != sp &&
			    cmp_priority(severity, srp->severity, sec,
			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
				tp = tp->next;
				srp = tp->status_rec_list->status_record;
			}
		}
		np->next = tp;
		np->prev = tp->prev;
		tp->prev->next = np;
		tp->prev = np;
	}
}

static void
add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
    resource_list_t **rpp)
{
	int order;
	resource_list_t *np, *end;
	status_record_t *srp;

	np = *rpp;
	end = np;
	while (np) {
		if (strcmp(fp->name, np->resource) == 0) {
			np->not_suppressed |= status_rec_p->not_suppressed;
			np->injected |= status_rec_p->injected;
			srp = np->status_rec_list->status_record;
			order = cmp_priority(status_rec_p->severity,
			    srp->severity, status_rec_p->uurec->sec,
			    srp->uurec->sec, fp->max_pct, np->max_pct);
			if (order > 0 && np != end) {
				/*
				 * remove from list and add again using
				 * new priority
				 */
				np->prev->next = np->next;
				np->next->prev = np->prev;
				add_resource(status_rec_p,
				    rpp, np);
			} else {
				add_rec_list(status_rec_p,
				    &np->status_rec_list);
			}
			break;
		}
		np = np->next;
		if (np == end) {
			np = NULL;
			break;
		}
	}
	if (np == NULL) {
		np = malloc(sizeof (resource_list_t));
		np->resource = fp->name;
		np->not_suppressed = status_rec_p->not_suppressed;
		np->injected = status_rec_p->injected;
		np->status_rec_list = NULL;
		np->max_pct = fp->max_pct;
		add_resource(status_rec_p, rpp, np);
	}
}

static void
add_list(status_record_t *status_rec_p, name_list_t *listp,
    resource_list_t **glistp)
{
	name_list_t *fp, *end;

	fp = listp;
	end = fp;
	while (fp) {
		add_resource_list(status_rec_p, fp, glistp);
		fp = fp->next;
		if (fp == end)
			break;
	}
}

/*
 * add record to rec, fru and asru lists.
 */
static void
catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
    name_list_t *fru, name_list_t *asru, name_list_t *resource,
    name_list_t *serial, boolean_t not_suppressed,
    hostid_t *hostid, boolean_t injected, boolean_t dummy_fru)
{
	status_record_t *status_rec_p;

	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
	    resource, serial, not_suppressed, hostid, injected);
	add_rec_list(status_rec_p, &status_rec_list);
	if (status_rec_p->fru && !dummy_fru)
		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
	if (status_rec_p->asru)
		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
}

static void
get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
{
	char *name;
	char *serial = NULL;
	char **lserial = NULL;
	uint64_t serint;
	name_list_t *nlp;
	int j;
	uint_t nelem;
	char buf[64];

	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
			    &serint) == 0) {
				(void) snprintf(buf, sizeof (buf), "%llX",
				    serint);
				nlp = alloc_name_list(buf, pct);
				(void) merge_name_list(serial_p, nlp, 1);
			}
		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
			if (nvlist_lookup_string_array(nvl,
			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
				nlp = alloc_name_list(lserial[0], pct);
				for (j = 1; j < nelem; j++) {
					name_list_t *n1lp;
					n1lp = alloc_name_list(lserial[j], pct);
					(void) merge_name_list(&nlp, n1lp, 1);
				}
				(void) merge_name_list(serial_p, nlp, 1);
			}
		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
			    &serial) == 0) {
				nlp = alloc_name_list(serial, pct);
				(void) merge_name_list(serial_p, nlp, 1);
			}
		}
	}
}

static void
extract_record_info(nvlist_t *nvl, name_list_t **class_p,
    name_list_t **fru_p, name_list_t **serial_p, name_list_t **resource_p,
    name_list_t **asru_p, boolean_t *dummy_fru, uint8_t status)
{
	nvlist_t *lfru, *lasru, *rsrc;
	name_list_t *nlp;
	char *name;
	uint8_t lpct = 0;
	char *lclass = NULL;
	char *label;

	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
		nlp = alloc_name_list(lclass, lpct);
		(void) merge_name_list(class_p, nlp, 1);
	}
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
		name = get_nvl2str_topo(lfru);
		if (name != NULL) {
			nlp = alloc_name_list(name, lpct);
			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
			    FM_SUSPECT_DEGRADED);
			free(name);
			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
			    &label) == 0)
				nlp->label = strdup(label);
			(void) merge_name_list(fru_p, nlp, 1);
		}
		get_serial_no(lfru, serial_p, lpct);
	} else if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0) {
		/*
		 * No FRU or resource. But we want to display the repair status
		 * somehow, so create a dummy FRU field.
		 */
		*dummy_fru = 1;
		nlp = alloc_name_list(dgettext("FMD", "None"), lpct);
		nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
		    FM_SUSPECT_DEGRADED);
		(void) merge_name_list(fru_p, nlp, 1);
	}
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
		name = get_nvl2str_topo(lasru);
		if (name != NULL) {
			nlp = alloc_name_list(name, lpct);
			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
			    FM_SUSPECT_ACQUITTED);
			free(name);
			(void) merge_name_list(asru_p, nlp, 1);
		}
		get_serial_no(lasru, serial_p, lpct);
	}
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
		name = get_nvl2str_topo(rsrc);
		if (name != NULL) {
			nlp = alloc_name_list(name, lpct);
			nlp->status = status;
			free(name);
			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
			    &label) == 0)
				nlp->label = strdup(label);
			(void) merge_name_list(resource_p, nlp, 1);
		}
	}
}

static void
add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
{
	char *msgid = "-";
	uint_t i, size = 0;
	name_list_t *class = NULL, *resource = NULL;
	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
	nvlist_t **nva;
	uint8_t *ba;
	uurec_t *uurec_p;
	hostid_t *host;
	boolean_t not_suppressed = 1;
	boolean_t any_present = 0;
	boolean_t injected = 0;
	boolean_t dummy_fru = 0;

	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
	    &not_suppressed);
	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_INJECTED, &injected);

	if (size != 0) {
		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
		    &nva, &size);
		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
		    &ba, &size);
		for (i = 0; i < size; i++) {
			extract_record_info(nva[i], &class, &fru, &serial,
			    &resource, &asru, &dummy_fru, ba[i]);
			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
			    (ba[i] & FM_SUSPECT_FAULTY))
				any_present = 1;
		}
		/*
		 * also suppress if no resources present
		 */
		if (any_present == 0)
			not_suppressed = 0;
	}

	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
	uurec_p->uuid = strdup(uuid);
	uurec_p->sec = sec;
	uurec_p->ari_uuid_list = NULL;
	uurec_p->event = NULL;
	(void) nvlist_dup(nvl, &uurec_p->event, 0);
	host = find_hostid(nvl);
	catalog_new_record(uurec_p, msgid, class, fru, asru,
	    resource, serial, not_suppressed, host, injected, dummy_fru);
}

static void
update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
{
	sr_list_t *srp;
	uurec_t *uurp;
	ari_list_t *ari_list;

	srp = status_rec_list;
	if (srp) {
		for (;;) {
			uurp = srp->status_record->uurec;
			while (uurp) {
				if (strcmp(uuid, uurp->uuid) == 0) {
					ari_list = (ari_list_t *)
					    malloc(sizeof (ari_list_t));
					ari_list->ari_uuid = strdup(ari_uuid);
					ari_list->next = uurp->ari_uuid_list;
					uurp->ari_uuid_list = ari_list;
					return;
				}
				uurp = uurp->next;
			}
			if (srp->next == status_rec_list)
				break;
			srp = srp->next;
		}
	}
}

static void
print_line(char *label, char *buf)
{
	char *cp, *ep, *wp;
	char c;
	int i;
	int lsz;
	char *padding;

	lsz = strlen(label);
	padding = malloc(lsz + 1);
	for (i = 0; i < lsz; i++)
		padding[i] = ' ';
	padding[i] = 0;
	cp = buf;
	ep = buf;
	c = *ep;
	(void) printf("\n");
	while (c) {
		i = lsz;
		wp = NULL;
		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
			if (c == ' ')
				wp = ep;
			else if (c == '\n') {
				i = 0;
				*ep = 0;
				do {
					ep++;
				} while ((c = *ep) != NULL && c == ' ');
				break;
			}
			ep++;
			i++;
		}
		if (i >= 80 && wp) {
			*wp = 0;
			ep = wp + 1;
			c = *ep;
		}
		(void) printf("%s%s\n", label, cp);
		cp = ep;
		label = padding;
	}
	free(padding);
}

static void
print_dict_info_line(nvlist_t *e, fmd_msg_item_t what, const char *linehdr)
{
	char *cp = fmd_msg_getitem_nv(fmadm_msghdl, NULL, e, what);

	if (cp) {
		print_line(dgettext("FMD", linehdr), cp);
		free(cp);
	}
}

static void
print_dict_info(nvlist_t *nvl)
{
	print_dict_info_line(nvl, FMD_MSG_ITEM_DESC, "Description : ");
	print_dict_info_line(nvl, FMD_MSG_ITEM_RESPONSE, "Response    : ");
	print_dict_info_line(nvl, FMD_MSG_ITEM_IMPACT, "Impact      : ");
	print_dict_info_line(nvl, FMD_MSG_ITEM_ACTION, "Action      : ");
}

static void
print_name(name_list_t *list, char *padding, int *np, int pct, int full)
{
	char *name;

	name = list->name;
	if (list->label) {
		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
		*np += 1;
	} else {
		(void) printf("%s %s", padding, name);
		*np += 1;
	}
	if (list->pct && pct > 0 && pct < 100) {
		if (list->count > 1) {
			if (full) {
				(void) printf(" %d @ %s %d%%\n", list->count,
				    dgettext("FMD", "max"),
				    list->max_pct);
			} else {
				(void) printf(" %s %d%%\n",
				    dgettext("FMD", "max"),
				    list->max_pct);
			}
		} else {
			(void) printf(" %d%%\n", list->pct);
		}
	} else {
		(void) printf("\n");
	}
}

static void
print_asru_status(int status, char *label)
{
	char *msg = NULL;

	switch (status) {
	case 0:
		msg = dgettext("FMD", "ok and in service");
		break;
	case FM_SUSPECT_DEGRADED:
		msg = dgettext("FMD", "service degraded, "
		    "but associated components no longer faulty");
		break;
	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
		msg = dgettext("FMD", "faulted but still "
		    "providing degraded service");
		break;
	case FM_SUSPECT_FAULTY:
		msg = dgettext("FMD", "faulted but still in service");
		break;
	case FM_SUSPECT_UNUSABLE:
		msg = dgettext("FMD", "out of service, "
		    "but associated components no longer faulty");
		break;
	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
		msg = dgettext("FMD", "faulted and taken out of service");
		break;
	default:
		break;
	}
	if (msg) {
		(void) printf("%s     %s\n", label, msg);
	}
}

static void
print_fru_status(int status, char *label)
{
	char *msg = NULL;

	if (status & FM_SUSPECT_NOT_PRESENT)
		msg = dgettext("FMD", "not present");
	else if (status & FM_SUSPECT_FAULTY)
		msg = dgettext("FMD", "faulty");
	else if (status & FM_SUSPECT_REPLACED)
		msg = dgettext("FMD", "replaced");
	else if (status & FM_SUSPECT_REPAIRED)
		msg = dgettext("FMD", "repair attempted");
	else if (status & FM_SUSPECT_ACQUITTED)
		msg = dgettext("FMD", "acquitted");
	else
		msg = dgettext("FMD", "removed");
	(void) printf("%s     %s\n", label, msg);
}

static void
print_rsrc_status(int status, char *label)
{
	char *msg = "";

	if (status & FM_SUSPECT_NOT_PRESENT)
		msg = dgettext("FMD", "not present");
	else if (status & FM_SUSPECT_FAULTY) {
		if (status & FM_SUSPECT_DEGRADED)
			msg = dgettext("FMD",
			    "faulted but still providing degraded service");
		else if (status & FM_SUSPECT_UNUSABLE)
			msg = dgettext("FMD",
			    "faulted and taken out of service");
		else
			msg = dgettext("FMD", "faulted but still in service");
	} else if (status & FM_SUSPECT_REPLACED)
		msg = dgettext("FMD", "replaced");
	else if (status & FM_SUSPECT_REPAIRED)
		msg = dgettext("FMD", "repair attempted");
	else if (status & FM_SUSPECT_ACQUITTED)
		msg = dgettext("FMD", "acquitted");
	else
		msg = dgettext("FMD", "removed");
	(void) printf("%s     %s\n", label, msg);
}

static void
print_name_list(name_list_t *list, char *label,
    int limit, int pct, void (func1)(int, char *), int full)
{
	char *name;
	char *padding;
	int i, j, l, n;
	name_list_t *end = list;

	l = strlen(label);
	padding = malloc(l + 1);
	for (i = 0; i < l; i++)
		padding[i] = ' ';
	padding[l] = 0;
	(void) printf("%s", label);
	name = list->name;
	if (list->label)
		(void) printf(" \"%s\" (%s)", list->label, name);
	else
		(void) printf(" %s", name);
	if (list->pct && pct > 0 && pct < 100) {
		if (list->count > 1) {
			if (full) {
				(void) printf(" %d @ %s %d%%\n", list->count,
				    dgettext("FMD", "max"), list->max_pct);
			} else {
				(void) printf(" %s %d%%\n",
				    dgettext("FMD", "max"), list->max_pct);
			}
		} else {
			(void) printf(" %d%%\n", list->pct);
		}
	} else {
		(void) printf("\n");
	}
	if (func1)
		func1(list->status, padding);
	n = 1;
	j = 0;
	while ((list = list->next) != end) {
		if (limit == 0 || n < limit) {
			print_name(list, padding, &n, pct, full);
			if (func1)
				func1(list->status, padding);
		} else
			j++;
	}
	if (j == 1) {
		print_name(list->prev, padding, &n, pct, full);
	} else if (j > 1) {
		(void) printf("%s... %d %s\n", padding, j,
		    dgettext("FMD", "more entries suppressed,"
		    " use -v option for full list"));
	}
	free(padding);
}

static int
asru_same_status(name_list_t *list)
{
	name_list_t *end = list;
	int status = list->status;

	while ((list = list->next) != end) {
		if (status == -1) {
			status = list->status;
			continue;
		}
		if (list->status != -1 && status != list->status) {
			status = -1;
			break;
		}
	}
	return (status);
}

static int
serial_in_fru(name_list_t *fru, name_list_t *serial)
{
	name_list_t *sp = serial;
	name_list_t *fp;
	int nserial = 0;
	int found = 0;
	char buf[128];

	while (sp) {
		fp = fru;
		nserial++;
		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
		buf[sizeof (buf) - 1] = 0;
		while (fp) {
			if (strstr(fp->name, buf) != NULL) {
				found++;
				break;
			}
			fp = fp->next;
			if (fp == fru)
				break;
		}
		sp = sp->next;
		if (sp == serial)
			break;
	}
	return (found == nserial ? 1 : 0);
}

static void
print_sup_record(status_record_t *srp, int opt_i, int full)
{
	char buf[32];
	uurec_t *uurp = srp->uurec;
	int n, j, k, max;
	int status;
	ari_list_t *ari_list;

	n = 0;
	max = max_fault;
	if (max < 0) {
		max = 0;
	}
	j = max / 2;
	max -= j;
	k = srp->nrecs - max;
	while ((uurp = uurp->next) != NULL) {
		if (full || n < j || n >= k || max_fault == 0 ||
		    srp->nrecs == max_fault+1) {
			if (opt_i) {
				ari_list = uurp->ari_uuid_list;
				while (ari_list) {
					(void) printf("%-15s %s\n",
					    format_date(buf, sizeof (buf),
					    uurp->sec), ari_list->ari_uuid);
					ari_list = ari_list->next;
				}
			} else {
				(void) printf("%-15s %s\n",
				    format_date(buf, sizeof (buf), uurp->sec),
				    uurp->uuid);
			}
		} else if (n == j)
			(void) printf("... %d %s\n", srp->nrecs - max_fault,
			    dgettext("FMD", "more entries suppressed"));
		n++;
	}
	(void) printf("\n");
	(void) printf("%s %s", dgettext("FMD", "Host        :"),
	    srp->host->server);
	if (srp->host->domain)
		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
		    srp->host->domain);
	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
	    srp->host->platform);
	(void) printf("\t%s %s", dgettext("FMD", "Chassis_id  :"),
	    srp->host->chassis ? srp->host->chassis : "");
	(void) printf("\n%s %s\n\n", dgettext("FMD", "Product_sn  :"),
	    srp->host->product_sn? srp->host->product_sn : "");
	if (srp->class)
		print_name_list(srp->class,
		    dgettext("FMD", "Fault class :"), 0, srp->class->pct,
		    NULL, full);
	if (srp->asru) {
		status = asru_same_status(srp->asru);
		if (status != -1) {
			print_name_list(srp->asru,
			    dgettext("FMD", "Affects     :"),
			    full ? 0 : max_display, 0, NULL, full);
			print_asru_status(status, "             ");
		} else
			print_name_list(srp->asru,
			    dgettext("FMD", "Affects     :"),
			    full ? 0 : max_display, 0, print_asru_status, full);
	}
	if (full || srp->fru == NULL || srp->asru == NULL) {
		if (srp->resource) {
			status = asru_same_status(srp->resource);
			if (status != -1) {
				print_name_list(srp->resource,
				    dgettext("FMD", "Problem in  :"),
				    full ? 0 : max_display, 0, NULL, full);
				print_rsrc_status(status, "             ");
			} else
				print_name_list(srp->resource,
				    dgettext("FMD", "Problem in  :"),
				    full ? 0 : max_display, 0,
				    print_rsrc_status, full);
		}
	}
	if (srp->fru) {
		status = asru_same_status(srp->fru);
		if (status != -1) {
			print_name_list(srp->fru, dgettext("FMD",
			    "FRU         :"), 0,
			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
			    NULL, full);
			print_fru_status(status, "             ");
		} else
			print_name_list(srp->fru, dgettext("FMD",
			    "FRU         :"), 0,
			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
			    print_fru_status, full);
	}
	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
	    !serial_in_fru(srp->asru, srp->serial)) {
		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
		    0, 0, NULL, full);
	}
	print_dict_info(srp->uurec->event);
	(void) printf("\n");
}

static void
print_status_record(status_record_t *srp, int summary, int opt_i, int full)
{
	char buf[32];
	uurec_t *uurp = srp->uurec;
	static int header = 0;
	char *head;
	ari_list_t *ari_list;

	if (!summary || !header) {
		if (opt_i) {
			head = "--------------- "
			    "------------------------------------  "
			    "-------------- ---------\n"
			    "TIME            CACHE-ID"
			    "                              MSG-ID"
			    "         SEVERITY\n--------------- "
			    "------------------------------------ "
			    " -------------- ---------";
		} else {
			head = "--------------- "
			    "------------------------------------  "
			    "-------------- ---------\n"
			    "TIME            EVENT-ID"
			    "                              MSG-ID"
			    "         SEVERITY\n--------------- "
			    "------------------------------------ "
			    " -------------- ---------";
		}
		(void) printf("%s\n", dgettext("FMD", head));
		header = 1;
	}
	if (opt_i) {
		ari_list = uurp->ari_uuid_list;
		while (ari_list) {
			(void) printf("%-15s %-37s %-14s %-9s %s\n",
			    format_date(buf, sizeof (buf), uurp->sec),
			    ari_list->ari_uuid, srp->msgid, srp->severity,
			    srp->injected ? dgettext("FMD", "injected") : "");
			ari_list = ari_list->next;
		}
	} else {
		(void) printf("%-15s %-37s %-14s %-9s %s\n",
		    format_date(buf, sizeof (buf), uurp->sec),
		    uurp->uuid, srp->msgid, srp->severity,
		    srp->injected ? dgettext("FMD", "injected") : "");
	}

	if (!summary)
		print_sup_record(srp, opt_i, full);
}

static void
print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
{
	status_record_t *srp;
	sr_list_t *slp;

	slp = status_rec_list;
	if (slp) {
		for (;;) {
			srp = slp->status_record;
			if (opt_a || srp->not_suppressed) {
				if (page_feed)
					(void) printf("\f\n");
				print_status_record(srp, summary, opt_i, full);
			}
			if (slp->next == status_rec_list)
				break;
			slp = slp->next;
		}
	}
}

static name_list_t *
find_fru(status_record_t *srp, char *resource)
{
	name_list_t *rt = NULL;
	name_list_t *fru = srp->fru;

	while (fru) {
		if (strcmp(resource, fru->name) == 0) {
			rt = fru;
			break;
		}
		fru = fru->next;
		if (fru == srp->fru)
			break;
	}
	return (rt);
}

static void
print_fru_line(name_list_t *fru, char *uuid)
{
	if (fru->pct == 100) {
		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
		    dgettext("FMD", "suspects in this FRU total certainty"),
		    100);
	} else {
		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
		    dgettext("FMD", "suspects in this FRU max certainty"),
		    fru->max_pct);
	}
}

static void
print_fru(int summary, int opt_a, int opt_i, int page_feed)
{
	resource_list_t *tp = status_fru_list;
	status_record_t *srp;
	sr_list_t *slp, *end;
	uurec_t *uurp;
	name_list_t *fru;
	int status;
	ari_list_t *ari_list;

	while (tp) {
		if (opt_a || tp->not_suppressed) {
			if (page_feed)
				(void) printf("\f\n");
			if (!summary)
				(void) printf("-----------------------------"
				    "---------------------------------------"
				    "----------\n");
			slp = tp->status_rec_list;
			end = slp;
			do {
				srp = slp->status_record;
				if (!srp->not_suppressed) {
					slp = slp->next;
					continue;
				}
				fru = find_fru(srp, tp->resource);
				if (fru) {
					if (fru->label)
						(void) printf("\"%s\" (%s) ",
						    fru->label, fru->name);
					else
						(void) printf("%s ",
						    fru->name);
					break;
				}
				slp = slp->next;
			} while (slp != end);

			slp = tp->status_rec_list;
			end = slp;
			status = 0;
			do {
				srp = slp->status_record;
				if (!srp->not_suppressed) {
					slp = slp->next;
					continue;
				}
				fru = srp->fru;
				while (fru) {
					if (strcmp(tp->resource,
					    fru->name) == 0)
						status |= fru->status;
					fru = fru->next;
					if (fru == srp->fru)
						break;
				}
				slp = slp->next;
			} while (slp != end);
			if (status & FM_SUSPECT_NOT_PRESENT)
				(void) printf(dgettext("FMD", "not present"));
			else if (status & FM_SUSPECT_FAULTY)
				(void) printf(dgettext("FMD", "faulty"));
			else if (status & FM_SUSPECT_REPLACED)
				(void) printf(dgettext("FMD", "replaced"));
			else if (status & FM_SUSPECT_REPAIRED)
				(void) printf(dgettext("FMD",
				    "repair attempted"));
			else if (status & FM_SUSPECT_ACQUITTED)
				(void) printf(dgettext("FMD", "acquitted"));
			else
				(void) printf(dgettext("FMD", "removed"));

			if (tp->injected)
				(void) printf(dgettext("FMD", " injected\n"));
			else
				(void) printf(dgettext("FMD", "\n"));

			slp = tp->status_rec_list;
			end = slp;
			do {
				srp = slp->status_record;
				if (!srp->not_suppressed) {
					slp = slp->next;
					continue;
				}
				uurp = srp->uurec;
				fru = find_fru(srp, tp->resource);
				if (fru) {
					if (opt_i) {
						ari_list = uurp->ari_uuid_list;
						while (ari_list) {
							print_fru_line(fru,
							    ari_list->ari_uuid);
							ari_list =
							    ari_list->next;
						}
					} else {
						print_fru_line(fru, uurp->uuid);
					}
				}
				slp = slp->next;
			} while (slp != end);
			if (!summary) {
				slp = tp->status_rec_list;
				end = slp;
				do {
					srp = slp->status_record;
					if (!srp->not_suppressed) {
						slp = slp->next;
						continue;
					}
					if (srp->serial &&
					    !serial_in_fru(srp->fru,
					    srp->serial)) {
						print_name_list(srp->serial,
						    dgettext("FMD",
						    "Serial ID.  :"),
						    0, 0, NULL, 1);
						break;
					}
					slp = slp->next;
				} while (slp != end);
			}
		}
		tp = tp->next;
		if (tp == status_fru_list)
			break;
	}
}

static void
print_asru(int opt_a)
{
	resource_list_t *tp = status_asru_list;
	status_record_t *srp;
	sr_list_t *slp, *end;
	char *msg;
	int status;
	name_list_t *asru;

	while (tp) {
		if (opt_a || tp->not_suppressed) {
			status = 0;
			slp = tp->status_rec_list;
			end = slp;
			do {
				srp = slp->status_record;
				if (!srp->not_suppressed) {
					slp = slp->next;
					continue;
				}
				asru = srp->asru;
				while (asru) {
					if (strcmp(tp->resource,
					    asru->name) == 0)
						status |= asru->status;
					asru = asru->next;
					if (asru == srp->asru)
						break;
				}
				slp = slp->next;
			} while (slp != end);
			switch (status) {
			case 0:
				msg = dgettext("FMD", "ok");
				break;
			case FM_SUSPECT_DEGRADED:
				msg = dgettext("FMD", "degraded");
				break;
			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
				msg = dgettext("FMD", "degraded");
				break;
			case FM_SUSPECT_FAULTY:
				msg = dgettext("FMD", "degraded");
				break;
			case FM_SUSPECT_UNUSABLE:
				msg = dgettext("FMD", "unknown");
				break;
			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
				msg = dgettext("FMD", "faulted");
				break;
			default:
				msg = "";
				break;
			}
			(void) printf("%-69s %s", tp->resource, msg);
			if (tp->injected)
				(void) printf(dgettext("FMD", " injected\n"));
			else
				(void) printf(dgettext("FMD", "\n"));
		}
		tp = tp->next;
		if (tp == status_asru_list)
			break;
	}
}

static int
uuid_in_list(char *uuid, uurec_select_t *uurecp)
{
	while (uurecp) {
		if (strcmp(uuid, uurecp->uuid) == 0)
			return (1);
		uurecp = uurecp->next;
	}
	return (0);
}

static int
dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
{
	int64_t *diag_time;
	uint_t nelem;
	int rt = 0;
	char *uuid = "-";
	uurec_select_t *uurecp = (uurec_select_t *)arg;

	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
	    &diag_time, &nelem) == 0 && nelem >= 2) {
		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
		    &uuid);
		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
			add_fault_record_to_catalog(acp->aci_event, *diag_time,
			    uuid);
	} else {
		rt = -1;
	}
	return (rt);
}

/*ARGSUSED*/
static int
dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
{
	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
	return (0);
}

static int
get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
{
	int rt = FMADM_EXIT_SUCCESS;

	/*
	 * These calls may fail with Protocol error if message payload is
	 * too big
	 */
	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
		die("failed to get case list from fmd");
	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
		die("failed to get case status from fmd");
	return (rt);
}

/*
 * fmadm faulty command
 *
 *	-a		show hidden fault records
 *	-f		show faulty fru's
 *	-g		force grouping of similar faults on the same fru
 *	-n		number of fault records to display
 *	-p		pipe output through pager
 *	-r		show faulty asru's
 *	-s		print summary of first fault
 *	-u		print listed uuid's only
 *	-v		full output
 */

int
cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
{
	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
	int opt_i = 0;
	char *pager;
	FILE *fp;
	int rt, c, stat;
	uurec_select_t *tp;
	uurec_select_t *uurecp = NULL;

	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
		switch (c) {
		case 'a':
			opt_a++;
			break;
		case 'f':
			opt_f++;
			break;
		case 'g':
			opt_g++;
			break;
		case 'i':
			opt_i++;
			break;
		case 'n':
			max_fault = atoi(optarg);
			break;
		case 'p':
			opt_p++;
			break;
		case 'r':
			opt_r++;
			break;
		case 's':
			opt_s++;
			break;
		case 'u':
			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
			tp->uuid = optarg;
			tp->next = uurecp;
			uurecp = tp;
			opt_a = 1;
			break;
		case 'v':
			opt_v++;
			break;
		default:
			return (FMADM_EXIT_USAGE);
		}
	}
	if (optind < argc)
		return (FMADM_EXIT_USAGE);

	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
		return (FMADM_EXIT_ERROR);
	rt = get_cases_from_fmd(adm, uurecp, opt_i);
	if (opt_p) {
		if ((pager = getenv("PAGER")) == NULL)
			pager = "/usr/bin/more";
		fp = popen(pager, "w");
		if (fp == NULL) {
			rt = FMADM_EXIT_ERROR;
			opt_p = 0;
		} else {
			(void) dup2(fileno(fp), 1);
			setbuf(stdout, NULL);
			(void) fclose(fp);
		}
	}
	max_display = max_fault;
	if (opt_f)
		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
	if (opt_r)
		print_asru(opt_a);
	if (opt_f == 0 && opt_r == 0)
		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
	fmd_msg_fini(fmadm_msghdl);
	if (topo_handle)
		topo_close(topo_handle);
	if (opt_p) {
		(void) fclose(stdout);
		(void) wait(&stat);
	}
	return (rt);
}

int
cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
{
	int i, status = FMADM_EXIT_SUCCESS;

	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
		return (FMADM_EXIT_USAGE);

	for (i = 1; i < argc; i++) {
		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
			warn("failed to flush %s", argv[i]);
			status = FMADM_EXIT_ERROR;
		} else
			note("flushed resource history for %s\n", argv[i]);
	}

	return (status);
}

int
cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
{
	int err;

	if (getopt(argc, argv, "") != EOF)
		return (FMADM_EXIT_USAGE);

	if (argc - optind != 1)
		return (FMADM_EXIT_USAGE);

	/*
	 * argument could be a uuid, an fmri (asru, fru or resource)
	 * or a label. Try uuid first, If that fails try the others.
	 */
	err = fmd_adm_case_repair(adm, argv[optind]);
	if (err != 0)
		err = fmd_adm_rsrc_repaired(adm, argv[optind]);

	if (err != 0)
		die("failed to record repair to %s", argv[optind]);

	note("recorded repair to %s\n", argv[optind]);
	return (FMADM_EXIT_SUCCESS);
}

int
cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
{
	int err;

	if (getopt(argc, argv, "") != EOF)
		return (FMADM_EXIT_USAGE);

	if (argc - optind != 1)
		return (FMADM_EXIT_USAGE);

	/*
	 * argument could be an fmri (asru, fru or resource) or a label.
	 */
	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
	if (err != 0)
		die("failed to record repair to %s", argv[optind]);

	note("recorded repair to of %s\n", argv[optind]);
	return (FMADM_EXIT_SUCCESS);
}

int
cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
{
	int err;

	if (getopt(argc, argv, "") != EOF)
		return (FMADM_EXIT_USAGE);

	if (argc - optind != 1)
		return (FMADM_EXIT_USAGE);

	/*
	 * argument could be an fmri (asru, fru or resource) or a label.
	 */
	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
	if (err != 0)
		die("failed to record replacement of %s", argv[optind]);

	note("recorded replacement of %s\n", argv[optind]);
	return (FMADM_EXIT_SUCCESS);
}

int
cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
{
	int err;

	if (getopt(argc, argv, "") != EOF)
		return (FMADM_EXIT_USAGE);

	if (argc - optind != 1 && argc - optind != 2)
		return (FMADM_EXIT_USAGE);

	/*
	 * argument could be a uuid, an fmri (asru, fru or resource)
	 * or a label. Or it could be a uuid and an fmri or label.
	 */
	if (argc - optind == 2) {
		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
		if (err != 0)
			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
			    argv[optind]);
	} else {
		err = fmd_adm_case_acquit(adm, argv[optind]);
		if (err != 0)
			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
	}

	if (err != 0)
		die("failed to record acquital of %s", argv[optind]);

	note("recorded acquital of %s\n", argv[optind]);
	return (FMADM_EXIT_SUCCESS);
}