xref: /onnv-gate/usr/src/cmd/fm/modules/common/fdd-msg/fdd_msg.c (revision 13040:6acc0c1efdf3)
110831SYanmin.Sun@Sun.COM /*
210831SYanmin.Sun@Sun.COM  * CDDL HEADER START
310831SYanmin.Sun@Sun.COM  *
410831SYanmin.Sun@Sun.COM  * The contents of this file are subject to the terms of the
510831SYanmin.Sun@Sun.COM  * Common Development and Distribution License (the "License").
610831SYanmin.Sun@Sun.COM  * You may not use this file except in compliance with the License.
710831SYanmin.Sun@Sun.COM  *
810831SYanmin.Sun@Sun.COM  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
910831SYanmin.Sun@Sun.COM  * or http://www.opensolaris.org/os/licensing.
1010831SYanmin.Sun@Sun.COM  * See the License for the specific language governing permissions
1110831SYanmin.Sun@Sun.COM  * and limitations under the License.
1210831SYanmin.Sun@Sun.COM  *
1310831SYanmin.Sun@Sun.COM  * When distributing Covered Code, include this CDDL HEADER in each
1410831SYanmin.Sun@Sun.COM  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1510831SYanmin.Sun@Sun.COM  * If applicable, add the following below this CDDL HEADER, with the
1610831SYanmin.Sun@Sun.COM  * fields enclosed by brackets "[]" replaced with your own identifying
1710831SYanmin.Sun@Sun.COM  * information: Portions Copyright [yyyy] [name of copyright owner]
1810831SYanmin.Sun@Sun.COM  *
1910831SYanmin.Sun@Sun.COM  * CDDL HEADER END
2010831SYanmin.Sun@Sun.COM  */
2110831SYanmin.Sun@Sun.COM 
2210831SYanmin.Sun@Sun.COM /*
23*13040SLuping.Quan@Oracle.COM  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
2410831SYanmin.Sun@Sun.COM  */
2510831SYanmin.Sun@Sun.COM 
2610831SYanmin.Sun@Sun.COM /*
27*13040SLuping.Quan@Oracle.COM  * FMA capability messenger
28*13040SLuping.Quan@Oracle.COM  *
29*13040SLuping.Quan@Oracle.COM  * fdd-msg module is called once when fmd starts up. It does the following
30*13040SLuping.Quan@Oracle.COM  * based on different scenarios
3110831SYanmin.Sun@Sun.COM  *
32*13040SLuping.Quan@Oracle.COM  * 1. If it's on a x86 platform, fdd-msg module sends fdd running on service
33*13040SLuping.Quan@Oracle.COM  * processor a message (ILOM) which indicates the Solaris host FMA capability.
34*13040SLuping.Quan@Oracle.COM  * The message is sent via the BMC driver (KCS interface) to the IPMI stack
35*13040SLuping.Quan@Oracle.COM  * of ILOM using the IPMI Sun OEM core tunnel command. The sub-command is
36*13040SLuping.Quan@Oracle.COM  * CORE_TUNNEL_SUBCMD_HOSTCAP. The IPMI stack posts an host FMA capability
37*13040SLuping.Quan@Oracle.COM  * event to the event manager upon receiving this message. fdd subscribes to
38*13040SLuping.Quan@Oracle.COM  * the event manager for this event. Upon receving this event, fdd will adjust
39*13040SLuping.Quan@Oracle.COM  * its configuration.
4010831SYanmin.Sun@Sun.COM  *
41*13040SLuping.Quan@Oracle.COM  * 2. If it's on a Sparc platform, fdd-msg module just exit for now.
4210831SYanmin.Sun@Sun.COM  */
4310831SYanmin.Sun@Sun.COM 
4410831SYanmin.Sun@Sun.COM #include <errno.h>
4510831SYanmin.Sun@Sun.COM #include <stdio.h>
4610831SYanmin.Sun@Sun.COM #include <strings.h>
4711944SLuping.Quan@Sun.COM #include <sys/systeminfo.h>
4810831SYanmin.Sun@Sun.COM #include <libipmi.h>
49*13040SLuping.Quan@Oracle.COM #include <sys/devfm.h>
5010831SYanmin.Sun@Sun.COM #include <fm/fmd_api.h>
51*13040SLuping.Quan@Oracle.COM #if defined(__x86)
52*13040SLuping.Quan@Oracle.COM #include <sys/x86_archext.h>
53*13040SLuping.Quan@Oracle.COM #include <fm/fmd_agent.h>
54*13040SLuping.Quan@Oracle.COM #include <libnvpair.h>
55*13040SLuping.Quan@Oracle.COM #endif
5610831SYanmin.Sun@Sun.COM 
5710831SYanmin.Sun@Sun.COM #define	CMD_SUNOEM_CORE_TUNNEL 0x44
5810831SYanmin.Sun@Sun.COM #define	CORE_TUNNEL_SUBCMD_HOSTFMACAP 2
5910831SYanmin.Sun@Sun.COM #define	OEM_DATA_LENGTH 3
6010831SYanmin.Sun@Sun.COM #define	VERSION 0x10
61*13040SLuping.Quan@Oracle.COM 
62*13040SLuping.Quan@Oracle.COM #if defined(__x86)
63*13040SLuping.Quan@Oracle.COM typedef struct cpu_tbl {
64*13040SLuping.Quan@Oracle.COM 	char vendor[X86_VENDOR_STRLEN];
65*13040SLuping.Quan@Oracle.COM 	int32_t family;
66*13040SLuping.Quan@Oracle.COM 	int32_t model;
67*13040SLuping.Quan@Oracle.COM 	char *propname;
68*13040SLuping.Quan@Oracle.COM } cpu_tbl_t;
69*13040SLuping.Quan@Oracle.COM 
70*13040SLuping.Quan@Oracle.COM static cpu_tbl_t fma_cap_list[] = {
71*13040SLuping.Quan@Oracle.COM 	{"GenuineIntel", 6, 26, "NHMEP_fma_cap"},
72*13040SLuping.Quan@Oracle.COM 	{"GenuineIntel", 6, 46, "NHMEX_fma_cap"},
73*13040SLuping.Quan@Oracle.COM 	{"GenuineIntel", 6, 44, "WSMEP_fma_cap"},
74*13040SLuping.Quan@Oracle.COM 	{"GenuineIntel", 6, 47, "INTLN_fma_cap"},
75*13040SLuping.Quan@Oracle.COM 	{NULL, 0, 0, NULL}
76*13040SLuping.Quan@Oracle.COM };
77*13040SLuping.Quan@Oracle.COM #endif
7810831SYanmin.Sun@Sun.COM 
7910831SYanmin.Sun@Sun.COM static int
check_sunoem(ipmi_handle_t * ipmi_hdl)8010831SYanmin.Sun@Sun.COM check_sunoem(ipmi_handle_t *ipmi_hdl)
8110831SYanmin.Sun@Sun.COM {
8210831SYanmin.Sun@Sun.COM 	ipmi_deviceid_t *devid;
8310831SYanmin.Sun@Sun.COM 
8410831SYanmin.Sun@Sun.COM 	if ((devid = ipmi_get_deviceid(ipmi_hdl)) == NULL)
8510831SYanmin.Sun@Sun.COM 		return (-1);
8610831SYanmin.Sun@Sun.COM 
8710831SYanmin.Sun@Sun.COM 	if (!ipmi_is_sun_ilom(devid))
8810831SYanmin.Sun@Sun.COM 		return (-2);
8910831SYanmin.Sun@Sun.COM 
9010831SYanmin.Sun@Sun.COM 	return (0);
9110831SYanmin.Sun@Sun.COM }
9210831SYanmin.Sun@Sun.COM 
93*13040SLuping.Quan@Oracle.COM #if defined(__x86)
94*13040SLuping.Quan@Oracle.COM static int32_t
fma_cap_cpu_info(cpu_tbl_t * ci)95*13040SLuping.Quan@Oracle.COM fma_cap_cpu_info(cpu_tbl_t *ci)
96*13040SLuping.Quan@Oracle.COM {
97*13040SLuping.Quan@Oracle.COM 	nvlist_t **cpus, *nvl;
98*13040SLuping.Quan@Oracle.COM 	uint_t ncpu, i;
99*13040SLuping.Quan@Oracle.COM 	fmd_agent_hdl_t *hdl;
100*13040SLuping.Quan@Oracle.COM 	char *ven;
101*13040SLuping.Quan@Oracle.COM 	int32_t family, model;
102*13040SLuping.Quan@Oracle.COM 
103*13040SLuping.Quan@Oracle.COM 	if ((hdl = fmd_agent_open(FMD_AGENT_VERSION)) == NULL)
104*13040SLuping.Quan@Oracle.COM 		return (-1);
105*13040SLuping.Quan@Oracle.COM 	if (fmd_agent_physcpu_info(hdl, &cpus, &ncpu) != 0) {
106*13040SLuping.Quan@Oracle.COM 		fmd_agent_close(hdl);
107*13040SLuping.Quan@Oracle.COM 		return (-1);
108*13040SLuping.Quan@Oracle.COM 	}
109*13040SLuping.Quan@Oracle.COM 	fmd_agent_close(hdl);
110*13040SLuping.Quan@Oracle.COM 
111*13040SLuping.Quan@Oracle.COM 	if (cpus == NULL)
112*13040SLuping.Quan@Oracle.COM 		return (-1);
113*13040SLuping.Quan@Oracle.COM 
114*13040SLuping.Quan@Oracle.COM 	/*
115*13040SLuping.Quan@Oracle.COM 	 * There is no mixed CPU type on x86 systems, it's ok to
116*13040SLuping.Quan@Oracle.COM 	 * just pick the first one
117*13040SLuping.Quan@Oracle.COM 	 */
118*13040SLuping.Quan@Oracle.COM 	nvl = cpus[0];
119*13040SLuping.Quan@Oracle.COM 	if (nvlist_lookup_string(nvl, FM_PHYSCPU_INFO_VENDOR_ID, &ven) != 0 ||
120*13040SLuping.Quan@Oracle.COM 	    nvlist_lookup_int32(nvl, FM_PHYSCPU_INFO_FAMILY, &family) != 0 ||
121*13040SLuping.Quan@Oracle.COM 	    nvlist_lookup_int32(nvl, FM_PHYSCPU_INFO_MODEL, &model) != 0) {
122*13040SLuping.Quan@Oracle.COM 		for (i = 0; i < ncpu; i++)
123*13040SLuping.Quan@Oracle.COM 			nvlist_free(cpus[i]);
124*13040SLuping.Quan@Oracle.COM 		umem_free(cpus, sizeof (nvlist_t *) * ncpu);
125*13040SLuping.Quan@Oracle.COM 		return (-1);
126*13040SLuping.Quan@Oracle.COM 	}
127*13040SLuping.Quan@Oracle.COM 
128*13040SLuping.Quan@Oracle.COM 	(void) snprintf(ci->vendor, X86_VENDOR_STRLEN, "%s", ven);
129*13040SLuping.Quan@Oracle.COM 	ci->family = family;
130*13040SLuping.Quan@Oracle.COM 	ci->model = model;
131*13040SLuping.Quan@Oracle.COM 
132*13040SLuping.Quan@Oracle.COM 	for (i = 0; i < ncpu; i++)
133*13040SLuping.Quan@Oracle.COM 		nvlist_free(cpus[i]);
134*13040SLuping.Quan@Oracle.COM 	umem_free(cpus, sizeof (nvlist_t *) * ncpu);
135*13040SLuping.Quan@Oracle.COM 	return (0);
136*13040SLuping.Quan@Oracle.COM }
137*13040SLuping.Quan@Oracle.COM #endif
138*13040SLuping.Quan@Oracle.COM 
139*13040SLuping.Quan@Oracle.COM static uint32_t
get_cap_conf(fmd_hdl_t * hdl)140*13040SLuping.Quan@Oracle.COM get_cap_conf(fmd_hdl_t *hdl)
141*13040SLuping.Quan@Oracle.COM {
142*13040SLuping.Quan@Oracle.COM 	uint32_t fma_cap;
143*13040SLuping.Quan@Oracle.COM #if defined(__x86)
144*13040SLuping.Quan@Oracle.COM 	int found = 0;
145*13040SLuping.Quan@Oracle.COM 	cpu_tbl_t *cl, ci;
146*13040SLuping.Quan@Oracle.COM 
147*13040SLuping.Quan@Oracle.COM 	if (fma_cap_cpu_info(&ci) == 0) {
148*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Got CPU info: vendor=%s, family=%d, "
149*13040SLuping.Quan@Oracle.COM 		    "model=%d\n", ci.vendor, ci.family, ci.model);
150*13040SLuping.Quan@Oracle.COM 		for (cl = fma_cap_list; cl->propname != NULL; cl++) {
151*13040SLuping.Quan@Oracle.COM 			if (strncmp(ci.vendor, cl->vendor,
152*13040SLuping.Quan@Oracle.COM 			    X86_VENDOR_STRLEN) == 0 &&
153*13040SLuping.Quan@Oracle.COM 			    ci.family == cl->family &&
154*13040SLuping.Quan@Oracle.COM 			    ci.model == cl->model) {
155*13040SLuping.Quan@Oracle.COM 				found++;
156*13040SLuping.Quan@Oracle.COM 				break;
157*13040SLuping.Quan@Oracle.COM 			}
158*13040SLuping.Quan@Oracle.COM 		}
159*13040SLuping.Quan@Oracle.COM 	} else {
160*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Failed to get CPU info");
161*13040SLuping.Quan@Oracle.COM 	}
162*13040SLuping.Quan@Oracle.COM 
163*13040SLuping.Quan@Oracle.COM 	if (found) {
164*13040SLuping.Quan@Oracle.COM 		fma_cap = fmd_prop_get_int32(hdl, cl->propname);
165*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Found property, FMA capability=0x%x",
166*13040SLuping.Quan@Oracle.COM 		    fma_cap);
167*13040SLuping.Quan@Oracle.COM 	} else {
168*13040SLuping.Quan@Oracle.COM #endif
169*13040SLuping.Quan@Oracle.COM 		fma_cap = fmd_prop_get_int32(hdl, "default_fma_cap");
170*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Didn't find FMA capability property, "
171*13040SLuping.Quan@Oracle.COM 		    "use default=0x%x", fma_cap);
172*13040SLuping.Quan@Oracle.COM #if defined(__x86)
173*13040SLuping.Quan@Oracle.COM 	}
174*13040SLuping.Quan@Oracle.COM #endif
175*13040SLuping.Quan@Oracle.COM 
176*13040SLuping.Quan@Oracle.COM 	return (fma_cap);
177*13040SLuping.Quan@Oracle.COM }
178*13040SLuping.Quan@Oracle.COM 
17910831SYanmin.Sun@Sun.COM static void
send_fma_cap_to_ilom(fmd_hdl_t * hdl,uint32_t fma_cap)180*13040SLuping.Quan@Oracle.COM send_fma_cap_to_ilom(fmd_hdl_t *hdl, uint32_t fma_cap)
18110831SYanmin.Sun@Sun.COM {
182*13040SLuping.Quan@Oracle.COM 	int error;
183*13040SLuping.Quan@Oracle.COM 	char *msg;
18410831SYanmin.Sun@Sun.COM 	ipmi_handle_t *ipmi_hdl;
18510831SYanmin.Sun@Sun.COM 	ipmi_cmd_t cmd;
18610831SYanmin.Sun@Sun.COM 	uint8_t oem_data[OEM_DATA_LENGTH];
18710831SYanmin.Sun@Sun.COM 
18811756SRobert.Johnston@Sun.COM 	if ((ipmi_hdl = ipmi_open(&error, &msg, IPMI_TRANSPORT_BMC, NULL))
18911756SRobert.Johnston@Sun.COM 	    == NULL) {
19010831SYanmin.Sun@Sun.COM 		/*
191*13040SLuping.Quan@Oracle.COM 		 * If /dev/bmc doesn't exist on the system, then return
192*13040SLuping.Quan@Oracle.COM 		 * without doing anything.
19310831SYanmin.Sun@Sun.COM 		 */
19410831SYanmin.Sun@Sun.COM 		if (error != EIPMI_BMC_OPEN_FAILED)
19510831SYanmin.Sun@Sun.COM 			fmd_hdl_abort(hdl, "Failed to initialize IPMI "
19610831SYanmin.Sun@Sun.COM 			    "connection: %s\n", msg);
197*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Failed: no IPMI connection present");
19810831SYanmin.Sun@Sun.COM 		return;
19910831SYanmin.Sun@Sun.COM 	}
20010831SYanmin.Sun@Sun.COM 
20110831SYanmin.Sun@Sun.COM 	/*
20210831SYanmin.Sun@Sun.COM 	 * Check if it's Sun ILOM
20310831SYanmin.Sun@Sun.COM 	 */
20410831SYanmin.Sun@Sun.COM 	if (check_sunoem(ipmi_hdl) != 0) {
20510831SYanmin.Sun@Sun.COM 		fmd_hdl_debug(hdl, "Service Processor does not run "
20610831SYanmin.Sun@Sun.COM 		    "Sun ILOM");
20710831SYanmin.Sun@Sun.COM 		ipmi_close(ipmi_hdl);
20810831SYanmin.Sun@Sun.COM 		return;
20910831SYanmin.Sun@Sun.COM 	}
21010831SYanmin.Sun@Sun.COM 
211*13040SLuping.Quan@Oracle.COM 	oem_data[0] = CORE_TUNNEL_SUBCMD_HOSTFMACAP;
212*13040SLuping.Quan@Oracle.COM 	oem_data[1] = VERSION;
213*13040SLuping.Quan@Oracle.COM 	oem_data[2] = fma_cap;
214*13040SLuping.Quan@Oracle.COM 
215*13040SLuping.Quan@Oracle.COM 	cmd.ic_netfn = IPMI_NETFN_OEM;
216*13040SLuping.Quan@Oracle.COM 	cmd.ic_lun = 0;
217*13040SLuping.Quan@Oracle.COM 	cmd.ic_cmd = CMD_SUNOEM_CORE_TUNNEL;
218*13040SLuping.Quan@Oracle.COM 	cmd.ic_dlen = OEM_DATA_LENGTH;
219*13040SLuping.Quan@Oracle.COM 	cmd.ic_data = oem_data;
220*13040SLuping.Quan@Oracle.COM 
221*13040SLuping.Quan@Oracle.COM 	if (ipmi_send(ipmi_hdl, &cmd) == NULL) {
222*13040SLuping.Quan@Oracle.COM 		fmd_hdl_debug(hdl, "Failed to send Solaris FMA "
223*13040SLuping.Quan@Oracle.COM 		    "capability to ilom: %s", ipmi_errmsg(ipmi_hdl));
224*13040SLuping.Quan@Oracle.COM 	}
225*13040SLuping.Quan@Oracle.COM 
226*13040SLuping.Quan@Oracle.COM 	ipmi_close(ipmi_hdl);
227*13040SLuping.Quan@Oracle.COM }
228*13040SLuping.Quan@Oracle.COM 
229*13040SLuping.Quan@Oracle.COM /*ARGSUSED*/
230*13040SLuping.Quan@Oracle.COM static void
fma_cap_init(fmd_hdl_t * hdl,id_t id,void * data)231*13040SLuping.Quan@Oracle.COM fma_cap_init(fmd_hdl_t *hdl, id_t id, void *data)
232*13040SLuping.Quan@Oracle.COM {
233*13040SLuping.Quan@Oracle.COM 	uint32_t	fma_cap;
234*13040SLuping.Quan@Oracle.COM 
235*13040SLuping.Quan@Oracle.COM 	fma_cap = get_cap_conf(hdl);
236*13040SLuping.Quan@Oracle.COM 	send_fma_cap_to_ilom(hdl, fma_cap);
237*13040SLuping.Quan@Oracle.COM 
238*13040SLuping.Quan@Oracle.COM 	fmd_hdl_unregister(hdl);
239*13040SLuping.Quan@Oracle.COM }
240*13040SLuping.Quan@Oracle.COM 
241*13040SLuping.Quan@Oracle.COM static const fmd_hdl_ops_t fmd_ops = {
242*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_recv */
243*13040SLuping.Quan@Oracle.COM 	fma_cap_init,	/* fmdo_timeout */
244*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_close */
245*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_stats */
246*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_gc */
247*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_send */
248*13040SLuping.Quan@Oracle.COM 	NULL,		/* fmdo_topo */
249*13040SLuping.Quan@Oracle.COM };
250*13040SLuping.Quan@Oracle.COM 
251*13040SLuping.Quan@Oracle.COM static const fmd_prop_t fmd_props[] = {
252*13040SLuping.Quan@Oracle.COM 	{ "interval", FMD_TYPE_TIME, "1s" },
253*13040SLuping.Quan@Oracle.COM 	{ "default_fma_cap", FMD_TYPE_UINT32, "0x3" },
254*13040SLuping.Quan@Oracle.COM 	{ "NHMEP_fma_cap", FMD_TYPE_UINT32, "0x3" },
255*13040SLuping.Quan@Oracle.COM 	{ "NHMEX_fma_cap", FMD_TYPE_UINT32, "0x2" },
256*13040SLuping.Quan@Oracle.COM 	{ "WSMEP_fma_cap", FMD_TYPE_UINT32, "0x3" },
257*13040SLuping.Quan@Oracle.COM 	{ "INTLN_fma_cap", FMD_TYPE_UINT32, "0x2" },
258*13040SLuping.Quan@Oracle.COM 	{ NULL, 0, NULL }
259*13040SLuping.Quan@Oracle.COM };
260*13040SLuping.Quan@Oracle.COM 
261*13040SLuping.Quan@Oracle.COM static const fmd_hdl_info_t fmd_info = {
262*13040SLuping.Quan@Oracle.COM 	"FMA Capability Messenger", "1.1", &fmd_ops, fmd_props
263*13040SLuping.Quan@Oracle.COM };
264*13040SLuping.Quan@Oracle.COM 
265*13040SLuping.Quan@Oracle.COM void
_fmd_init(fmd_hdl_t * hdl)266*13040SLuping.Quan@Oracle.COM _fmd_init(fmd_hdl_t *hdl)
267*13040SLuping.Quan@Oracle.COM {
268*13040SLuping.Quan@Oracle.COM 	char isa[8];
269*13040SLuping.Quan@Oracle.COM 
270*13040SLuping.Quan@Oracle.COM 	/*
271*13040SLuping.Quan@Oracle.COM 	 * For now the module only sends message to ILOM on i386 platforms
272*13040SLuping.Quan@Oracle.COM 	 * till CR 6933053 is fixed. Module unregister may cause etm module
273*13040SLuping.Quan@Oracle.COM 	 * core dump due to 6933053.
274*13040SLuping.Quan@Oracle.COM 	 */
275*13040SLuping.Quan@Oracle.COM 	if ((sysinfo(SI_ARCHITECTURE, isa, sizeof (isa)) == -1) ||
276*13040SLuping.Quan@Oracle.COM 	    (strncmp(isa, "i386", 4) != 0))
277*13040SLuping.Quan@Oracle.COM 		return;
278*13040SLuping.Quan@Oracle.COM 
279*13040SLuping.Quan@Oracle.COM 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0)
280*13040SLuping.Quan@Oracle.COM 		return;
28110831SYanmin.Sun@Sun.COM 
28210831SYanmin.Sun@Sun.COM 	/*
28310831SYanmin.Sun@Sun.COM 	 * Setup the timer.
28410831SYanmin.Sun@Sun.COM 	 */
28511944SLuping.Quan@Sun.COM 	(void) fmd_timer_install(hdl, NULL, NULL, 2000000000ULL);
28610831SYanmin.Sun@Sun.COM }
28710831SYanmin.Sun@Sun.COM 
288*13040SLuping.Quan@Oracle.COM /*ARGSUSED*/
28910831SYanmin.Sun@Sun.COM void
_fmd_fini(fmd_hdl_t * hdl)29010831SYanmin.Sun@Sun.COM _fmd_fini(fmd_hdl_t *hdl)
29110831SYanmin.Sun@Sun.COM {
29210831SYanmin.Sun@Sun.COM }
293