xref: /onnv-gate/usr/src/uts/common/io/comstar/port/fcoet/fcoet.c (revision 12571:05943d9c379f)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * The following notice accompanied the original version of this file:
 *
 * BSD LICENSE
 *
 * Copyright(c) 2007 Intel Corporation. All rights reserved.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in
 *     the documentation and/or other materials provided with the
 *     distribution.
 *   * Neither the name of Intel Corporation nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Driver kernel header files
 */
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/pci.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/file.h>
#include <sys/cred.h>
#include <sys/byteorder.h>
#include <sys/atomic.h>
#include <sys/modhash.h>
#include <sys/scsi/scsi.h>
#include <sys/ethernet.h>

/*
 * COMSTAR header files
 */
#include <sys/stmf_defines.h>
#include <sys/fct_defines.h>
#include <sys/stmf.h>
#include <sys/portif.h>
#include <sys/fct.h>

/*
 * FCoE header files
 */
#include <sys/fcoe/fcoe_common.h>

/*
 * Driver's own header files
 */
#include "fcoet.h"
#include "fcoet_eth.h"
#include "fcoet_fc.h"

/*
 * static function forward declaration
 */
static int fcoet_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
static int fcoet_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
static int fcoet_open(dev_t *devp, int flag, int otype, cred_t *credp);
static int fcoet_close(dev_t dev, int flag, int otype, cred_t *credp);
static int fcoet_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
    cred_t *credp, int *rval);
static fct_status_t fcoet_attach_init(fcoet_soft_state_t *ss);
static fct_status_t fcoet_detach_uninit(fcoet_soft_state_t *ss);
static void fcoet_watchdog(void *arg);
static void fcoet_handle_sol_flogi(fcoet_soft_state_t *ss);
static stmf_data_buf_t *fcoet_dbuf_alloc(fct_local_port_t *port,
    uint32_t size, uint32_t *pminsize, uint32_t flags);
static void fcoet_dbuf_free(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf);
static int fcoet_dbuf_init(fcoet_soft_state_t *ss);
static void fcoet_dbuf_destroy(fcoet_soft_state_t *ss);
static uint_t
fcoet_sol_oxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg);
static uint_t
fcoet_unsol_rxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg);

/*
 * Driver identificaton stuff
 */
static struct cb_ops fcoet_cb_ops = {
	fcoet_open,
	fcoet_close,
	nodev,
	nodev,
	nodev,
	nodev,
	nodev,
	fcoet_ioctl,
	nodev,
	nodev,
	nodev,
	nochpoll,
	ddi_prop_op,
	0,
	D_MP | D_NEW
};

static struct dev_ops fcoet_ops = {
	DEVO_REV,
	0,
	nodev,
	nulldev,
	nulldev,
	fcoet_attach,
	fcoet_detach,
	nodev,
	&fcoet_cb_ops,
	NULL,
	ddi_power,
	ddi_quiesce_not_needed
};

static struct modldrv modldrv = {
	&mod_driverops,
	FCOET_MOD_NAME,
	&fcoet_ops,
};

static struct modlinkage modlinkage = {
	MODREV_1, &modldrv, NULL
};

/*
 * Driver's global variables
 */
static kmutex_t	 fcoet_mutex;
static void	*fcoet_state = NULL;

int fcoet_use_ext_log = 1;
static char				 fcoet_provider_name[] = "fcoet";
static struct stmf_port_provider	*fcoet_pp	= NULL;

/*
 * Common loadable module entry points _init, _fini, _info
 */

int
_init(void)
{
	int ret;

	ret = ddi_soft_state_init(&fcoet_state, sizeof (fcoet_soft_state_t), 0);
	if (ret == 0) {
		fcoet_pp = (stmf_port_provider_t *)
		    stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0);
		fcoet_pp->pp_portif_rev = PORTIF_REV_1;
		fcoet_pp->pp_name = fcoet_provider_name;
		if (stmf_register_port_provider(fcoet_pp) != STMF_SUCCESS) {
			stmf_free(fcoet_pp);
			ddi_soft_state_fini(&fcoet_state);
			return (EIO);
		}

		mutex_init(&fcoet_mutex, 0, MUTEX_DRIVER, 0);
		ret = mod_install(&modlinkage);
		if (ret) {
			(void) stmf_deregister_port_provider(fcoet_pp);
			stmf_free(fcoet_pp);
			mutex_destroy(&fcoet_mutex);
			ddi_soft_state_fini(&fcoet_state);
		}
	}

	FCOET_LOG("_init", "exit _init with %x", ret);
	return (ret);
}

int
_fini(void)
{
	int ret;

	ret = mod_remove(&modlinkage);
	if (ret == 0) {
		(void) stmf_deregister_port_provider(fcoet_pp);
		stmf_free(fcoet_pp);
		mutex_destroy(&fcoet_mutex);
		ddi_soft_state_fini(&fcoet_state);
	}

	FCOET_LOG("_fini", "exit _fini with %x", ret);
	return (ret);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * Autoconfiguration entry points: attach, detach, getinfo
 */

static int
fcoet_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int			 ret = DDI_FAILURE;
	int			 instance;
	fcoet_soft_state_t	*ss;

	instance = ddi_get_instance(dip);
	FCOET_LOG("fcoet_attach", "get instance %d", instance);

	switch (cmd) {
	case DDI_ATTACH:
		ret = ddi_soft_state_zalloc(fcoet_state, instance);
		if (ret != DDI_SUCCESS) {
			return (ret);
		}

		ss = ddi_get_soft_state(fcoet_state, instance);
		ss->ss_instance = instance;
		ss->ss_dip = dip;

		ret = fcoet_attach_init(ss);
		if (ret != FCOE_SUCCESS) {
			ddi_soft_state_free(fcoet_state, instance);
			ret = DDI_FAILURE;
		}

		FCOET_LOG("fcoet_attach", "end with-%x", ret);
		break;

	case DDI_RESUME:
		ret = DDI_SUCCESS;
		break;

	default:
		FCOET_LOG("fcoet_attach", "unspported attach cmd-%x", cmd);
		break;
	}

	return (ret);
}

static int
fcoet_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	int			 ret = DDI_FAILURE;
	int			 fcoe_ret;
	int			 instance;
	fcoet_soft_state_t	*ss;

	instance = ddi_get_instance(dip);
	ss = ddi_get_soft_state(fcoet_state, instance);
	if (ss == NULL) {
		return (ret);
	}

	switch (cmd) {
	case DDI_DETACH:
		fcoe_ret = fcoet_detach_uninit(ss);
		if (fcoe_ret == FCOE_SUCCESS) {
			ret = DDI_SUCCESS;
		}

		FCOET_LOG("fcoet_detach", "fcoet_detach_uninit end with-%x",
		    fcoe_ret);
		break;

	case DDI_SUSPEND:
		ret = DDI_SUCCESS;
		break;

	default:
		FCOET_LOG("fcoet_detach", "unsupported detach cmd-%x", cmd);
		break;
	}

	return (ret);
}

/*
 * Device access entry points
 */
static int
fcoet_open(dev_t *devp, int flag, int otype, cred_t *credp)
{
	int			 instance;
	fcoet_soft_state_t	*ss;

	if (otype != OTYP_CHR) {
		return (EINVAL);
	}

	/*
	 * Since this is for debugging only, only allow root to issue ioctl now
	 */
	if (drv_priv(credp)) {
		return (EPERM);
	}

	instance = (int)getminor(*devp);
	ss = ddi_get_soft_state(fcoet_state, instance);
	if (ss == NULL) {
		return (ENXIO);
	}

	mutex_enter(&ss->ss_ioctl_mutex);
	if (ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_EXCL) {
		/*
		 * It is already open for exclusive access.
		 * So shut the door on this caller.
		 */
		mutex_exit(&ss->ss_ioctl_mutex);
		return (EBUSY);
	}

	if (flag & FEXCL) {
		if (ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_OPEN) {
			/*
			 * Exclusive operation not possible
			 * as it is already opened
			 */
			mutex_exit(&ss->ss_ioctl_mutex);
			return (EBUSY);
		}
		ss->ss_ioctl_flags |= FCOET_IOCTL_FLAG_EXCL;
	}
	ss->ss_ioctl_flags |= FCOET_IOCTL_FLAG_OPEN;
	mutex_exit(&ss->ss_ioctl_mutex);

	return (0);
}

/* ARGSUSED */
static int
fcoet_close(dev_t dev, int flag, int otype, cred_t *credp)
{
	int			 instance;
	fcoet_soft_state_t	*ss;

	if (otype != OTYP_CHR) {
		return (EINVAL);
	}

	instance = (int)getminor(dev);
	ss = ddi_get_soft_state(fcoet_state, instance);
	if (ss == NULL) {
		return (ENXIO);
	}

	mutex_enter(&ss->ss_ioctl_mutex);
	if ((ss->ss_ioctl_flags & FCOET_IOCTL_FLAG_OPEN) == 0) {
		mutex_exit(&ss->ss_ioctl_mutex);
		return (ENODEV);
	}

	/*
	 * It looks there's one hole here, maybe there could several concurrent
	 * shareed open session, but we never check this case.
	 * But it will not hurt too much, disregard it now.
	 */
	ss->ss_ioctl_flags &= ~FCOET_IOCTL_FLAG_MASK;
	mutex_exit(&ss->ss_ioctl_mutex);

	return (0);
}

/* ARGSUSED */
static int
fcoet_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
    cred_t *credp, int *rval)
{
	fcoet_soft_state_t	*ss;
	int		 ret = 0;

	if (drv_priv(credp) != 0) {
		return (EPERM);
	}

	ss = ddi_get_soft_state(fcoet_state, (int32_t)getminor(dev));
	if (ss == NULL) {
		return (ENXIO);
	}

	switch (cmd) {
	default:
		FCOET_LOG("fcoet_ioctl", "ioctl-0x%02X", cmd);
		ret = ENOTTY;
		break;
	}

	*rval = ret;
	return (ret);
}

static fct_status_t
fcoet_attach_init(fcoet_soft_state_t *ss)
{
	fcoe_client_t		 client_fcoet;
	fcoe_port_t		*eport;
	fct_local_port_t	*port;
	fct_dbuf_store_t	*fds;
	char			 taskq_name[FCOET_TASKQ_NAME_LEN];
	int			 ret;

	/*
	 * FCoE (fcoe is fcoet's dependent driver)
	 * First we need register fcoet to FCoE as one client
	 */
	client_fcoet.ect_eport_flags = EPORT_FLAG_TGT_MODE |
	    EPORT_FLAG_IS_DIRECT_P2P;
	client_fcoet.ect_max_fc_frame_size = 2136;
	client_fcoet.ect_private_frame_struct_size = sizeof (fcoet_frame_t);
	client_fcoet.ect_rx_frame = fcoet_rx_frame;
	client_fcoet.ect_port_event = fcoet_port_event;
	client_fcoet.ect_release_sol_frame = fcoet_release_sol_frame;
	client_fcoet.ect_client_port_struct = ss;
	client_fcoet.ect_fcoe_ver = FCOE_VER_NOW;
	FCOET_LOG(__FUNCTION__, "version: %x %x", FCOE_VER_NOW, fcoe_ver_now);
	ret = ddi_prop_get_int(DDI_DEV_T_ANY, ss->ss_dip,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "mac_id", -1);
	if (ret == -1) {
		FCOET_LOG("fcoet_attach_init", "get mac_id failed");
		return (DDI_FAILURE);
	} else {
		client_fcoet.ect_channelid = ret;
	}
	FCOET_LOG("fcoet_attach_init", "channel_id is %d",
	    client_fcoet.ect_channelid);

	/*
	 * It's FCoE's responsiblity to initialize eport's all elements
	 */
	eport = fcoe_register_client(&client_fcoet);
	if (eport == NULL) {
		goto fail_register_client;
	}

	/*
	 * Now it's time to register local port to FCT
	 */
	if (fcoet_dbuf_init(ss) != FCOE_SUCCESS) {
		goto fail_init_dbuf;
	}

	fds = (fct_dbuf_store_t *)fct_alloc(FCT_STRUCT_DBUF_STORE, 0, 0);
	if (fds == NULL) {
		goto fail_alloc_dbuf;
	} else {
		fds->fds_alloc_data_buf = fcoet_dbuf_alloc;
		fds->fds_free_data_buf = fcoet_dbuf_free;
		fds->fds_fca_private = (void *)ss;
	}

	port = (fct_local_port_t *)fct_alloc(FCT_STRUCT_LOCAL_PORT, 0, 0);
	if (port == NULL) {
		goto fail_alloc_port;
	} else {
		/*
		 * Do ss's initialization now
		 */
		(void) snprintf(ss->ss_alias, sizeof (ss->ss_alias), "fcoet%d",
		    ss->ss_instance);
		ret = ddi_create_minor_node(ss->ss_dip, "admin",
		    S_IFCHR, ss->ss_instance, DDI_NT_STMF_PP, 0);
		if (ret != DDI_SUCCESS) {
			goto fail_minor_node;
		}

		ss->ss_state = FCT_STATE_OFFLINE;
		ss->ss_state_not_acked = 1;
		ss->ss_flags = 0;
		ss->ss_port = port;
		ss->ss_eport = eport;
		FCOE_SET_DEFAULT_FPORT_ADDR(eport->eport_efh_dst);

		ss->ss_rportid_in_dereg = 0;
		ss->ss_rport_dereg_state = 0;

		ss->ss_next_sol_oxid = 0xFFFF;
		ss->ss_next_unsol_rxid = 0xFFFF;
		ss->ss_sol_oxid_hash = mod_hash_create_idhash(
		    "ss_sol_oxid_hash", FCOET_SOL_HASH_SIZE,
		    mod_hash_null_valdtor);
		ss->ss_unsol_rxid_hash = mod_hash_create_idhash(
		    "ss_unsol_rxid_hash", FCOET_SOL_HASH_SIZE,
		    mod_hash_null_valdtor);

		ss->ss_watch_count = 0;
		mutex_init(&ss->ss_watch_mutex, 0, MUTEX_DRIVER, 0);
		cv_init(&ss->ss_watch_cv, NULL, CV_DRIVER, NULL);

		list_create(&ss->ss_abort_xchg_list, sizeof (fcoet_exchange_t),
		    offsetof(fcoet_exchange_t, xch_abort_node));

		ss->ss_sol_flogi = NULL;
		ss->ss_sol_flogi_state = SFS_WAIT_LINKUP;

		bzero(&ss->ss_link_info, sizeof (fct_link_info_t));

		ss->ss_ioctl_flags = 0;
		mutex_init(&ss->ss_ioctl_mutex, 0, MUTEX_DRIVER, 0);

		ss->ss_change_state_flags = 0;
	}

	/*
	 * Do port's initialization
	 *
	 * port_fct_private and port_lport have been initialized by fct_alloc
	 */
	port->port_fca_private = ss;
	port->port_fca_version = FCT_FCA_MODREV_1;
	bcopy(ss->ss_eport->eport_nodewwn, port->port_nwwn, 8);
	bcopy(ss->ss_eport->eport_portwwn, port->port_pwwn, 8);
	port->port_default_alias = ss->ss_alias;
	port->port_sym_node_name = NULL;
	port->port_sym_port_name = NULL;

	port->port_pp = fcoet_pp;

	port->port_hard_address = 0;
	port->port_max_logins = FCOET_MAX_LOGINS;
	port->port_max_xchges = FCOET_MAX_XCHGES;
	port->port_fca_fcp_cmd_size = sizeof (fcoet_exchange_t);
	port->port_fca_rp_private_size = 0;
	port->port_fca_sol_els_private_size = sizeof (fcoet_exchange_t);
	port->port_fca_sol_ct_private_size = sizeof (fcoet_exchange_t);

	port->port_fca_abort_timeout = 5 * 1000;	/* 5 seconds */
	port->port_fds = fds;

	port->port_get_link_info = fcoet_get_link_info;
	port->port_register_remote_port = fcoet_register_remote_port;
	port->port_deregister_remote_port = fcoet_deregister_remote_port;
	port->port_send_cmd = fcoet_send_cmd;
	port->port_xfer_scsi_data = fcoet_xfer_scsi_data;
	port->port_send_cmd_response = fcoet_send_cmd_response;
	port->port_abort_cmd = fcoet_abort_cmd;
	port->port_ctl = fcoet_ctl;
	port->port_flogi_xchg = fcoet_do_flogi;
	port->port_populate_hba_details = fcoet_populate_hba_fru_details;
	if (fct_register_local_port(port) != FCT_SUCCESS) {
		goto fail_register_port;
	}

	/*
	 * Start watchdog thread
	 */
	(void) snprintf(taskq_name, sizeof (taskq_name),
	    "stmf_fct_fcoet_%d_taskq", ss->ss_instance);
	if ((ss->ss_watchdog_taskq = ddi_taskq_create(NULL,
	    taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
		goto fail_create_taskq;
	}

	atomic_and_32(&ss->ss_flags, ~SS_FLAG_TERMINATE_WATCHDOG);
	(void) ddi_taskq_dispatch(ss->ss_watchdog_taskq,
	    fcoet_watchdog, ss, DDI_SLEEP);
	while ((ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) == 0) {
		delay(10);
	}

	ddi_report_dev(ss->ss_dip);
	return (DDI_SUCCESS);

fail_create_taskq:
	if (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
		atomic_or_32(&ss->ss_flags, SS_FLAG_TERMINATE_WATCHDOG);
		cv_broadcast(&ss->ss_watch_cv);
		while (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
			delay(10);
		}
	}

	ddi_taskq_destroy(ss->ss_watchdog_taskq);
	FCOET_LOG("fcoet_attach_init", "fail_register_port");

fail_register_port:
	mutex_destroy(&ss->ss_ioctl_mutex);
	mutex_destroy(&ss->ss_watch_mutex);
	cv_destroy(&ss->ss_watch_cv);
	mod_hash_destroy_hash(ss->ss_sol_oxid_hash);
	mod_hash_destroy_hash(ss->ss_unsol_rxid_hash);
	list_destroy(&ss->ss_abort_xchg_list);
	FCOET_LOG("fcoet_attach_init", "fail_create_taskq");

fail_minor_node:
	fct_free(port);
	FCOET_LOG("fcoet_attach_init", "fail_minor_node");

fail_alloc_port:
	fct_free(fds);
	FCOET_LOG("fcoet_attach_init", "fail_alloc_port");

fail_alloc_dbuf:
	fcoet_dbuf_destroy(ss);
	FCOET_LOG("fcoet_attach_init", "fail_alloc_dbuf");

fail_init_dbuf:
	ss->ss_eport->eport_deregister_client(ss->ss_eport);
	FCOET_LOG("fcoet_attach_init", "fail_init_dbuf");

fail_register_client:
	FCOET_LOG("fcoet_attach_init", "fail_register_client");
	return (DDI_FAILURE);
}

static fct_status_t
fcoet_detach_uninit(fcoet_soft_state_t *ss)
{
	if ((ss->ss_state != FCT_STATE_OFFLINE) ||
	    ss->ss_state_not_acked) {
		return (FCOE_FAILURE);
	}

	/*
	 * Avoid modunload before running fcinfo remove-target-port
	 */
	if (ss->ss_eport != NULL &&
	    ss->ss_eport->eport_flags & EPORT_FLAG_MAC_IN_USE) {
		return (FCOE_FAILURE);
	}

	if (ss->ss_port == NULL) {
		return (FCOE_SUCCESS);
	}

	ss->ss_sol_oxid_hash_empty = 1;
	ss->ss_unsol_rxid_hash_empty = 1;
	mod_hash_walk(ss->ss_sol_oxid_hash, fcoet_sol_oxid_hash_empty, ss);
	mod_hash_walk(ss->ss_unsol_rxid_hash, fcoet_unsol_rxid_hash_empty, ss);
	if ((!ss->ss_sol_oxid_hash_empty) || (!ss->ss_unsol_rxid_hash_empty)) {
		return (FCOE_FAILURE);
	}

	/*
	 * We need offline the port manually, before we want to detach it
	 * or it will not succeed.
	 */
	if (fct_deregister_local_port(ss->ss_port) != FCT_SUCCESS) {
		FCOET_LOG("fcoet_detach_uninit",
		    "fct_deregister_local_port failed");
		return (FCOE_FAILURE);
	}

	/*
	 * Stop watchdog
	 */
	if (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
		atomic_or_32(&ss->ss_flags, SS_FLAG_TERMINATE_WATCHDOG);
		cv_broadcast(&ss->ss_watch_cv);
		while (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
			delay(10);
		}
	}

	ddi_taskq_destroy(ss->ss_watchdog_taskq);

	/*
	 * Release all resources
	 */
	mutex_destroy(&ss->ss_ioctl_mutex);
	mutex_destroy(&ss->ss_watch_mutex);
	cv_destroy(&ss->ss_watch_cv);
	mod_hash_destroy_hash(ss->ss_sol_oxid_hash);
	mod_hash_destroy_hash(ss->ss_unsol_rxid_hash);
	list_destroy(&ss->ss_abort_xchg_list);

	fct_free(ss->ss_port->port_fds);
	fct_free(ss->ss_port);
	ss->ss_port = NULL;

	fcoet_dbuf_destroy(ss);

	if (ss->ss_eport != NULL &&
	    ss->ss_eport->eport_deregister_client != NULL) {
		ss->ss_eport->eport_deregister_client(ss->ss_eport);
	}
	ddi_soft_state_free(fcoet_state, ss->ss_instance);
	return (FCOE_SUCCESS);
}

static void
fcoet_watchdog(void *arg)
{
	fcoet_soft_state_t	*ss = (fcoet_soft_state_t *)arg;
	clock_t			 tmp_delay = 0;
	fcoet_exchange_t	*xchg, *xchg_next;

	FCOET_LOG("fcoet_watchdog", "fcoet_soft_state is %p", ss);

	mutex_enter(&ss->ss_watch_mutex);
	atomic_or_32(&ss->ss_flags, SS_FLAG_WATCHDOG_RUNNING);
	tmp_delay = STMF_SEC2TICK(1)/2;

	while ((ss->ss_flags & SS_FLAG_TERMINATE_WATCHDOG) == 0) {
		ss->ss_watch_count++;

		if (ss->ss_sol_flogi_state != SFS_FLOGI_DONE) {
			fcoet_handle_sol_flogi(ss);
		}
		for (xchg = list_head(&ss->ss_abort_xchg_list); xchg; ) {
			xchg_next = list_next(&ss->ss_abort_xchg_list, xchg);
			if (xchg->xch_ref == 0) {
				list_remove(&ss->ss_abort_xchg_list, xchg);
				mutex_exit(&ss->ss_watch_mutex);
				/* xchg abort done */
				if (xchg->xch_dbuf_num) {
					kmem_free((void*)xchg->xch_dbufs,
					    xchg->xch_dbuf_num *
					    sizeof (void *));
					xchg->xch_dbufs = NULL;
					xchg->xch_dbuf_num = 0;
				}
				fct_cmd_fca_aborted(xchg->xch_cmd,
				    FCT_ABORT_SUCCESS, FCT_IOF_FCA_DONE);
				mutex_enter(&ss->ss_watch_mutex);
			}
			xchg = xchg_next;
		}

		atomic_or_32(&ss->ss_flags, SS_FLAG_DOG_WAITING);
		(void) cv_reltimedwait(&ss->ss_watch_cv, &ss->ss_watch_mutex,
		    (clock_t)tmp_delay, TR_CLOCK_TICK);
		atomic_and_32(&ss->ss_flags, ~SS_FLAG_DOG_WAITING);
	}

	/*
	 * Ensure no ongoing FLOGI, before terminate the watchdog
	 */
	if (ss->ss_sol_flogi) {
		fcoet_clear_sol_exchange(ss->ss_sol_flogi);
		fct_free(ss->ss_sol_flogi->xch_cmd);
		ss->ss_sol_flogi = NULL;
	}

	atomic_and_32(&ss->ss_flags, ~SS_FLAG_WATCHDOG_RUNNING);
	mutex_exit(&ss->ss_watch_mutex);
}

static void
fcoet_handle_sol_flogi(fcoet_soft_state_t *ss)
{
	clock_t			twosec = STMF_SEC2TICK(2);

check_state_again:
	if (ss->ss_flags & SS_FLAG_PORT_DISABLED) {
		ss->ss_sol_flogi_state = SFS_WAIT_LINKUP;
	}

	switch (ss->ss_sol_flogi_state) {
	case SFS_WAIT_LINKUP:
		if (ss->ss_sol_flogi) {
			if (ss->ss_sol_flogi->xch_ref == 0) {
				fcoet_clear_sol_exchange(ss->ss_sol_flogi);
				fct_free(ss->ss_sol_flogi->xch_cmd);
				ss->ss_sol_flogi = NULL;
			}
		}
		break;

	case SFS_FLOGI_INIT:
		if (ss->ss_sol_flogi) {
			/*
			 * wait for the response to finish
			 */
			ss->ss_sol_flogi_state = SFS_CLEAR_FLOGI;
			break;
		}
		fcoet_send_sol_flogi(ss);
		ss->ss_sol_flogi_state++;
		break;

	case SFS_FLOGI_CHECK_TIMEOUT:
		if ((ss->ss_sol_flogi->xch_start_time + twosec) <
		    ddi_get_lbolt()) {
			ss->ss_sol_flogi_state++;
		}
		break;

	case SFS_ABTS_INIT:
		fcoet_send_sol_abts(ss->ss_sol_flogi);
		ss->ss_sol_flogi_state++;
		break;

	case SFS_CLEAR_FLOGI:
		if (ss->ss_sol_flogi) {
			if (ss->ss_sol_flogi->xch_ref) {
				break;
			}
			fcoet_clear_sol_exchange(ss->ss_sol_flogi);
			fct_free(ss->ss_sol_flogi->xch_cmd);
			ss->ss_sol_flogi = NULL;
		}
		ss->ss_sol_flogi_state = SFS_FLOGI_INIT;
		goto check_state_again;

	case SFS_FLOGI_ACC:
		ss->ss_sol_flogi_state++;
		goto check_state_again;

	case SFS_FLOGI_DONE:
		if (!(ss->ss_flags & SS_FLAG_PORT_DISABLED) &&
		    ss->ss_sol_flogi) {
			fcoet_clear_sol_exchange(ss->ss_sol_flogi);
			fct_free(ss->ss_sol_flogi->xch_cmd);
			ss->ss_sol_flogi = NULL;
		}

		/*
		 * We'd better to offline it first, and delay 0.1 seconds,
		 * before we say it's on again.
		 */
		fct_handle_event(ss->ss_port,
		    FCT_EVENT_LINK_DOWN, 0, NULL);
		delay(STMF_SEC2TICK(1)/10);
		fct_handle_event(ss->ss_port,
		    FCT_EVENT_LINK_UP, 0, NULL);
		break;

	default:
		ASSERT(0);
		break;
	}
}

/* ARGSUSED */
static int
fcoet_dbuf_init(fcoet_soft_state_t *ss)
{
	return (FCOE_SUCCESS);
}

/* ARGSUSED */
static void
fcoet_dbuf_destroy(fcoet_soft_state_t *ss)
{

}

/* ARGSUSED */
static stmf_data_buf_t *
fcoet_dbuf_alloc(fct_local_port_t *port, uint32_t size, uint32_t *pminsize,
    uint32_t flags)
{
	stmf_data_buf_t	*dbuf;
	int		 add_size;
	int		 sge_num;
	int		 sge_size;
	int		 idx;
	int		 ii;
	void		*netb;
	uint8_t		*fc_buf;
	fcoet_soft_state_t	*ss =
	    (fcoet_soft_state_t *)port->port_fca_private;

	if (size > FCOET_MAX_DBUF_LEN) {
		if (*pminsize > FCOET_MAX_DBUF_LEN) {
			return (NULL);
		}

		size = FCOET_MAX_DBUF_LEN;
	}

	sge_num = (size - 1) / ss->ss_fcp_data_payload_size + 1;
	add_size = (sge_num - 1) * sizeof (struct stmf_sglist_ent) +
	    sge_num * sizeof (mblk_t *);
	dbuf = stmf_alloc(STMF_STRUCT_DATA_BUF, add_size, 0);
	if (dbuf == NULL) {
		return (NULL);
	}
	dbuf->db_buf_size = size;
	dbuf->db_data_size = size;
	dbuf->db_sglist_length = 0;
	dbuf->db_flags |= DB_DONT_REUSE;
	FCOET_SET_SEG_NUM(dbuf, sge_num);

	/*
	 * Initialize non-last sg entries
	 */
	for (idx = 0; idx < sge_num - 1; idx++) {
		sge_size = ss->ss_fcp_data_payload_size;
		netb = ss->ss_eport->eport_alloc_netb(
		    ss->ss_eport, sizeof (fcoe_fc_frame_header_t) +
		    sge_size, &fc_buf);
		if (netb == NULL) {
			for (ii = 0; ii < idx; ii++) {
				ss->ss_eport->eport_free_netb(
				    FCOET_GET_NETB(dbuf, ii));
			}
			stmf_free(dbuf);
			FCOET_LOG("fcoe_dbuf_alloc", "no netb");
			return (NULL);
		}
		FCOET_SET_NETB(dbuf, idx, netb);
		dbuf->db_sglist[idx].seg_addr = fc_buf +
		    sizeof (fcoe_fc_frame_header_t);
		dbuf->db_sglist[idx].seg_length = sge_size;
	}

	/*
	 * Initialize the last sg entry
	 */
	if (size % ss->ss_fcp_data_payload_size) {
		sge_size = P2ROUNDUP(size % ss->ss_fcp_data_payload_size, 4);
	} else {
		sge_size = ss->ss_fcp_data_payload_size;
	}

	netb = ss->ss_eport->eport_alloc_netb(
	    ss->ss_eport,
	    sizeof (fcoe_fc_frame_header_t) +
	    sge_size, &fc_buf);
	if (netb == NULL) {
		for (ii = 0; ii < idx; ii++) {
			ss->ss_eport->eport_free_netb(
			    FCOET_GET_NETB(dbuf, ii));
		}
		stmf_free(dbuf);
		FCOET_LOG("fcoe_dbuf_alloc", "no netb");
		return (NULL);
	}

	FCOET_SET_NETB(dbuf, idx, netb);
	dbuf->db_sglist[idx].seg_addr = fc_buf +
	    sizeof (fcoe_fc_frame_header_t);
	dbuf->db_sglist[idx].seg_length = sge_size;

	/*
	 * Let COMSTAR know how many sg entries we will use
	 */
	dbuf->db_sglist_length = idx + 1;

	return (dbuf);
}

static void
fcoet_dbuf_free(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf)
{
	int	idx;
	fcoet_soft_state_t	*ss =
	    (fcoet_soft_state_t *)fds->fds_fca_private;

	for (idx = 0; idx < FCOET_GET_SEG_NUM(dbuf); idx++) {
		if (FCOET_GET_NETB(dbuf, idx)) {
			ss->ss_eport->eport_free_netb(
			    FCOET_GET_NETB(dbuf, idx));
		}
	}

	stmf_free(dbuf);
}

/*
 * We should have initialized fcoe_frame_t before
 */
void
fcoet_init_tfm(fcoe_frame_t *frm, fcoet_exchange_t *xch)
{
	FRM2TFM(frm)->tfm_fcoe_frame = frm;
	FRM2TFM(frm)->tfm_xch = xch;
	FRM2TFM(frm)->tfm_seq = NULL;
}

/* ARGSUSED */
static uint_t
fcoet_sol_oxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
{
	fcoet_soft_state_t	*ss = (fcoet_soft_state_t *)arg;

	ss->ss_sol_oxid_hash_empty = 0;
	FCOET_LOG("fcoet_sol_oxid_hash_empty", "one ongoing xch: %p", val);
	return (MH_WALK_CONTINUE);
}

/* ARGSUSED */
static uint_t
fcoet_unsol_rxid_hash_empty(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
{
	fcoet_soft_state_t	*ss = (fcoet_soft_state_t *)arg;

	ss->ss_sol_oxid_hash_empty = 0;
	FCOET_LOG("fcoet_unsol_rxid_hash_empty", "one ongoing xch: %p", val);
	return (MH_WALK_CONTINUE);
}

/* ARGSUSED */
void
fcoet_modhash_find_cb(mod_hash_key_t key, mod_hash_val_t val)
{
	ASSERT(val != NULL);
	fcoet_exchange_t *xch = (fcoet_exchange_t *)val;
	FCOET_BUSY_XCHG(xch);
}