xref: /onnv-gate/usr/src/uts/common/io/ib/adapters/hermon/hermon_ci.c (revision 12965:b65a8427f8fe)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * hermon_ci.c
 *    Hermon Channel Interface (CI) Routines
 *
 *    Implements all the routines necessary to interface with the IBTF.
 *    Pointers to all of these functions are passed to the IBTF at attach()
 *    time in the ibc_operations_t structure.  These functions include all
 *    of the necessary routines to implement the required InfiniBand "verbs"
 *    and additional IBTF-specific interfaces.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

#include <sys/ib/adapters/hermon/hermon.h>

extern uint32_t hermon_kernel_data_ro;
extern uint32_t hermon_user_data_ro;

/* HCA and port related operations */
static ibt_status_t hermon_ci_query_hca_ports(ibc_hca_hdl_t, uint8_t,
    ibt_hca_portinfo_t *);
static ibt_status_t hermon_ci_modify_ports(ibc_hca_hdl_t, uint8_t,
    ibt_port_modify_flags_t, uint8_t);
static ibt_status_t hermon_ci_modify_system_image(ibc_hca_hdl_t, ib_guid_t);

/* Protection Domains */
static ibt_status_t hermon_ci_alloc_pd(ibc_hca_hdl_t, ibt_pd_flags_t,
    ibc_pd_hdl_t *);
static ibt_status_t hermon_ci_free_pd(ibc_hca_hdl_t, ibc_pd_hdl_t);

/* Reliable Datagram Domains */
static ibt_status_t hermon_ci_alloc_rdd(ibc_hca_hdl_t, ibc_rdd_flags_t,
    ibc_rdd_hdl_t *);
static ibt_status_t hermon_ci_free_rdd(ibc_hca_hdl_t, ibc_rdd_hdl_t);

/* Address Handles */
static ibt_status_t hermon_ci_alloc_ah(ibc_hca_hdl_t, ibt_ah_flags_t,
    ibc_pd_hdl_t, ibt_adds_vect_t *, ibc_ah_hdl_t *);
static ibt_status_t hermon_ci_free_ah(ibc_hca_hdl_t, ibc_ah_hdl_t);
static ibt_status_t hermon_ci_query_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
    ibc_pd_hdl_t *, ibt_adds_vect_t *);
static ibt_status_t hermon_ci_modify_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
    ibt_adds_vect_t *);

/* Queue Pairs */
static ibt_status_t hermon_ci_alloc_qp(ibc_hca_hdl_t, ibtl_qp_hdl_t,
    ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *, ib_qpn_t *,
    ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_special_qp(ibc_hca_hdl_t, uint8_t,
    ibtl_qp_hdl_t, ibt_sqp_type_t, ibt_qp_alloc_attr_t *,
    ibt_chan_sizes_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_qp_range(ibc_hca_hdl_t, uint_t,
    ibtl_qp_hdl_t *, ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *,
    ibc_cq_hdl_t *, ibc_cq_hdl_t *, ib_qpn_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_free_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibc_free_qp_flags_t, ibc_qpn_hdl_t *);
static ibt_status_t hermon_ci_release_qpn(ibc_hca_hdl_t, ibc_qpn_hdl_t);
static ibt_status_t hermon_ci_query_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_qp_query_attr_t *);
static ibt_status_t hermon_ci_modify_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_cep_modify_flags_t, ibt_qp_info_t *, ibt_queue_sizes_t *);

/* Completion Queues */
static ibt_status_t hermon_ci_alloc_cq(ibc_hca_hdl_t, ibt_cq_hdl_t,
    ibt_cq_attr_t *, ibc_cq_hdl_t *, uint_t *);
static ibt_status_t hermon_ci_free_cq(ibc_hca_hdl_t, ibc_cq_hdl_t);
static ibt_status_t hermon_ci_query_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t *, uint_t *, uint_t *, ibt_cq_handler_id_t *);
static ibt_status_t hermon_ci_resize_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t, uint_t *);
static ibt_status_t hermon_ci_modify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t, uint_t, ibt_cq_handler_id_t);
static ibt_status_t hermon_ci_alloc_cq_sched(ibc_hca_hdl_t,
    ibt_cq_sched_attr_t *, ibc_sched_hdl_t *);
static ibt_status_t hermon_ci_free_cq_sched(ibc_hca_hdl_t, ibc_sched_hdl_t);
static ibt_status_t hermon_ci_query_cq_handler_id(ibc_hca_hdl_t,
    ibt_cq_handler_id_t, ibt_cq_handler_attr_t *);

/* EE Contexts */
static ibt_status_t hermon_ci_alloc_eec(ibc_hca_hdl_t, ibc_eec_flags_t,
    ibt_eec_hdl_t, ibc_rdd_hdl_t, ibc_eec_hdl_t *);
static ibt_status_t hermon_ci_free_eec(ibc_hca_hdl_t, ibc_eec_hdl_t);
static ibt_status_t hermon_ci_query_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
    ibt_eec_query_attr_t *);
static ibt_status_t hermon_ci_modify_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
    ibt_cep_modify_flags_t, ibt_eec_info_t *);

/* Memory Registration */
static ibt_status_t hermon_ci_register_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_mr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_buf(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_smr_attr_t *, struct buf *, void *, ibt_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_shared_mr(ibc_hca_hdl_t,
    ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_smr_attr_t *, void *,
    ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_deregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t);
static ibt_status_t hermon_ci_query_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibt_mr_query_attr_t *);
static ibt_status_t hermon_ci_reregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibc_pd_hdl_t, ibt_mr_attr_t *, void *, ibc_mr_hdl_t *,
    ibt_mr_desc_t *);
static ibt_status_t hermon_ci_reregister_buf(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibc_pd_hdl_t, ibt_smr_attr_t *, struct buf *, void *, ibc_mr_hdl_t *,
    ibt_mr_desc_t *);
static ibt_status_t hermon_ci_sync_mr(ibc_hca_hdl_t, ibt_mr_sync_t *, size_t);
static ibt_status_t hermon_ci_register_dma_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_dmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);

/* Memory Windows */
static ibt_status_t hermon_ci_alloc_mw(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_mw_flags_t, ibc_mw_hdl_t *, ibt_rkey_t *);
static ibt_status_t hermon_ci_free_mw(ibc_hca_hdl_t, ibc_mw_hdl_t);
static ibt_status_t hermon_ci_query_mw(ibc_hca_hdl_t, ibc_mw_hdl_t,
    ibt_mw_query_attr_t *);

/* Multicast Groups */
static ibt_status_t hermon_ci_attach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ib_gid_t, ib_lid_t);
static ibt_status_t hermon_ci_detach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ib_gid_t, ib_lid_t);

/* Work Request and Completion Processing */
static ibt_status_t hermon_ci_post_send(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_send_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_recv(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_recv_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_poll_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    ibt_wc_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_notify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    ibt_cq_notify_flags_t);

/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_in(ibc_hca_hdl_t, ibt_ci_data_flags_t,
    ibt_object_type_t, void *, void *, size_t);

/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_out(ibc_hca_hdl_t, ibt_ci_data_flags_t,
    ibt_object_type_t, void *, void *, size_t);

/* Shared Receive Queues */
static ibt_status_t hermon_ci_alloc_srq(ibc_hca_hdl_t, ibt_srq_flags_t,
    ibt_srq_hdl_t, ibc_pd_hdl_t, ibt_srq_sizes_t *, ibc_srq_hdl_t *,
    ibt_srq_sizes_t *);
static ibt_status_t hermon_ci_free_srq(ibc_hca_hdl_t, ibc_srq_hdl_t);
static ibt_status_t hermon_ci_query_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibc_pd_hdl_t *, ibt_srq_sizes_t *, uint_t *);
static ibt_status_t hermon_ci_modify_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibt_srq_modify_flags_t, uint_t, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibt_recv_wr_t *, uint_t, uint_t *);

/* Address translation */
static ibt_status_t hermon_ci_map_mem_area(ibc_hca_hdl_t, ibt_va_attr_t *,
    void *, uint_t, ibt_reg_req_t *, ibc_ma_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_area(ibc_hca_hdl_t, ibc_ma_hdl_t);
static ibt_status_t hermon_ci_map_mem_iov(ibc_hca_hdl_t, ibt_iov_attr_t *,
    ibt_all_wr_t *, ibc_mi_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_iov(ibc_hca_hdl_t, ibc_mi_hdl_t);

/* Allocate L_Key */
static ibt_status_t hermon_ci_alloc_lkey(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_lkey_flags_t, uint_t, ibc_mr_hdl_t *, ibt_pmr_desc_t *);

/* Physical Register Memory Region */
static ibt_status_t hermon_ci_register_physical_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_pmr_desc_t *);
static ibt_status_t hermon_ci_reregister_physical_mr(ibc_hca_hdl_t,
    ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *,
    ibt_pmr_desc_t *);

/* Mellanox FMR */
static ibt_status_t hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca,
    ibc_pd_hdl_t pd, ibt_fmr_pool_attr_t *fmr_params,
    ibc_fmr_pool_hdl_t *fmr_pool);
static ibt_status_t hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
    void *ibtl_reserved, ibc_mr_hdl_t *mr_hdl_p, ibt_pmr_desc_t *mem_desc_p);
static ibt_status_t hermon_ci_deregister_fmr(ibc_hca_hdl_t hca,
    ibc_mr_hdl_t mr);

/* Memory Allocation/Deallocation */
static ibt_status_t hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size,
    ibt_mr_flags_t mr_flag, caddr_t *kaddrp,
    ibc_mem_alloc_hdl_t *mem_alloc_hdl_p);
static ibt_status_t hermon_ci_free_io_mem(ibc_hca_hdl_t hca,
    ibc_mem_alloc_hdl_t mem_alloc_hdl);
static ibt_status_t hermon_ci_not_supported();

/*
 * This ibc_operations_t structure includes pointers to all the entry points
 * provided by the Hermon driver.  This structure is passed to the IBTF at
 * driver attach time, using the ibc_attach() call.
 */
ibc_operations_t hermon_ibc_ops = {
	/* HCA and port related operations */
	hermon_ci_query_hca_ports,
	hermon_ci_modify_ports,
	hermon_ci_modify_system_image,

	/* Protection Domains */
	hermon_ci_alloc_pd,
	hermon_ci_free_pd,

	/* Reliable Datagram Domains */
	hermon_ci_alloc_rdd,
	hermon_ci_free_rdd,

	/* Address Handles */
	hermon_ci_alloc_ah,
	hermon_ci_free_ah,
	hermon_ci_query_ah,
	hermon_ci_modify_ah,

	/* Queue Pairs */
	hermon_ci_alloc_qp,
	hermon_ci_alloc_special_qp,
	hermon_ci_alloc_qp_range,
	hermon_ci_free_qp,
	hermon_ci_release_qpn,
	hermon_ci_query_qp,
	hermon_ci_modify_qp,

	/* Completion Queues */
	hermon_ci_alloc_cq,
	hermon_ci_free_cq,
	hermon_ci_query_cq,
	hermon_ci_resize_cq,
	hermon_ci_modify_cq,
	hermon_ci_alloc_cq_sched,
	hermon_ci_free_cq_sched,
	hermon_ci_query_cq_handler_id,

	/* EE Contexts */
	hermon_ci_alloc_eec,
	hermon_ci_free_eec,
	hermon_ci_query_eec,
	hermon_ci_modify_eec,

	/* Memory Registration */
	hermon_ci_register_mr,
	hermon_ci_register_buf,
	hermon_ci_register_shared_mr,
	hermon_ci_deregister_mr,
	hermon_ci_query_mr,
	hermon_ci_reregister_mr,
	hermon_ci_reregister_buf,
	hermon_ci_sync_mr,

	/* Memory Windows */
	hermon_ci_alloc_mw,
	hermon_ci_free_mw,
	hermon_ci_query_mw,

	/* Multicast Groups */
	hermon_ci_attach_mcg,
	hermon_ci_detach_mcg,

	/* Work Request and Completion Processing */
	hermon_ci_post_send,
	hermon_ci_post_recv,
	hermon_ci_poll_cq,
	hermon_ci_notify_cq,

	/* CI Object Mapping Data */
	hermon_ci_ci_data_in,
	hermon_ci_ci_data_out,

	/* Shared Receive Queue */
	hermon_ci_alloc_srq,
	hermon_ci_free_srq,
	hermon_ci_query_srq,
	hermon_ci_modify_srq,
	hermon_ci_post_srq,

	/* Address translation */
	hermon_ci_map_mem_area,
	hermon_ci_unmap_mem_area,
	hermon_ci_map_mem_iov,
	hermon_ci_unmap_mem_iov,

	/* Allocate L_key */
	hermon_ci_alloc_lkey,

	/* Physical Register Memory Region */
	hermon_ci_register_physical_mr,
	hermon_ci_reregister_physical_mr,

	/* Mellanox FMR */
	hermon_ci_create_fmr_pool,
	hermon_ci_destroy_fmr_pool,
	hermon_ci_flush_fmr_pool,
	hermon_ci_register_physical_fmr,
	hermon_ci_deregister_fmr,

	/* Memory allocation */
	hermon_ci_alloc_io_mem,
	hermon_ci_free_io_mem,

	/* XRC not yet supported */
	hermon_ci_not_supported,	/* ibc_alloc_xrc_domain */
	hermon_ci_not_supported,	/* ibc_free_xrc_domain */
	hermon_ci_not_supported,	/* ibc_alloc_xrc_srq */
	hermon_ci_not_supported,	/* ibc_free_xrc_srq */
	hermon_ci_not_supported,	/* ibc_query_xrc_srq */
	hermon_ci_not_supported,	/* ibc_modify_xrc_srq */
	hermon_ci_not_supported,	/* ibc_alloc_xrc_tgt_qp */
	hermon_ci_not_supported,	/* ibc_free_xrc_tgt_qp */
	hermon_ci_not_supported,	/* ibc_query_xrc_tgt_qp */
	hermon_ci_not_supported,	/* ibc_modify_xrc_tgt_qp */

	/* Memory Region (physical) */
	hermon_ci_register_dma_mr,

	/* Next enhancements */
	hermon_ci_not_supported,	/* ibc_enhancement1 */
	hermon_ci_not_supported,	/* ibc_enhancement2 */
	hermon_ci_not_supported,	/* ibc_enhancement3 */
	hermon_ci_not_supported,	/* ibc_enhancement4 */
};

/*
 * Not yet implemented OPS
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_not_supported()
{
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_query_hca_ports()
 *    Returns HCA port attributes for either one or all of the HCA's ports.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_query_hca_ports(ibc_hca_hdl_t hca, uint8_t query_port,
    ibt_hca_portinfo_t *info_p)
{
	hermon_state_t	*state;
	uint_t		start, end, port;
	int		status, indx;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/*
	 * If the specified port is zero, then we are supposed to query all
	 * ports.  Otherwise, we query only the port number specified.
	 * Setup the start and end port numbers as appropriate for the loop
	 * below.  Note:  The first Hermon port is port number one (1).
	 */
	if (query_port == 0) {
		start = 1;
		end = start + (state->hs_cfg_profile->cp_num_ports - 1);
	} else {
		end = start = query_port;
	}

	/* Query the port(s) */
	for (port = start, indx = 0; port <= end; port++, indx++) {
		status = hermon_port_query(state, port, &info_p[indx]);
		if (status != DDI_SUCCESS) {
			return (status);
		}
	}
	return (IBT_SUCCESS);
}


/*
 * hermon_ci_modify_ports()
 *    Modify HCA port attributes
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_modify_ports(ibc_hca_hdl_t hca, uint8_t port,
    ibt_port_modify_flags_t flags, uint8_t init_type)
{
	hermon_state_t	*state;
	int		status;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Modify the port(s) */
	status = hermon_port_modify(state, port, flags, init_type);
	return (status);
}

/*
 * hermon_ci_modify_system_image()
 *    Modify the System Image GUID
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_system_image(ibc_hca_hdl_t hca, ib_guid_t sys_guid)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support modification of the System
	 * Image GUID.  Hermon is only capable of modifying this parameter
	 * once (during driver initialization).
	 */
	return (IBT_NOT_SUPPORTED);
}

/*
 * hermon_ci_alloc_pd()
 *    Allocate a Protection Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_pd(ibc_hca_hdl_t hca, ibt_pd_flags_t flags, ibc_pd_hdl_t *pd_p)
{
	hermon_state_t	*state;
	hermon_pdhdl_t	pdhdl;
	int		status;

	ASSERT(pd_p != NULL);

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Allocate the PD */
	status = hermon_pd_alloc(state, &pdhdl, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Return the Hermon PD handle */
	*pd_p = (ibc_pd_hdl_t)pdhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_pd()
 *    Free a Protection Domain
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_free_pd(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	int			status;

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Free the PD */
	status = hermon_pd_free(state, &pdhdl);
	return (status);
}


/*
 * hermon_ci_alloc_rdd()
 *    Allocate a Reliable Datagram Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_rdd(ibc_hca_hdl_t hca, ibc_rdd_flags_t flags,
    ibc_rdd_hdl_t *rdd_p)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_free_rdd()
 *    Free a Reliable Datagram Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_rdd(ibc_hca_hdl_t hca, ibc_rdd_hdl_t rdd)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_alloc_ah()
 *    Allocate an Address Handle
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_ah(ibc_hca_hdl_t hca, ibt_ah_flags_t flags, ibc_pd_hdl_t pd,
    ibt_adds_vect_t *attr_p, ibc_ah_hdl_t *ah_p)
{
	hermon_state_t	*state;
	hermon_ahhdl_t	ahhdl;
	hermon_pdhdl_t	pdhdl;
	int		status;

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Allocate the AH */
	status = hermon_ah_alloc(state, pdhdl, attr_p, &ahhdl, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Return the Hermon AH handle */
	*ah_p = (ibc_ah_hdl_t)ahhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_ah()
 *    Free an Address Handle
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah)
{
	hermon_state_t	*state;
	hermon_ahhdl_t	ahhdl;
	int		status;

	/* Grab the Hermon softstate pointer and AH handle */
	state = (hermon_state_t *)hca;
	ahhdl = (hermon_ahhdl_t)ah;

	/* Free the AH */
	status = hermon_ah_free(state, &ahhdl, HERMON_NOSLEEP);

	return (status);
}


/*
 * hermon_ci_query_ah()
 *    Return the Address Vector information for a specified Address Handle
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibc_pd_hdl_t *pd_p,
    ibt_adds_vect_t *attr_p)
{
	hermon_state_t	*state;
	hermon_ahhdl_t	ahhdl;
	hermon_pdhdl_t	pdhdl;
	int		status;

	/* Grab the Hermon softstate pointer and AH handle */
	state = (hermon_state_t *)hca;
	ahhdl = (hermon_ahhdl_t)ah;

	/* Query the AH */
	status = hermon_ah_query(state, ahhdl, &pdhdl, attr_p);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Return the Hermon PD handle */
	*pd_p = (ibc_pd_hdl_t)pdhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_modify_ah()
 *    Modify the Address Vector information of a specified Address Handle
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_modify_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibt_adds_vect_t *attr_p)
{
	hermon_state_t	*state;
	hermon_ahhdl_t	ahhdl;
	int		status;

	/* Grab the Hermon softstate pointer and AH handle */
	state = (hermon_state_t *)hca;
	ahhdl = (hermon_ahhdl_t)ah;

	/* Modify the AH */
	status = hermon_ah_modify(state, ahhdl, attr_p);

	return (status);
}


/*
 * hermon_ci_alloc_qp()
 *    Allocate a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_alloc_qp(ibc_hca_hdl_t hca, ibtl_qp_hdl_t ibt_qphdl,
    ibt_qp_type_t type, ibt_qp_alloc_attr_t *attr_p,
    ibt_chan_sizes_t *queue_sizes_p, ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
	hermon_state_t		*state;
	hermon_qp_info_t	qpinfo;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Allocate the QP */
	qpinfo.qpi_attrp	= attr_p;
	qpinfo.qpi_type		= type;
	qpinfo.qpi_ibt_qphdl	= ibt_qphdl;
	qpinfo.qpi_queueszp	= queue_sizes_p;
	qpinfo.qpi_qpn		= qpn;
	status = hermon_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Return the Hermon QP handle */
	*qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_alloc_special_qp()
 *    Allocate a Special Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_alloc_special_qp(ibc_hca_hdl_t hca, uint8_t port,
    ibtl_qp_hdl_t ibt_qphdl, ibt_sqp_type_t type,
    ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
    ibc_qp_hdl_t *qp_p)
{
	hermon_state_t		*state;
	hermon_qp_info_t	qpinfo;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Allocate the Special QP */
	qpinfo.qpi_attrp	= attr_p;
	qpinfo.qpi_type		= type;
	qpinfo.qpi_port		= port;
	qpinfo.qpi_ibt_qphdl	= ibt_qphdl;
	qpinfo.qpi_queueszp	= queue_sizes_p;
	status = hermon_special_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	/* Return the Hermon QP handle */
	*qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_qp_range()
 *    Free a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_qp_range(ibc_hca_hdl_t hca, uint_t log2,
    ibtl_qp_hdl_t *ibtl_qp, ibt_qp_type_t type,
    ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
    ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
    ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
	hermon_state_t		*state;
	hermon_qp_info_t	qpinfo;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Allocate the QP */
	qpinfo.qpi_attrp	= attr_p;
	qpinfo.qpi_type		= type;
	qpinfo.qpi_queueszp	= queue_sizes_p;
	qpinfo.qpi_qpn		= qpn;
	status = hermon_qp_alloc_range(state, log2, &qpinfo, ibtl_qp,
	    send_cq, recv_cq, (hermon_qphdl_t *)qp_p, HERMON_NOSLEEP);
	return (status);
}

/*
 * hermon_ci_free_qp()
 *    Free a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh_p)
{
	hermon_state_t	*state;
	hermon_qphdl_t	qphdl;
	int		status;

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Free the QP */
	status = hermon_qp_free(state, &qphdl, free_qp_flags, qpnh_p,
	    HERMON_NOSLEEP);

	return (status);
}


/*
 * hermon_ci_release_qpn()
 *    Release a Queue Pair Number (QPN)
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_release_qpn(ibc_hca_hdl_t hca, ibc_qpn_hdl_t qpnh)
{
	hermon_state_t		*state;
	hermon_qpn_entry_t	*entry;

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	entry = (hermon_qpn_entry_t *)qpnh;

	/* Release the QP number */
	hermon_qp_release_qpn(state, entry, HERMON_QPN_RELEASE);

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_query_qp()
 *    Query a Queue Pair
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibt_qp_query_attr_t *attr_p)
{
	hermon_state_t	*state;
	hermon_qphdl_t	qphdl;
	int		status;

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Query the QP */
	status = hermon_qp_query(state, qphdl, attr_p);
	return (status);
}


/*
 * hermon_ci_modify_qp()
 *    Modify a Queue Pair
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_modify_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
    ibt_queue_sizes_t *actual_sz)
{
	hermon_state_t	*state;
	hermon_qphdl_t	qphdl;
	int		status;

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Modify the QP */
	status = hermon_qp_modify(state, qphdl, flags, info_p, actual_sz);
	return (status);
}


/*
 * hermon_ci_alloc_cq()
 *    Allocate a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq(ibc_hca_hdl_t hca, ibt_cq_hdl_t ibt_cqhdl,
    ibt_cq_attr_t *attr_p, ibc_cq_hdl_t *cq_p, uint_t *actual_size)
{
	hermon_state_t	*state;
	hermon_cqhdl_t	cqhdl;
	int		status;

	state = (hermon_state_t *)hca;

	/* Allocate the CQ */
	status = hermon_cq_alloc(state, ibt_cqhdl, attr_p, actual_size,
	    &cqhdl, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Return the Hermon CQ handle */
	*cq_p = (ibc_cq_hdl_t)cqhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_cq()
 *    Free a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq)
{
	hermon_state_t	*state;
	hermon_cqhdl_t	cqhdl;
	int		status;

	/* Grab the Hermon softstate pointer and CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq;


	/* Free the CQ */
	status = hermon_cq_free(state, &cqhdl, HERMON_NOSLEEP);
	return (status);
}


/*
 * hermon_ci_query_cq()
 *    Return the size of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_query_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t *entries_p,
    uint_t *count_p, uint_t *usec_p, ibt_cq_handler_id_t *hid_p)
{
	hermon_state_t	*state;
	hermon_cqhdl_t	cqhdl;

	/* Grab the CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq;

	/* Query the current CQ size */
	*entries_p = cqhdl->cq_bufsz;
	*count_p = cqhdl->cq_intmod_count;
	*usec_p = cqhdl->cq_intmod_usec;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cqhdl))
	*hid_p = HERMON_EQNUM_TO_HID(state, cqhdl->cq_eqnum);

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_resize_cq()
 *    Change the size of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_resize_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t size,
    uint_t *actual_size)
{
	hermon_state_t		*state;
	hermon_cqhdl_t		cqhdl;
	int			status;

	/* Grab the Hermon softstate pointer and CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq;

	/* Resize the CQ */
	status = hermon_cq_resize(state, cqhdl, size, actual_size,
	    HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	return (IBT_SUCCESS);
}

/*
 * hermon_ci_modify_cq()
 *    Change the interrupt moderation values of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_modify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t count,
    uint_t usec, ibt_cq_handler_id_t hid)
{
	hermon_state_t		*state;
	hermon_cqhdl_t		cqhdl;
	int			status;

	/* Grab the Hermon softstate pointer and CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq;

	/* Resize the CQ */
	status = hermon_cq_modify(state, cqhdl, count, usec, hid,
	    HERMON_NOSLEEP);
	return (status);
}


/*
 * hermon_ci_alloc_cq_sched()
 *    Reserve a CQ scheduling class resource
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq_sched(ibc_hca_hdl_t hca, ibt_cq_sched_attr_t *attr,
    ibc_sched_hdl_t *sched_hdl_p)
{
	int	status;

	status = hermon_cq_sched_alloc((hermon_state_t *)hca, attr,
	    (hermon_cq_sched_t **)sched_hdl_p);
	return (status);
}


/*
 * hermon_ci_free_cq_sched()
 *    Free a CQ scheduling class resource
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_cq_sched(ibc_hca_hdl_t hca, ibc_sched_hdl_t sched_hdl)
{
	int	status;

	status = hermon_cq_sched_free((hermon_state_t *)hca,
	    (hermon_cq_sched_t *)sched_hdl);
	return (status);
}

static ibt_status_t
hermon_ci_query_cq_handler_id(ibc_hca_hdl_t hca,
    ibt_cq_handler_id_t hid, ibt_cq_handler_attr_t *attrs)
{
	hermon_state_t		*state;

	state = (hermon_state_t *)hca;
	if (!HERMON_HID_VALID(state, hid))
		return (IBT_CQ_HID_INVALID);
	if (attrs == NULL)
		return (IBT_INVALID_PARAM);
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attrs))
	attrs->cha_ih = state->hs_intrmsi_hdl[hid - 1];
	attrs->cha_dip = state->hs_dip;
	return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_eec()
 *    Allocate an End-to-End context
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_eec(ibc_hca_hdl_t hca, ibc_eec_flags_t flags,
    ibt_eec_hdl_t ibt_eec, ibc_rdd_hdl_t rdd, ibc_eec_hdl_t *eec_p)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_free_eec()
 *    Free an End-to-End context
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_query_eec()
 *    Query an End-to-End context
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
    ibt_eec_query_attr_t *attr_p)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_modify_eec()
 *    Modify an End-to-End context
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
    ibt_cep_modify_flags_t flags, ibt_eec_info_t *info_p)
{
	/*
	 * This is an unsupported interface for the Hermon driver.  This
	 * interface is necessary to support Reliable Datagram (RD)
	 * operations.  Hermon does not support RD.
	 */
	return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_register_mr()
 *    Prepare a virtually addressed Memory Region for use by an HCA
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_mr_desc_t *mr_desc)
{
	hermon_mr_options_t	op;
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_attr != NULL);
	ASSERT(mr_p != NULL);
	ASSERT(mr_desc != NULL);

	/*
	 * Validate the access flags.  Both Remote Write and Remote Atomic
	 * require the Local Write flag to be set
	 */
	if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
	    !(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
		return (IBT_MR_ACCESS_REQ_INVALID);
	}

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Register the memory region */
	op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
	op.mro_bind_dmahdl = NULL;
	op.mro_bind_override_addr = 0;
	status = hermon_mr_register(state, pdhdl, mr_attr, &mrhdl,
	    &op, HERMON_MPT_DMPT);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
	mr_desc->md_lkey  = mrhdl->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl->mr_rkey;
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_register_buf()
 *    Prepare a Memory Region specified by buf structure for use by an HCA
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_buf(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
    ibt_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
	hermon_mr_options_t	op;
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl;
	int			status;
	ibt_mr_flags_t		flags = attrp->mr_flags;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_p != NULL);
	ASSERT(mr_desc != NULL);

	/*
	 * Validate the access flags.  Both Remote Write and Remote Atomic
	 * require the Local Write flag to be set
	 */
	if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
	    !(flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
		return (IBT_MR_ACCESS_REQ_INVALID);
	}

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Register the memory region */
	op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
	op.mro_bind_dmahdl = NULL;
	op.mro_bind_override_addr = 0;
	status = hermon_mr_register_buf(state, pdhdl, attrp, buf,
	    &mrhdl, &op, HERMON_MPT_DMPT);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
	mr_desc->md_lkey  = mrhdl->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl->mr_rkey;
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_deregister_mr()
 *    Deregister a Memory Region from an HCA translation table
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_deregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
	hermon_state_t		*state;
	hermon_mrhdl_t		mrhdl;
	int			status;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;
	mrhdl = (hermon_mrhdl_t)mr;

	/*
	 * Deregister the memory region.
	 */
	status = hermon_mr_deregister(state, &mrhdl, HERMON_MR_DEREG_ALL,
	    HERMON_NOSLEEP);
	return (status);
}


/*
 * hermon_ci_query_mr()
 *    Retrieve information about a specified Memory Region
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibt_mr_query_attr_t *mr_attr)
{
	hermon_state_t		*state;
	hermon_mrhdl_t		mrhdl;
	int			status;

	ASSERT(mr_attr != NULL);

	/* Grab the Hermon softstate pointer and MR handle */
	state = (hermon_state_t *)hca;
	mrhdl = (hermon_mrhdl_t)mr;

	/* Query the memory region */
	status = hermon_mr_query(state, mrhdl, mr_attr);
	return (status);
}


/*
 * hermon_ci_register_shared_mr()
 *    Create a shared memory region matching an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_shared_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibc_pd_hdl_t pd, ibt_smr_attr_t *mr_attr, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl, mrhdl_new;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_attr != NULL);
	ASSERT(mr_p != NULL);
	ASSERT(mr_desc != NULL);

	/*
	 * Validate the access flags.  Both Remote Write and Remote Atomic
	 * require the Local Write flag to be set
	 */
	if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
	    !(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
		return (IBT_MR_ACCESS_REQ_INVALID);
	}

	/* Grab the Hermon softstate pointer and handles */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;
	mrhdl = (hermon_mrhdl_t)mr;

	/* Register the shared memory region */
	status = hermon_mr_register_shared(state, mrhdl, pdhdl, mr_attr,
	    &mrhdl_new);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
	mr_desc->md_lkey  = mrhdl_new->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl_new->mr_rkey;
	}

	/*
	 * If shared region is mapped for streaming (i.e. noncoherent), then
	 * set sync is required
	 */
	mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl_new;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_reregister_mr()
 *    Modify the attributes of an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
    ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_new,
    ibt_mr_desc_t *mr_desc)
{
	hermon_mr_options_t	op;
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl, mrhdl_new;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_attr != NULL);
	ASSERT(mr_new != NULL);
	ASSERT(mr_desc != NULL);

	/* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
	state = (hermon_state_t *)hca;
	mrhdl = (hermon_mrhdl_t)mr;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Reregister the memory region */
	op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
	status = hermon_mr_reregister(state, mrhdl, pdhdl, mr_attr,
	    &mrhdl_new, &op);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
	mr_desc->md_lkey  = mrhdl_new->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl_new->mr_rkey;
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

	/* Return the Hermon MR handle */
	*mr_new = (ibc_mr_hdl_t)mrhdl_new;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_reregister_buf()
 *    Modify the attributes of an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_buf(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
    ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_new, ibt_mr_desc_t *mr_desc)
{
	hermon_mr_options_t	op;
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl, mrhdl_new;
	int			status;
	ibt_mr_flags_t		flags = attrp->mr_flags;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_new != NULL);
	ASSERT(mr_desc != NULL);

	/* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
	state = (hermon_state_t *)hca;
	mrhdl = (hermon_mrhdl_t)mr;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Reregister the memory region */
	op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
	status = hermon_mr_reregister_buf(state, mrhdl, pdhdl, attrp, buf,
	    &mrhdl_new, &op);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
	mr_desc->md_lkey  = mrhdl_new->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl_new->mr_rkey;
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

	/* Return the Hermon MR handle */
	*mr_new = (ibc_mr_hdl_t)mrhdl_new;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_sync_mr()
 *    Synchronize access to a Memory Region
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_sync_mr(ibc_hca_hdl_t hca, ibt_mr_sync_t *mr_segs, size_t num_segs)
{
	hermon_state_t		*state;
	int			status;

	ASSERT(mr_segs != NULL);

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Sync the memory region */
	status = hermon_mr_sync(state, mr_segs, num_segs);
	return (status);
}


/*
 * hermon_ci_alloc_mw()
 *    Allocate a Memory Window
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_alloc_mw(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd, ibt_mw_flags_t flags,
    ibc_mw_hdl_t *mw_p, ibt_rkey_t *rkey_p)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mwhdl_t		mwhdl;
	int			status;

	ASSERT(mw_p != NULL);
	ASSERT(rkey_p != NULL);

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	/* Allocate the memory window */
	status = hermon_mw_alloc(state, pdhdl, flags, &mwhdl);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mwhdl))

	/* Return the MW handle and RKey */
	*mw_p = (ibc_mw_hdl_t)mwhdl;
	*rkey_p = mwhdl->mr_rkey;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_mw()
 *    Free a Memory Window
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_free_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw)
{
	hermon_state_t		*state;
	hermon_mwhdl_t		mwhdl;
	int			status;

	/* Grab the Hermon softstate pointer and MW handle */
	state = (hermon_state_t *)hca;
	mwhdl = (hermon_mwhdl_t)mw;

	/* Free the memory window */
	status = hermon_mw_free(state, &mwhdl, HERMON_NOSLEEP);
	return (status);
}


/*
 * hermon_ci_query_mw()
 *    Return the attributes of the specified Memory Window
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw,
    ibt_mw_query_attr_t *mw_attr_p)
{
	hermon_mwhdl_t		mwhdl;

	ASSERT(mw_attr_p != NULL);

	/* Query the memory window pointer and fill in the return values */
	mwhdl = (hermon_mwhdl_t)mw;
	mutex_enter(&mwhdl->mr_lock);
	mw_attr_p->mw_pd   = (ibc_pd_hdl_t)mwhdl->mr_pdhdl;
	mw_attr_p->mw_rkey = mwhdl->mr_rkey;
	mutex_exit(&mwhdl->mr_lock);

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_register_dma_mr()
 *    Allocate a memory region that maps physical addresses.
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_dma_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_dmr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_mr_desc_t *mr_desc)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

	ASSERT(mr_attr != NULL);
	ASSERT(mr_p != NULL);
	ASSERT(mr_desc != NULL);

	/*
	 * Validate the access flags.  Both Remote Write and Remote Atomic
	 * require the Local Write flag to be set
	 */
	if (((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
	    !(mr_attr->dmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
		return (IBT_MR_ACCESS_REQ_INVALID);
	}

	/* Grab the Hermon softstate pointer and PD handle */
	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	status = hermon_dma_mr_register(state, pdhdl, mr_attr, &mrhdl);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

	/* Fill in the mr_desc structure */
	mr_desc->md_vaddr = mr_attr->dmr_paddr;
	mr_desc->md_lkey  = mrhdl->mr_lkey;
	/* Only set RKey if remote access was requested */
	if ((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
	    (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
		mr_desc->md_rkey = mrhdl->mr_rkey;
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	mr_desc->md_sync_required = B_FALSE;

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl;

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_attach_mcg()
 *    Attach a Queue Pair to a Multicast Group
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_attach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
    ib_lid_t lid)
{
	hermon_state_t		*state;
	hermon_qphdl_t		qphdl;
	int			status;

	/* Grab the Hermon softstate pointer and QP handles */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Attach the QP to the multicast group */
	status = hermon_mcg_attach(state, qphdl, gid, lid);
	return (status);
}


/*
 * hermon_ci_detach_mcg()
 *    Detach a Queue Pair to a Multicast Group
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_detach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
    ib_lid_t lid)
{
	hermon_state_t		*state;
	hermon_qphdl_t		qphdl;
	int			status;

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Detach the QP from the multicast group */
	status = hermon_mcg_detach(state, qphdl, gid, lid);
	return (status);
}


/*
 * hermon_ci_post_send()
 *    Post send work requests to the send queue on the specified QP
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_send(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_send_wr_t *wr_p,
    uint_t num_wr, uint_t *num_posted_p)
{
	hermon_state_t		*state;
	hermon_qphdl_t		qphdl;
	int			status;

	ASSERT(wr_p != NULL);
	ASSERT(num_wr != 0);

	/* Grab the Hermon softstate pointer and QP handle */
	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Post the send WQEs */
	status = hermon_post_send(state, qphdl, wr_p, num_wr, num_posted_p);
	return (status);
}


/*
 * hermon_ci_post_recv()
 *    Post receive work requests to the receive queue on the specified QP
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_recv(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_recv_wr_t *wr_p,
    uint_t num_wr, uint_t *num_posted_p)
{
	hermon_state_t		*state;
	hermon_qphdl_t		qphdl;
	int			status;

	ASSERT(wr_p != NULL);
	ASSERT(num_wr != 0);

	state = (hermon_state_t *)hca;
	qphdl = (hermon_qphdl_t)qp;

	/* Post the receive WQEs */
	status = hermon_post_recv(state, qphdl, wr_p, num_wr, num_posted_p);
	return (status);
}


/*
 * hermon_ci_poll_cq()
 *    Poll for a work request completion
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_poll_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, ibt_wc_t *wc_p,
    uint_t num_wc, uint_t *num_polled)
{
	hermon_state_t		*state;
	hermon_cqhdl_t		cqhdl;
	int			status;

	ASSERT(wc_p != NULL);

	/* Check for valid num_wc field */
	if (num_wc == 0) {
		return (IBT_INVALID_PARAM);
	}

	/* Grab the Hermon softstate pointer and CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq;

	/* Poll for work request completions */
	status = hermon_cq_poll(state, cqhdl, wc_p, num_wc, num_polled);
	return (status);
}


/*
 * hermon_ci_notify_cq()
 *    Enable notification events on the specified CQ
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_notify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq_hdl,
    ibt_cq_notify_flags_t flags)
{
	hermon_state_t		*state;
	hermon_cqhdl_t		cqhdl;
	int			status;

	/* Grab the Hermon softstate pointer and CQ handle */
	state = (hermon_state_t *)hca;
	cqhdl = (hermon_cqhdl_t)cq_hdl;

	/* Enable the CQ notification */
	status = hermon_cq_notify(state, cqhdl, flags);
	return (status);
}

/*
 * hermon_ci_ci_data_in()
 *    Exchange CI-specific data.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_ci_data_in(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
    ibt_object_type_t object, void *ibc_object_handle, void *data_p,
    size_t data_sz)
{
	hermon_state_t		*state;
	int			status;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Get the Hermon userland mapping information */
	status = hermon_umap_ci_data_in(state, flags, object,
	    ibc_object_handle, data_p, data_sz);
	return (status);
}

/*
 * hermon_ci_ci_data_out()
 *    Exchange CI-specific data.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_ci_data_out(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
    ibt_object_type_t object, void *ibc_object_handle, void *data_p,
    size_t data_sz)
{
	hermon_state_t		*state;
	int			status;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	/* Get the Hermon userland mapping information */
	status = hermon_umap_ci_data_out(state, flags, object,
	    ibc_object_handle, data_p, data_sz);
	return (status);
}


/*
 * hermon_ci_alloc_srq()
 *    Allocate a Shared Receive Queue (SRQ)
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_alloc_srq(ibc_hca_hdl_t hca, ibt_srq_flags_t flags,
    ibt_srq_hdl_t ibt_srq, ibc_pd_hdl_t pd, ibt_srq_sizes_t *sizes,
    ibc_srq_hdl_t *ibc_srq_p, ibt_srq_sizes_t *ret_sizes_p)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_srqhdl_t		srqhdl;
	hermon_srq_info_t	srqinfo;
	int			status;

	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	srqinfo.srqi_ibt_srqhdl = ibt_srq;
	srqinfo.srqi_pd		= pdhdl;
	srqinfo.srqi_sizes	= sizes;
	srqinfo.srqi_real_sizes	= ret_sizes_p;
	srqinfo.srqi_srqhdl	= &srqhdl;
	srqinfo.srqi_flags	= flags;

	status = hermon_srq_alloc(state, &srqinfo, HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	*ibc_srq_p = (ibc_srq_hdl_t)srqhdl;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_free_srq()
 *    Free a Shared Receive Queue (SRQ)
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_free_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq)
{
	hermon_state_t	*state;
	hermon_srqhdl_t	srqhdl;
	int		status;

	state = (hermon_state_t *)hca;

	/* Check for valid SRQ handle pointer */
	if (srq == NULL) {
		return (IBT_SRQ_HDL_INVALID);
	}

	srqhdl = (hermon_srqhdl_t)srq;

	/* Free the SRQ */
	status = hermon_srq_free(state, &srqhdl, HERMON_NOSLEEP);
	return (status);
}

/*
 * hermon_ci_query_srq()
 *    Query properties of a Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq, ibc_pd_hdl_t *pd_p,
    ibt_srq_sizes_t *sizes_p, uint_t *limit_p)
{
	hermon_srqhdl_t	srqhdl;

	srqhdl = (hermon_srqhdl_t)srq;

	mutex_enter(&srqhdl->srq_lock);
	if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
		mutex_exit(&srqhdl->srq_lock);
		return (IBT_SRQ_ERROR_STATE);
	}

	*pd_p   = (ibc_pd_hdl_t)srqhdl->srq_pdhdl;
	sizes_p->srq_wr_sz = srqhdl->srq_real_sizes.srq_wr_sz - 1;
	sizes_p->srq_sgl_sz = srqhdl->srq_real_sizes.srq_sgl_sz;
	mutex_exit(&srqhdl->srq_lock);
	*limit_p  = 0;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_modify_srq()
 *    Modify properties of a Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
    ibt_srq_modify_flags_t flags, uint_t size, uint_t limit, uint_t *ret_size_p)
{
	hermon_state_t	*state;
	hermon_srqhdl_t	srqhdl;
	uint_t		resize_supported, cur_srq_size;
	int		status;

	state = (hermon_state_t *)hca;
	srqhdl = (hermon_srqhdl_t)srq;

	/*
	 * Check Error State of SRQ.
	 * Also, while we are holding the lock we save away the current SRQ
	 * size for later use.
	 */
	mutex_enter(&srqhdl->srq_lock);
	cur_srq_size = srqhdl->srq_wq_bufsz;
	if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
		mutex_exit(&srqhdl->srq_lock);
		return (IBT_SRQ_ERROR_STATE);
	}
	mutex_exit(&srqhdl->srq_lock);

	/*
	 * Setting the limit watermark is not currently supported.  This is a
	 * hermon hardware (firmware) limitation.  We return NOT_SUPPORTED here,
	 * and have the limit code commented out for now.
	 *
	 * XXX If we enable the limit watermark support, we need to do checks
	 * and set the 'srq->srq_wr_limit' here, instead of returning not
	 * supported.  The 'hermon_srq_modify' operation below is for resizing
	 * the SRQ only, the limit work should be done here.  If this is
	 * changed to use the 'limit' field, the 'ARGSUSED' comment for this
	 * function should also be removed at that time.
	 */
	if (flags & IBT_SRQ_SET_LIMIT) {
		return (IBT_NOT_SUPPORTED);
	}

	/*
	 * Check the SET_SIZE flag.  If not set, we simply return success here.
	 * However if it is set, we check if resize is supported and only then
	 * do we continue on with our resize processing.
	 */
	if (!(flags & IBT_SRQ_SET_SIZE)) {
		return (IBT_SUCCESS);
	}

	resize_supported = state->hs_ibtfinfo.hca_attr->hca_flags &
	    IBT_HCA_RESIZE_SRQ;

	if ((flags & IBT_SRQ_SET_SIZE) && !resize_supported) {
		return (IBT_NOT_SUPPORTED);
	}

	/*
	 * We do not support resizing an SRQ to be smaller than it's current
	 * size.  If a smaller (or equal) size is requested, then we simply
	 * return success, and do nothing.
	 */
	if (size <= cur_srq_size) {
		*ret_size_p = cur_srq_size;
		return (IBT_SUCCESS);
	}

	status = hermon_srq_modify(state, srqhdl, size, ret_size_p,
	    HERMON_NOSLEEP);
	if (status != DDI_SUCCESS) {
		/* Set return value to current SRQ size */
		*ret_size_p = cur_srq_size;
		return (status);
	}

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_post_srq()
 *    Post a Work Request to the specified Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
    ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted_p)
{
	hermon_state_t	*state;
	hermon_srqhdl_t	srqhdl;
	int		status;

	state = (hermon_state_t *)hca;
	srqhdl = (hermon_srqhdl_t)srq;

	status = hermon_post_srq(state, srqhdl, wr, num_wr, num_posted_p);
	return (status);
}

/* Address translation */

struct ibc_ma_s {
	int			h_ma_addr_list_len;
	void			*h_ma_addr_list;
	ddi_dma_handle_t	h_ma_dmahdl;
	ddi_dma_handle_t	h_ma_list_hdl;
	ddi_acc_handle_t	h_ma_list_acc_hdl;
	size_t			h_ma_real_len;
	caddr_t			h_ma_kaddr;
	ibt_phys_addr_t		h_ma_list_cookie;
};

static ibt_status_t
hermon_map_mem_area_fmr(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
    uint_t list_len, ibt_pmr_attr_t *pmr, ibc_ma_hdl_t *ma_hdl_p)
{
	int			status;
	ibt_status_t		ibt_status;
	ibc_ma_hdl_t		ma_hdl;
	ib_memlen_t		len;
	ddi_dma_attr_t		dma_attr;
	uint_t			cookie_cnt;
	ddi_dma_cookie_t	dmacookie;
	hermon_state_t		*state;
	uint64_t		*kaddr;
	uint64_t		addr, endaddr, pagesize;
	int			i, kmflag;
	int			(*callback)(caddr_t);

	if ((va_attrs->va_flags & IBT_VA_BUF) == 0) {
		return (IBT_NOT_SUPPORTED);	/* XXX - not yet implemented */
	}

	state = (hermon_state_t *)hca;
	hermon_dma_attr_init(state, &dma_attr);
	if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
		kmflag = KM_NOSLEEP;
		callback = DDI_DMA_DONTWAIT;
	} else {
		kmflag = KM_SLEEP;
		callback = DDI_DMA_SLEEP;
	}

	ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
	if (ma_hdl == NULL) {
		return (IBT_INSUFF_RESOURCE);
	}
#ifdef	__sparc
	if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
		dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

	if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
		dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))
	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
	    callback, NULL, &ma_hdl->h_ma_dmahdl);
	if (status != DDI_SUCCESS) {
		kmem_free(ma_hdl, sizeof (*ma_hdl));
		return (IBT_INSUFF_RESOURCE);
	}
	status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
	    va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    callback, NULL, &dmacookie, &cookie_cnt);
	if (status != DDI_DMA_MAPPED) {
		status = ibc_get_ci_failure(0);
		goto marea_fail3;
	}

	ma_hdl->h_ma_real_len = list_len * sizeof (ibt_phys_addr_t);
	ma_hdl->h_ma_kaddr = kmem_zalloc(ma_hdl->h_ma_real_len, kmflag);
	if (ma_hdl->h_ma_kaddr == NULL) {
		ibt_status = IBT_INSUFF_RESOURCE;
		goto marea_fail4;
	}

	i = 0;
	len = 0;
	pagesize = PAGESIZE;
	kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
	while (cookie_cnt-- > 0) {
		addr	= dmacookie.dmac_laddress;
		len	+= dmacookie.dmac_size;
		endaddr	= addr + (dmacookie.dmac_size - 1);
		addr	= addr & ~(pagesize - 1);
		while (addr <= endaddr) {
			if (i >= list_len) {
				status = IBT_PBL_TOO_SMALL;
				goto marea_fail5;
			}
			kaddr[i] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
			i++;
			addr += pagesize;
			if (addr == 0) {
				static int do_once = 1;
				_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
				    do_once))
				if (do_once) {
					do_once = 0;
					cmn_err(CE_NOTE, "probable error in "
					    "dma_cookie address: map_mem_area");
				}
				break;
			}
		}
		if (cookie_cnt != 0)
			ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
	pmr->pmr_addr_list = (ibt_phys_addr_t *)(void *)ma_hdl->h_ma_kaddr;
	pmr->pmr_iova = va_attrs->va_vaddr;
	pmr->pmr_len = len;
	pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
	pmr->pmr_buf_sz = PAGESHIFT;	/* PRM says "Page Sice", but... */
	pmr->pmr_num_buf = i;
	pmr->pmr_ma = ma_hdl;

	*ma_hdl_p = ma_hdl;
	return (IBT_SUCCESS);

marea_fail5:
	kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
marea_fail4:
	status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
marea_fail3:
	ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
	kmem_free(ma_hdl, sizeof (*ma_hdl));
	*ma_hdl_p = NULL;
	return (ibt_status);
}

/*
 * hermon_ci_map_mem_area()
 *    Context: Can be called from user or base context.
 *
 *	Creates the memory mapping suitable for a subsequent posting of an
 *	FRWR work request.  All the info about the memory area for the
 *	FRWR work request (wr member of "union ibt_reg_req_u") is filled
 *	such that the client only needs to point wr.rc.rcwr.reg_pmr to it,
 *	and then fill in the additional information only it knows.
 *
 *	Alternatively, creates the memory mapping for FMR.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_area(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
    void *ibtl_reserved, uint_t list_len, ibt_reg_req_t *reg_req,
    ibc_ma_hdl_t *ma_hdl_p)
{
	ibt_status_t		ibt_status;
	int			status;
	ibc_ma_hdl_t		ma_hdl;
	ibt_wr_reg_pmr_t	*pmr;
	ib_memlen_t		len;
	ddi_dma_attr_t		dma_attr;
	ddi_dma_handle_t	khdl;
	uint_t			cookie_cnt;
	ddi_dma_cookie_t	dmacookie, kcookie;
	hermon_state_t		*state;
	uint64_t		*kaddr;
	uint64_t		addr, endaddr, pagesize, kcookie_paddr;
	int			i, j, kmflag;
	int			(*callback)(caddr_t);

	if (va_attrs->va_flags & (IBT_VA_FMR | IBT_VA_REG_FN)) {
		/* delegate FMR and Physical Register to other function */
		return (hermon_map_mem_area_fmr(hca, va_attrs, list_len,
		    &reg_req->fn_arg, ma_hdl_p));
	}

	/* FRWR */

	state = (hermon_state_t *)hca;
	if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
		return (IBT_NOT_SUPPORTED);
	hermon_dma_attr_init(state, &dma_attr);
#ifdef	__sparc
	if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
		dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

	if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
		dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif
	if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
		kmflag = KM_NOSLEEP;
		callback = DDI_DMA_DONTWAIT;
	} else {
		kmflag = KM_SLEEP;
		callback = DDI_DMA_SLEEP;
	}

	ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
	if (ma_hdl == NULL) {
		return (IBT_INSUFF_RESOURCE);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))

	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
	    callback, NULL, &ma_hdl->h_ma_dmahdl);
	if (status != DDI_SUCCESS) {
		ibt_status = IBT_INSUFF_RESOURCE;
		goto marea_fail0;
	}
	dma_attr.dma_attr_align = 64;	/* as per PRM */
	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
	    callback, NULL, &ma_hdl->h_ma_list_hdl);
	if (status != DDI_SUCCESS) {
		ibt_status = IBT_INSUFF_RESOURCE;
		goto marea_fail1;
	}
	/*
	 * Entries in the list in the last slot on each page cannot be used,
	 * so 1 extra ibt_phys_addr_t is allocated per page.  We add 1 more
	 * to deal with the possibility of a less than 1 page allocation
	 * across a page boundary.
	 */
	status = ddi_dma_mem_alloc(ma_hdl->h_ma_list_hdl, (list_len + 1 +
	    list_len / (HERMON_PAGESIZE / sizeof (ibt_phys_addr_t))) *
	    sizeof (ibt_phys_addr_t),
	    &state->hs_reg_accattr, DDI_DMA_CONSISTENT, callback, NULL,
	    &ma_hdl->h_ma_kaddr, &ma_hdl->h_ma_real_len,
	    &ma_hdl->h_ma_list_acc_hdl);
	if (status != DDI_SUCCESS) {
		ibt_status = IBT_INSUFF_RESOURCE;
		goto marea_fail2;
	}
	status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_list_hdl, NULL,
	    ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len, DDI_DMA_RDWR |
	    DDI_DMA_CONSISTENT, callback, NULL,
	    &kcookie, &cookie_cnt);
	if (status != DDI_SUCCESS) {
		ibt_status = IBT_INSUFF_RESOURCE;
		goto marea_fail3;
	}
	if ((kcookie.dmac_laddress & 0x3f) != 0) {
		cmn_err(CE_NOTE, "64-byte alignment assumption wrong");
		ibt_status = ibc_get_ci_failure(0);
		goto marea_fail4;
	}
	ma_hdl->h_ma_list_cookie.p_laddr = kcookie.dmac_laddress;

	if (va_attrs->va_flags & IBT_VA_BUF) {
		status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
		    va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
		    callback, NULL, &dmacookie, &cookie_cnt);
	} else {
		status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_dmahdl,
		    va_attrs->va_as, (caddr_t)(uintptr_t)va_attrs->va_vaddr,
		    va_attrs->va_len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
		    callback, NULL, &dmacookie, &cookie_cnt);
	}
	if (status != DDI_DMA_MAPPED) {
		ibt_status = ibc_get_ci_failure(0);
		goto marea_fail4;
	}
	i = 0;	/* count the number of pbl entries */
	j = 0;	/* count the number of links to next HERMON_PAGE */
	len = 0;
	pagesize = PAGESIZE;
	kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
	kcookie.dmac_size += kcookie.dmac_laddress & HERMON_PAGEOFFSET;
	kcookie_paddr = kcookie.dmac_laddress & HERMON_PAGEMASK;
	khdl = ma_hdl->h_ma_list_hdl;
	while (cookie_cnt-- > 0) {
		addr	= dmacookie.dmac_laddress;
		len	+= dmacookie.dmac_size;
		endaddr	= addr + (dmacookie.dmac_size - 1);
		addr	= addr & ~(pagesize - 1);
		while (addr <= endaddr) {
			if (i >= list_len) {
				ibt_status = IBT_PBL_TOO_SMALL;
				goto marea_fail5;
			}
			/* Deal with last entry on page. */
			if (!((uintptr_t)&kaddr[i+j+1] & HERMON_PAGEOFFSET)) {
				if (kcookie.dmac_size > HERMON_PAGESIZE) {
					kcookie_paddr += HERMON_PAGESIZE;
					kcookie.dmac_size -= HERMON_PAGESIZE;
				} else {
					ddi_dma_nextcookie(khdl, &kcookie);
					kcookie_paddr = kcookie.dmac_laddress;
				}
				kaddr[i+j] = htonll(kcookie_paddr);
				j++;
			}
			kaddr[i+j] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
			i++;
			addr += pagesize;
			if (addr == 0) {
				static int do_once = 1;
				_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
				    do_once))
				if (do_once) {
					do_once = 0;
					cmn_err(CE_NOTE, "probable error in "
					    "dma_cookie address: map_mem_area");
				}
				break;
			}
		}
		if (cookie_cnt != 0)
			ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
	}

	pmr = &reg_req->wr;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
	pmr->pmr_len = len;
	pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
	pmr->pmr_buf_sz = PAGESHIFT;	/* PRM says "Page Size", but... */
	pmr->pmr_num_buf = i;
	pmr->pmr_addr_list = &ma_hdl->h_ma_list_cookie;

	*ma_hdl_p = ma_hdl;
	return (IBT_SUCCESS);

marea_fail5:
	status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
	if (status != DDI_SUCCESS)
		HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail4:
	status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
	if (status != DDI_SUCCESS)
		HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail3:
	ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
marea_fail2:
	ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
marea_fail1:
	ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
marea_fail0:
	kmem_free(ma_hdl, sizeof (*ma_hdl));
	*ma_hdl_p = NULL;
	return (ibt_status);
}

/*
 * hermon_ci_unmap_mem_area()
 * Unmap the memory area
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_unmap_mem_area(ibc_hca_hdl_t hca, ibc_ma_hdl_t ma_hdl)
{
	int			status;
	hermon_state_t		*state;

	if (ma_hdl == NULL) {
		return (IBT_MA_HDL_INVALID);
	}
	state = (hermon_state_t *)hca;
	if (ma_hdl->h_ma_list_hdl != NULL) {
		status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
		if (status != DDI_SUCCESS)
			HERMON_WARNING(state, "failed to unbind DMA mapping");
		ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
		ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
	} else {
		kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
	}
	status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
	if (status != DDI_SUCCESS)
		HERMON_WARNING(state, "failed to unbind DMA mapping");
	ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
	kmem_free(ma_hdl, sizeof (*ma_hdl));
	return (IBT_SUCCESS);
}

struct ibc_mi_s {
	int			imh_len;
	ddi_dma_handle_t	imh_dmahandle[1];
};
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
    ibc_mi_s::imh_len
    ibc_mi_s::imh_dmahandle))


/*
 * hermon_ci_map_mem_iov()
 * Map the memory
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_iov(ibc_hca_hdl_t hca, ibt_iov_attr_t *iov_attr,
    ibt_all_wr_t *wr, ibc_mi_hdl_t *mi_hdl_p)
{
	int			status;
	int			i, j, nds, max_nds;
	uint_t			len;
	ibt_status_t		ibt_status;
	ddi_dma_handle_t	dmahdl;
	ddi_dma_cookie_t	dmacookie;
	ddi_dma_attr_t		dma_attr;
	uint_t			cookie_cnt;
	ibc_mi_hdl_t		mi_hdl;
	ibt_lkey_t		rsvd_lkey;
	ibt_wr_ds_t		*sgl;
	hermon_state_t		*state;
	int			kmflag;
	int			(*callback)(caddr_t);

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wr))

	state = (hermon_state_t *)hca;
	hermon_dma_attr_init(state, &dma_attr);
#ifdef	__sparc
	if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
		dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

	if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
		dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif

	nds = 0;
	max_nds = iov_attr->iov_wr_nds;
	if (iov_attr->iov_lso_hdr_sz)
		max_nds -= (iov_attr->iov_lso_hdr_sz + sizeof (uint32_t) +
		    0xf) >> 4;	/* 0xf is for rounding up to a multiple of 16 */
	rsvd_lkey = (iov_attr->iov_flags & IBT_IOV_ALT_LKEY) ?
	    iov_attr->iov_alt_lkey : state->hs_devlim.rsv_lkey;
	if ((iov_attr->iov_flags & IBT_IOV_NOSLEEP) == 0) {
		kmflag = KM_SLEEP;
		callback = DDI_DMA_SLEEP;
	} else {
		kmflag = KM_NOSLEEP;
		callback = DDI_DMA_DONTWAIT;
	}

	if (iov_attr->iov_flags & IBT_IOV_BUF) {
		mi_hdl = kmem_alloc(sizeof (*mi_hdl), kmflag);
		if (mi_hdl == NULL)
			return (IBT_INSUFF_RESOURCE);
		sgl = wr->send.wr_sgl;
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))

		status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
		    callback, NULL, &dmahdl);
		if (status != DDI_SUCCESS) {
			kmem_free(mi_hdl, sizeof (*mi_hdl));
			return (IBT_INSUFF_RESOURCE);
		}
		status = ddi_dma_buf_bind_handle(dmahdl, iov_attr->iov_buf,
		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
		    &dmacookie, &cookie_cnt);
		if (status != DDI_DMA_MAPPED) {
			ddi_dma_free_handle(&dmahdl);
			kmem_free(mi_hdl, sizeof (*mi_hdl));
			return (ibc_get_ci_failure(0));
		}
		while (cookie_cnt-- > 0) {
			if (nds > max_nds) {
				status = ddi_dma_unbind_handle(dmahdl);
				if (status != DDI_SUCCESS)
					HERMON_WARNING(state, "failed to "
					    "unbind DMA mapping");
				ddi_dma_free_handle(&dmahdl);
				return (IBT_SGL_TOO_SMALL);
			}
			sgl[nds].ds_va = dmacookie.dmac_laddress;
			sgl[nds].ds_key = rsvd_lkey;
			sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
			nds++;
			if (cookie_cnt != 0)
				ddi_dma_nextcookie(dmahdl, &dmacookie);
		}
		wr->send.wr_nds = nds;
		mi_hdl->imh_len = 1;
		mi_hdl->imh_dmahandle[0] = dmahdl;
		*mi_hdl_p = mi_hdl;
		return (IBT_SUCCESS);
	}

	if (iov_attr->iov_flags & IBT_IOV_RECV)
		sgl = wr->recv.wr_sgl;
	else
		sgl = wr->send.wr_sgl;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))

	len = iov_attr->iov_list_len;
	for (i = 0, j = 0; j < len; j++) {
		if (iov_attr->iov[j].iov_len == 0)
			continue;
		i++;
	}
	mi_hdl = kmem_alloc(sizeof (*mi_hdl) +
	    (i - 1) * sizeof (ddi_dma_handle_t), kmflag);
	if (mi_hdl == NULL)
		return (IBT_INSUFF_RESOURCE);
	mi_hdl->imh_len = i;
	for (i = 0, j = 0; j < len; j++) {
		if (iov_attr->iov[j].iov_len == 0)
			continue;
		status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
		    callback, NULL, &dmahdl);
		if (status != DDI_SUCCESS) {
			ibt_status = IBT_INSUFF_RESOURCE;
			goto fail2;
		}
		status = ddi_dma_addr_bind_handle(dmahdl, iov_attr->iov_as,
		    iov_attr->iov[j].iov_addr, iov_attr->iov[j].iov_len,
		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
		    &dmacookie, &cookie_cnt);
		if (status != DDI_DMA_MAPPED) {
			ibt_status = ibc_get_ci_failure(0);
			goto fail1;
		}
		if (nds + cookie_cnt > max_nds) {
			ibt_status = IBT_SGL_TOO_SMALL;
			goto fail2;
		}
		while (cookie_cnt-- > 0) {
			sgl[nds].ds_va = dmacookie.dmac_laddress;
			sgl[nds].ds_key = rsvd_lkey;
			sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
			nds++;
			if (cookie_cnt != 0)
				ddi_dma_nextcookie(dmahdl, &dmacookie);
		}
		mi_hdl->imh_dmahandle[i] = dmahdl;
		i++;
	}

	if (iov_attr->iov_flags & IBT_IOV_RECV)
		wr->recv.wr_nds = nds;
	else
		wr->send.wr_nds = nds;
	*mi_hdl_p = mi_hdl;
	return (IBT_SUCCESS);

fail1:
	ddi_dma_free_handle(&dmahdl);
fail2:
	while (--i >= 0) {
		status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
		if (status != DDI_SUCCESS)
			HERMON_WARNING(state, "failed to unbind DMA mapping");
		ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
	}
	kmem_free(mi_hdl, sizeof (*mi_hdl) +
	    (len - 1) * sizeof (ddi_dma_handle_t));
	*mi_hdl_p = NULL;
	return (ibt_status);
}

/*
 * hermon_ci_unmap_mem_iov()
 * Unmap the memory
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_unmap_mem_iov(ibc_hca_hdl_t hca, ibc_mi_hdl_t mi_hdl)
{
	int		status, i;
	hermon_state_t	*state;

	state = (hermon_state_t *)hca;

	for (i = mi_hdl->imh_len; --i >= 0; ) {
		status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
		if (status != DDI_SUCCESS)
			HERMON_WARNING(state, "failed to unbind DMA mapping");
		ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
	}
	kmem_free(mi_hdl, sizeof (*mi_hdl) +
	    (mi_hdl->imh_len - 1) * sizeof (ddi_dma_handle_t));
	return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_lkey()
 * Allocate an empty memory region for use with FRWR.
 *    Context: Can be called from user or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_lkey(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_lkey_flags_t flags, uint_t list_sz, ibc_mr_hdl_t *mr_p,
    ibt_pmr_desc_t *mem_desc_p)
{
	hermon_state_t		*state;
	hermon_pdhdl_t		pdhdl;
	hermon_mrhdl_t		mrhdl;
	int			status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))

	ASSERT(mr_p != NULL);
	ASSERT(mem_desc_p != NULL);

	state = (hermon_state_t *)hca;
	pdhdl = (hermon_pdhdl_t)pd;

	if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
		return (IBT_NOT_SUPPORTED);

	status = hermon_mr_alloc_lkey(state, pdhdl, flags, list_sz, &mrhdl);
	if (status != DDI_SUCCESS) {
		return (status);
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

	/* Fill in the mem_desc_p structure */
	mem_desc_p->pmd_iova = 0;
	mem_desc_p->pmd_phys_buf_list_sz = list_sz;
	mem_desc_p->pmd_lkey = mrhdl->mr_lkey;
	/* Only set RKey if remote access was requested */
	if (flags & IBT_KEY_REMOTE) {
		mem_desc_p->pmd_rkey = mrhdl->mr_rkey;
	}
	mem_desc_p->pmd_sync_required = B_FALSE;

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl;
	return (IBT_SUCCESS);
}

/* Physical Register Memory Region */
/*
 * hermon_ci_register_physical_mr()
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_pmr_desc_t *mem_desc_p)
{
	return (IBT_NOT_SUPPORTED);
}

/*
 * hermon_ci_reregister_physical_mr()
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_physical_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibc_pd_hdl_t pd, ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mr_desc_p)
{
	return (IBT_NOT_SUPPORTED);
}

/* Mellanox FMR Support */
/*
 * hermon_ci_create_fmr_pool()
 * Creates a pool of memory regions suitable for FMR registration
 *    Context: Can be called from base context only
 */
static ibt_status_t
hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_fmr_pool_attr_t *params, ibc_fmr_pool_hdl_t *fmr_pool_p)
{
	hermon_state_t	*state;
	hermon_pdhdl_t	pdhdl;
	hermon_fmrhdl_t	fmrpoolhdl;
	int		status;

	state = (hermon_state_t *)hca;

	/* Check for valid PD handle pointer */
	if (pd == NULL) {
		return (IBT_PD_HDL_INVALID);
	}

	pdhdl = (hermon_pdhdl_t)pd;

	/*
	 * Validate the access flags.  Both Remote Write and Remote Atomic
	 * require the Local Write flag to be set
	 */
	if (((params->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
	    (params->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
	    !(params->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
		return (IBT_MR_ACCESS_REQ_INVALID);
	}

	status = hermon_create_fmr_pool(state, pdhdl, params, &fmrpoolhdl);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/* Set fmr_pool from hermon handle */
	*fmr_pool_p = (ibc_fmr_pool_hdl_t)fmrpoolhdl;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_destroy_fmr_pool()
 * Free all resources associated with an FMR pool.
 *    Context: Can be called from base context only.
 */
static ibt_status_t
hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
	hermon_state_t	*state;
	hermon_fmrhdl_t	fmrpoolhdl;
	int		status;

	state = (hermon_state_t *)hca;
	fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;

	status = hermon_destroy_fmr_pool(state, fmrpoolhdl);
	return (status);
}

/*
 * hermon_ci_flush_fmr_pool()
 * Force a flush of the memory tables, cleaning up used FMR resources.
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
	hermon_state_t	*state;
	hermon_fmrhdl_t	fmrpoolhdl;
	int		status;

	state = (hermon_state_t *)hca;

	fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;
	status = hermon_flush_fmr_pool(state, fmrpoolhdl);
	return (status);
}

/*
 * hermon_ci_register_physical_fmr()
 * From the 'pool' of FMR regions passed in, performs register physical
 * operation.
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
    void *ibtl_reserved, ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mem_desc_p)
{
	hermon_state_t		*state;
	hermon_mrhdl_t		mrhdl;
	hermon_fmrhdl_t		fmrpoolhdl;
	int			status;

	ASSERT(mem_pattr != NULL);
	ASSERT(mr_p != NULL);
	ASSERT(mem_desc_p != NULL);

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;

	fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;

	status = hermon_register_physical_fmr(state, fmrpoolhdl, mem_pattr,
	    &mrhdl, mem_desc_p);
	if (status != DDI_SUCCESS) {
		return (status);
	}

	/*
	 * If region is mapped for streaming (i.e. noncoherent), then set
	 * sync is required
	 */
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))
	mem_desc_p->pmd_sync_required = (mrhdl->mr_bindinfo.bi_flags &
	    IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
	if (mem_desc_p->pmd_sync_required == B_TRUE) {
		/* Fill in DMA handle for future sync operations */
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mrhdl->mr_bindinfo))
		mrhdl->mr_bindinfo.bi_dmahdl =
		    (ddi_dma_handle_t)mem_pattr->pmr_ma;
	}

	/* Return the Hermon MR handle */
	*mr_p = (ibc_mr_hdl_t)mrhdl;

	return (IBT_SUCCESS);
}

/*
 * hermon_ci_deregister_fmr()
 * Moves an FMR (specified by 'mr') to the deregistered state.
 *    Context: Can be called from base context only.
 */
static ibt_status_t
hermon_ci_deregister_fmr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
	hermon_state_t		*state;
	hermon_mrhdl_t		mrhdl;
	int			status;

	/* Grab the Hermon softstate pointer */
	state = (hermon_state_t *)hca;
	mrhdl = (hermon_mrhdl_t)mr;

	/*
	 * Deregister the memory region, either "unmap" the FMR or deregister
	 * the normal memory region.
	 */
	status = hermon_deregister_fmr(state, mrhdl);
	return (status);
}

static int
hermon_mem_alloc(hermon_state_t *state, size_t size, ibt_mr_flags_t flags,
    caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_hdl)
{
	ddi_dma_handle_t	dma_hdl;
	ddi_dma_attr_t		dma_attr;
	ddi_acc_handle_t	acc_hdl;
	size_t			real_len;
	int			status;
	int			(*ddi_cb)(caddr_t);
	ibc_mem_alloc_hdl_t	mem_alloc_hdl;

	hermon_dma_attr_init(state, &dma_attr);

	ddi_cb = (flags & IBT_MR_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;

	/* Allocate a DMA handle */
	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, ddi_cb,
	    NULL, &dma_hdl);
	if (status != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	/* Allocate DMA memory */
	status = ddi_dma_mem_alloc(dma_hdl, size,
	    &state->hs_reg_accattr, DDI_DMA_CONSISTENT, ddi_cb,
	    NULL, kaddrp, &real_len, &acc_hdl);
	if (status != DDI_SUCCESS) {
		ddi_dma_free_handle(&dma_hdl);
		return (DDI_FAILURE);
	}

	/* Package the hermon_dma_info contents and return */
	mem_alloc_hdl = kmem_alloc(sizeof (**mem_hdl),
	    (flags & IBT_MR_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP);
	if (mem_alloc_hdl == NULL) {
		ddi_dma_mem_free(&acc_hdl);
		ddi_dma_free_handle(&dma_hdl);
		return (DDI_FAILURE);
	}
	mem_alloc_hdl->ibc_dma_hdl = dma_hdl;
	mem_alloc_hdl->ibc_acc_hdl = acc_hdl;

	*mem_hdl = mem_alloc_hdl;

	return (DDI_SUCCESS);
}

/*
 * hermon_ci_alloc_io_mem()
 *	Allocate dma-able memory
 *
 */
static ibt_status_t
hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size, ibt_mr_flags_t mr_flag,
    caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_alloc_hdl_p)
{
	hermon_state_t	*state;
	int		status;

	/* Grab the Hermon softstate pointer and mem handle */
	state = (hermon_state_t *)hca;

	/* Allocate the memory and handles */
	status = hermon_mem_alloc(state, size, mr_flag, kaddrp,
	    mem_alloc_hdl_p);

	if (status != DDI_SUCCESS) {
		*mem_alloc_hdl_p = NULL;
		*kaddrp = NULL;
		return (status);
	}

	return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_io_mem()
 * Unbind handl and free the memory
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_io_mem(ibc_hca_hdl_t hca, ibc_mem_alloc_hdl_t mem_alloc_hdl)
{
	/* Unbind the handles and free the memory */
	(void) ddi_dma_unbind_handle(mem_alloc_hdl->ibc_dma_hdl);
	ddi_dma_mem_free(&mem_alloc_hdl->ibc_acc_hdl);
	ddi_dma_free_handle(&mem_alloc_hdl->ibc_dma_hdl);
	kmem_free(mem_alloc_hdl, sizeof (*mem_alloc_hdl));

	return (IBT_SUCCESS);
}