xref: /onnv-gate/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c (revision 9879:2547a76dc28b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * ibcm_impl.c
28  *
29  * contains internal functions of IB CM module.
30  *
31  * TBD:
32  * 1. HCA CATASTROPHIC/RECOVERED not handled yet
33  */
34 
35 #include <sys/ib/mgt/ibcm/ibcm_impl.h>
36 #include <sys/disp.h>
37 
38 
39 /* function prototypes */
40 static ibcm_status_t	ibcm_init(void);
41 static ibcm_status_t	ibcm_fini(void);
42 
43 /* Routines to initialize and destory CM global locks and CVs */
44 static void		ibcm_init_locks(void);
45 static void		ibcm_fini_locks(void);
46 
47 /* Routines that initialize/teardown CM's global hca structures */
48 static void		ibcm_init_hcas();
49 static ibcm_status_t	ibcm_fini_hcas();
50 
51 static void		ibcm_init_classportinfo();
52 static void		ibcm_stop_timeout_thread();
53 
54 /* Routines that handle HCA attach/detach asyncs */
55 static void		ibcm_hca_attach(ib_guid_t);
56 static ibcm_status_t	ibcm_hca_detach(ibcm_hca_info_t *);
57 
58 /* Routines that initialize the HCA's port related fields */
59 static ibt_status_t	ibcm_hca_init_port(ibcm_hca_info_t *hcap,
60 			    uint8_t port_index);
61 static ibcm_status_t	ibcm_hca_fini_port(ibcm_hca_info_t *hcap,
62 			    uint8_t port_index);
63 
64 static void ibcm_rc_flow_control_init(void);
65 static void ibcm_rc_flow_control_fini(void);
66 
67 /*
68  * Routines that check if hca's avl trees and sidr lists are free of any
69  * active client resources ie., RC or UD state structures in certain states
70  */
71 static ibcm_status_t	ibcm_check_avl_clean(ibcm_hca_info_t *hcap);
72 static ibcm_status_t	ibcm_check_sidr_clean(ibcm_hca_info_t *hcap);
73 
74 /* Add a new hca structure to CM's global hca list */
75 static ibcm_hca_info_t	*ibcm_add_hca_entry(ib_guid_t hcaguid, uint_t nports);
76 
77 static void		ibcm_comm_est_handler(ibt_async_event_t *);
78 void			ibcm_async_handler(void *, ibt_hca_hdl_t,
79 			    ibt_async_code_t, ibt_async_event_t *);
80 
81 /* Global variables */
82 char			cmlog[] = "ibcm";	/* for debug log messages */
83 ibt_clnt_hdl_t		ibcm_ibt_handle;	/* IBT handle */
84 kmutex_t		ibcm_svc_info_lock;	/* list lock */
85 kcondvar_t		ibcm_svc_info_cv;	/* cv for deregister */
86 kmutex_t		ibcm_recv_mutex;
87 avl_tree_t		ibcm_svc_avl_tree;
88 taskq_t			*ibcm_taskq = NULL;
89 int			taskq_dispatch_fail_cnt;
90 
91 kmutex_t		ibcm_trace_mutex;	/* Trace mutex */
92 kmutex_t		ibcm_trace_print_mutex;	/* Trace print mutex */
93 int			ibcm_conn_max_trcnt = IBCM_MAX_CONN_TRCNT;
94 
95 int			ibcm_enable_trace = 2;	/* Trace level 4 by default */
96 int			ibcm_dtrace = 0; /* conditionally enable more dtrace */
97 
98 _NOTE(MUTEX_PROTECTS_DATA(ibcm_svc_info_lock, ibcm_svc_info_s::{svc_bind_list
99     svc_ref_cnt svc_to_delete}))
100 
101 _NOTE(MUTEX_PROTECTS_DATA(ibcm_svc_info_lock, ibcm_svc_bind_s::{sbind_link}))
102 
103 _NOTE(MUTEX_PROTECTS_DATA(ibcm_trace_mutex, ibcm_conn_trace_s))
104 
105 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_conn_trace_s))
106 
107 _NOTE(MUTEX_PROTECTS_DATA(ibcm_trace_print_mutex, ibcm_debug_buf))
108 
109 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_debug_buf))
110 
111 /*
112  * Initial state is INIT. All hca dr's return success immediately in this
113  * state, without adding or deleting any hca's to CM.
114  */
115 ibcm_finit_state_t	ibcm_finit_state = IBCM_FINIT_INIT;
116 
117 /* mutex and cv to manage hca's reference and resource count(s) */
118 kmutex_t		ibcm_global_hca_lock;
119 kcondvar_t		ibcm_global_hca_cv;
120 
121 /* mutex and cv to sa session open */
122 kmutex_t		ibcm_sa_open_lock;
123 kcondvar_t		ibcm_sa_open_cv;
124 int			ibcm_sa_timeout_delay = 1;		/* in ticks */
125 _NOTE(MUTEX_PROTECTS_DATA(ibcm_sa_open_lock,
126     ibcm_port_info_s::{port_ibmf_saa_hdl port_saa_open_in_progress}))
127 
128 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_saa_hdl}))
129 
130 /* serialize sm notice callbacks */
131 kmutex_t		ibcm_sm_notice_serialize_lock;
132 
133 _NOTE(LOCK_ORDER(ibcm_sm_notice_serialize_lock ibcm_global_hca_lock))
134 
135 _NOTE(MUTEX_PROTECTS_DATA(ibcm_global_hca_lock, ibcm_hca_info_s::{hca_state
136     hca_svc_cnt hca_acc_cnt hca_res_cnt hca_next}))
137 
138 _NOTE(MUTEX_PROTECTS_DATA(ibcm_global_hca_lock,
139     ibcm_port_info_s::{port_ibmf_hdl}))
140 
141 _NOTE(MUTEX_PROTECTS_DATA(ibcm_sm_notice_serialize_lock,
142     ibcm_port_info_s::{port_event_status}))
143 
144 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_hca_info_s::{hca_state}))
145 _NOTE(DATA_READABLE_WITHOUT_LOCK(
146     ibcm_hca_info_s::{hca_port_info.port_ibmf_hdl}))
147 
148 /* mutex for CM's qp list management */
149 kmutex_t		ibcm_qp_list_lock;
150 
151 _NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_port_info_s::{port_qplist}))
152 _NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_qp_list_s))
153 _NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_qp_list_s))
154 
155 kcondvar_t		ibcm_timeout_list_cv;
156 kcondvar_t		ibcm_timeout_thread_done_cv;
157 kt_did_t		ibcm_timeout_thread_did;
158 ibcm_state_data_t	*ibcm_timeout_list_hdr, *ibcm_timeout_list_tail;
159 ibcm_ud_state_data_t	*ibcm_ud_timeout_list_hdr, *ibcm_ud_timeout_list_tail;
160 kmutex_t		ibcm_timeout_list_lock;
161 uint8_t			ibcm_timeout_list_flags = 0;
162 pri_t			ibcm_timeout_thread_pri = MINCLSYSPRI;
163 
164 _NOTE(MUTEX_PROTECTS_DATA(ibcm_timeout_list_lock,
165     ibcm_state_data_s::timeout_next))
166 
167 _NOTE(MUTEX_PROTECTS_DATA(ibcm_timeout_list_lock,
168     ibcm_ud_state_data_s::ud_timeout_next))
169 
170 /*
171  * Flow control logic for open_rc_channel uses the following.
172  */
173 
174 struct ibcm_open_s {
175 	kmutex_t		mutex;
176 	kcondvar_t		cv;
177 	uint8_t			task_running;
178 	uint_t			queued;
179 	uint_t			exit_deferred;
180 	uint_t			in_progress;
181 	uint_t			in_progress_max;
182 	uint_t			sends;
183 	uint_t			sends_max;
184 	uint_t			sends_lowat;
185 	uint_t			sends_hiwat;
186 	ibcm_state_data_t	*tail;
187 	ibcm_state_data_t	head;
188 } ibcm_open;
189 
190 static void ibcm_open_task(void *);
191 
192 /*
193  * Flow control logic for SA access and close_rc_channel calls follows.
194  */
195 
196 int ibcm_close_simul_max	= 12;
197 int ibcm_lapr_simul_max		= 12;
198 int ibcm_saa_simul_max		= 8;
199 
200 typedef struct ibcm_flow1_s {
201 	struct ibcm_flow1_s	*link;
202 	kcondvar_t		cv;
203 	uint8_t			waiters;	/* 1 to IBCM_FLOW_SIMUL_MAX */
204 } ibcm_flow1_t;
205 
206 typedef struct ibcm_flow_s {
207 	ibcm_flow1_t		*list;
208 	uint_t			simul;	/* #requests currently outstanding */
209 	uint_t			simul_max;
210 	uint_t			waiters_per_chunk;
211 	uint_t			lowat;
212 	uint_t			lowat_default;
213 	/* statistics */
214 	uint_t			total;
215 } ibcm_flow_t;
216 
217 ibcm_flow_t ibcm_saa_flow;
218 ibcm_flow_t ibcm_close_flow;
219 ibcm_flow_t ibcm_lapr_flow;
220 
221 /* NONBLOCKING close requests are queued */
222 struct ibcm_close_s {
223 	kmutex_t		mutex;
224 	ibcm_state_data_t	*tail;
225 	ibcm_state_data_t	head;
226 } ibcm_close;
227 
228 static ibt_clnt_modinfo_t ibcm_ibt_modinfo = {	/* Client's modinfop */
229 	IBTI_V_CURR,
230 	IBT_CM,
231 	ibcm_async_handler,
232 	NULL,
233 	"IBCM"
234 };
235 
236 /* IBCM's list of HCAs registered with it */
237 static ibcm_hca_info_t	*ibcm_hca_listp = NULL;	/* CM's HCA list */
238 
239 /* Array of CM state call table functions */
240 ibcm_state_handler_t	ibcm_sm_funcs_tbl[] = {
241 	ibcm_process_req_msg,
242 	ibcm_process_mra_msg,
243 	ibcm_process_rej_msg,
244 	ibcm_process_rep_msg,
245 	ibcm_process_rtu_msg,
246 	ibcm_process_dreq_msg,
247 	ibcm_process_drep_msg,
248 	ibcm_process_sidr_req_msg,
249 	ibcm_process_sidr_rep_msg,
250 	ibcm_process_lap_msg,
251 	ibcm_process_apr_msg
252 };
253 
254 /* the following globals are CM tunables */
255 ibt_rnr_nak_time_t	ibcm_default_rnr_nak_time = IBT_RNR_NAK_655ms;
256 
257 uint32_t	ibcm_max_retries = IBCM_MAX_RETRIES;
258 clock_t		ibcm_local_processing_time = IBCM_LOCAL_RESPONSE_TIME;
259 clock_t		ibcm_remote_response_time = IBCM_REMOTE_RESPONSE_TIME;
260 ib_time_t	ibcm_max_sidr_rep_proctime = IBCM_MAX_SIDR_PROCESS_TIME;
261 ib_time_t	ibcm_max_sidr_pktlife_time = IBCM_MAX_SIDR_PKT_LIFE_TIME;
262 
263 ib_time_t	ibcm_max_sidr_rep_store_time = 18;
264 uint32_t	ibcm_wait_for_acc_cnt_timeout = 2000000;	/* 2 sec */
265 uint32_t	ibcm_wait_for_res_cnt_timeout = 2000000;	/* 2 sec */
266 
267 ib_time_t	ibcm_max_ib_pkt_lt = IBCM_MAX_IB_PKT_LT;
268 ib_time_t	ibcm_max_ib_mad_pkt_lt = IBCM_MAX_IB_MAD_PKT_LT;
269 
270 /*
271  * This delay accounts for time involved in various activities as follows :
272  *
273  * IBMF delays for posting the MADs in non-blocking mode
274  * IBMF delays for receiving the MADs and delivering to CM
275  * CM delays in processing the MADs before invoking client handlers,
276  * Any other delays associated with HCA driver in processing the MADs and
277  * 	other subsystems that CM may invoke (ex : SA, HCA driver)
278  */
279 uint32_t	ibcm_sw_delay	= 1000;	/* 1000us / 1ms */
280 uint32_t	ibcm_max_sa_retries = IBCM_MAX_SA_RETRIES + 1;
281 
282 /*	approx boot time */
283 uint32_t	ibcm_adj_btime = 4;	/* 4 seconds */
284 
285 /*
286  * The information in ibcm_clpinfo is kept in wireformat and is setup at
287  * init time, and used read-only after that
288  */
289 ibcm_classportinfo_msg_t	ibcm_clpinfo;
290 
291 char	*event_str[] = {
292 	"NEVER SEE THIS             ",
293 	"SESSION_ID                 ",
294 	"CHAN_HDL                   ",
295 	"LOCAL_COMID/HCA/PORT       ",
296 	"LOCAL_QPN                  ",
297 	"REMOTE_COMID/HCA           ",
298 	"REMOTE_QPN                 ",
299 	"BASE_TIME                  ",
300 	"INCOMING_REQ               ",
301 	"INCOMING_REP               ",
302 	"INCOMING_RTU               ",
303 	"INCOMING_COMEST            ",
304 	"INCOMING_MRA               ",
305 	"INCOMING_REJ               ",
306 	"INCOMING_LAP               ",
307 	"INCOMING_APR               ",
308 	"INCOMING_DREQ              ",
309 	"INCOMING_DREP              ",
310 	"OUTGOING_REQ               ",
311 	"OUTGOING_REP               ",
312 	"OUTGOING_RTU               ",
313 	"OUTGOING_LAP               ",
314 	"OUTGOING_APR               ",
315 	"OUTGOING_MRA               ",
316 	"OUTGOING_REJ               ",
317 	"OUTGOING_DREQ              ",
318 	"OUTGOING_DREP              ",
319 	"REQ_POST_COMPLETE          ",
320 	"REP_POST_COMPLETE          ",
321 	"RTU_POST_COMPLETE          ",
322 	"MRA_POST_COMPLETE          ",
323 	"REJ_POST_COMPLETE          ",
324 	"LAP_POST_COMPLETE          ",
325 	"APR_POST_COMPLETE          ",
326 	"DREQ_POST_COMPLETE         ",
327 	"DREP_POST_COMPLETE         ",
328 	"TIMEOUT_REP                ",
329 	"CALLED_REQ_RCVD_EVENT      ",
330 	"RET_REQ_RCVD_EVENT         ",
331 	"CALLED_REP_RCVD_EVENT      ",
332 	"RET_REP_RCVD_EVENT         ",
333 	"CALLED_CONN_EST_EVENT      ",
334 	"RET_CONN_EST_EVENT         ",
335 	"CALLED_CONN_FAIL_EVENT     ",
336 	"RET_CONN_FAIL_EVENT        ",
337 	"CALLED_CONN_CLOSE_EVENT    ",
338 	"RET_CONN_CLOSE_EVENT       ",
339 	"INIT_INIT                  ",
340 	"INIT_INIT_FAIL             ",
341 	"INIT_RTR                   ",
342 	"INIT_RTR_FAIL              ",
343 	"RTR_RTS                    ",
344 	"RTR_RTS_FAIL               ",
345 	"RTS_RTS                    ",
346 	"RTS_RTS_FAIL               ",
347 	"TO_ERROR                   ",
348 	"ERROR_FAIL                 ",
349 	"SET_ALT                    ",
350 	"SET_ALT_FAIL               ",
351 	"STALE_DETECT               ",
352 	"OUTGOING_REQ_RETRY         ",
353 	"OUTGOING_REP_RETRY         ",
354 	"OUTGOING_LAP_RETRY         ",
355 	"OUTGOING_MRA_RETRY         ",
356 	"OUTGOING_DREQ_RETRY        ",
357 	"NEVER SEE THIS             "
358 };
359 
360 char	ibcm_debug_buf[IBCM_DEBUG_BUF_SIZE];
361 
362 _NOTE(SCHEME_PROTECTS_DATA("used in a localized function consistently",
363     ibcm_debug_buf))
364 _NOTE(READ_ONLY_DATA(ibcm_taskq))
365 
366 _NOTE(MUTEX_PROTECTS_DATA(ibcm_timeout_list_lock, ibcm_timeout_list_flags))
367 _NOTE(MUTEX_PROTECTS_DATA(ibcm_timeout_list_lock, ibcm_timeout_list_hdr))
368 _NOTE(MUTEX_PROTECTS_DATA(ibcm_timeout_list_lock, ibcm_ud_timeout_list_hdr))
369 
370 #ifdef DEBUG
371 int		ibcm_test_mode = 0;	/* set to 1, if running tests */
372 #endif
373 
374 
375 /* Module Driver Info */
376 static struct modlmisc ibcm_modlmisc = {
377 	&mod_miscops,
378 	"IB Communication Manager"
379 };
380 
381 /* Module Linkage */
382 static struct modlinkage ibcm_modlinkage = {
383 	MODREV_1,
384 	&ibcm_modlmisc,
385 	NULL
386 };
387 
388 
389 int
390 _init(void)
391 {
392 	int		rval;
393 	ibcm_status_t	status;
394 
395 	status = ibcm_init();
396 	if (status != IBCM_SUCCESS) {
397 		IBTF_DPRINTF_L2(cmlog, "_init: ibcm failed %d", status);
398 		return (EINVAL);
399 	}
400 
401 	rval = mod_install(&ibcm_modlinkage);
402 	if (rval != 0) {
403 		IBTF_DPRINTF_L2(cmlog, "_init: ibcm mod_install failed %d",
404 		    rval);
405 		(void) ibcm_fini();
406 	}
407 
408 	IBTF_DPRINTF_L5(cmlog, "_init: ibcm successful");
409 	return (rval);
410 
411 }
412 
413 
414 int
415 _info(struct modinfo *modinfop)
416 {
417 	return (mod_info(&ibcm_modlinkage, modinfop));
418 }
419 
420 
421 int
422 _fini(void)
423 {
424 	int status;
425 
426 	if (ibcm_fini() != IBCM_SUCCESS)
427 		return (EBUSY);
428 
429 	if ((status = mod_remove(&ibcm_modlinkage)) != 0) {
430 		IBTF_DPRINTF_L2(cmlog, "_fini: ibcm mod_remove failed %d",
431 		    status);
432 		return (status);
433 	}
434 
435 	IBTF_DPRINTF_L5(cmlog, "_fini: ibcm successful");
436 
437 	return (status);
438 }
439 
440 /* Initializes all global mutex and CV in cm module */
441 static void
442 ibcm_init_locks()
443 {
444 
445 	/* Verify CM MAD sizes */
446 #ifdef DEBUG
447 
448 	if (ibcm_test_mode > 1) {
449 
450 		IBTF_DPRINTF_L1(cmlog, "REQ MAD SIZE %d",
451 		    sizeof (ibcm_req_msg_t));
452 		IBTF_DPRINTF_L1(cmlog, "REP MAD SIZE %d",
453 		    sizeof (ibcm_rep_msg_t));
454 		IBTF_DPRINTF_L1(cmlog, "RTU MAD SIZE %d",
455 		    sizeof (ibcm_rtu_msg_t));
456 		IBTF_DPRINTF_L1(cmlog, "MRA MAD SIZE %d",
457 		    sizeof (ibcm_mra_msg_t));
458 		IBTF_DPRINTF_L1(cmlog, "REJ MAD SIZE %d",
459 		    sizeof (ibcm_rej_msg_t));
460 		IBTF_DPRINTF_L1(cmlog, "LAP MAD SIZE %d",
461 		    sizeof (ibcm_lap_msg_t));
462 		IBTF_DPRINTF_L1(cmlog, "APR MAD SIZE %d",
463 		    sizeof (ibcm_apr_msg_t));
464 		IBTF_DPRINTF_L1(cmlog, "DREQ MAD SIZE %d",
465 		    sizeof (ibcm_dreq_msg_t));
466 		IBTF_DPRINTF_L1(cmlog, "DREP MAD SIZE %d",
467 		    sizeof (ibcm_drep_msg_t));
468 		IBTF_DPRINTF_L1(cmlog, "SIDR REQ MAD SIZE %d",
469 		    sizeof (ibcm_sidr_req_msg_t));
470 		IBTF_DPRINTF_L1(cmlog, "SIDR REP MAD SIZE %d",
471 		    sizeof (ibcm_sidr_rep_msg_t));
472 	}
473 
474 #endif
475 
476 	/* Create all global locks within cm module */
477 	mutex_init(&ibcm_svc_info_lock, NULL, MUTEX_DEFAULT, NULL);
478 	mutex_init(&ibcm_timeout_list_lock, NULL, MUTEX_DEFAULT, NULL);
479 	mutex_init(&ibcm_global_hca_lock, NULL, MUTEX_DEFAULT, NULL);
480 	mutex_init(&ibcm_sa_open_lock, NULL, MUTEX_DEFAULT, NULL);
481 	mutex_init(&ibcm_recv_mutex, NULL, MUTEX_DEFAULT, NULL);
482 	mutex_init(&ibcm_sm_notice_serialize_lock, NULL, MUTEX_DEFAULT, NULL);
483 	mutex_init(&ibcm_qp_list_lock, NULL, MUTEX_DEFAULT, NULL);
484 	mutex_init(&ibcm_trace_mutex, NULL, MUTEX_DEFAULT, NULL);
485 	mutex_init(&ibcm_trace_print_mutex, NULL, MUTEX_DEFAULT, NULL);
486 	cv_init(&ibcm_svc_info_cv, NULL, CV_DRIVER, NULL);
487 	cv_init(&ibcm_timeout_list_cv, NULL, CV_DRIVER, NULL);
488 	cv_init(&ibcm_timeout_thread_done_cv, NULL, CV_DRIVER, NULL);
489 	cv_init(&ibcm_global_hca_cv, NULL, CV_DRIVER, NULL);
490 	cv_init(&ibcm_sa_open_cv, NULL, CV_DRIVER, NULL);
491 	avl_create(&ibcm_svc_avl_tree, ibcm_svc_compare,
492 	    sizeof (ibcm_svc_info_t),
493 	    offsetof(struct ibcm_svc_info_s, svc_link));
494 
495 	IBTF_DPRINTF_L5(cmlog, "ibcm_init_locks: done");
496 }
497 
498 /* Destroys all global mutex and CV in cm module */
499 static void
500 ibcm_fini_locks()
501 {
502 	/* Destroy all global locks within cm module */
503 	mutex_destroy(&ibcm_svc_info_lock);
504 	mutex_destroy(&ibcm_timeout_list_lock);
505 	mutex_destroy(&ibcm_global_hca_lock);
506 	mutex_destroy(&ibcm_sa_open_lock);
507 	mutex_destroy(&ibcm_recv_mutex);
508 	mutex_destroy(&ibcm_sm_notice_serialize_lock);
509 	mutex_destroy(&ibcm_qp_list_lock);
510 	mutex_destroy(&ibcm_trace_mutex);
511 	mutex_destroy(&ibcm_trace_print_mutex);
512 	cv_destroy(&ibcm_svc_info_cv);
513 	cv_destroy(&ibcm_timeout_list_cv);
514 	cv_destroy(&ibcm_timeout_thread_done_cv);
515 	cv_destroy(&ibcm_global_hca_cv);
516 	cv_destroy(&ibcm_sa_open_cv);
517 	avl_destroy(&ibcm_svc_avl_tree);
518 
519 	IBTF_DPRINTF_L5(cmlog, "ibcm_fini_locks: done");
520 }
521 
522 
523 /* Initialize CM's classport info */
524 static void
525 ibcm_init_classportinfo()
526 {
527 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_clpinfo));
528 
529 	ibcm_clpinfo.BaseVersion = IBCM_MAD_BASE_VERSION;
530 	ibcm_clpinfo.ClassVersion = IBCM_MAD_CLASS_VERSION;
531 
532 	/* For now, CM supports same capabilities at all ports */
533 	ibcm_clpinfo.CapabilityMask =
534 	    h2b16(IBCM_CPINFO_CAP_RC | IBCM_CPINFO_CAP_SIDR);
535 
536 	/* Bits 0-7 are all 0 for Communication Mgmt Class */
537 
538 	/* For now, CM has the same respvalue at all ports */
539 	ibcm_clpinfo.RespTimeValue_plus =
540 	    h2b32(ibt_usec2ib(ibcm_local_processing_time) & 0x1f);
541 
542 	/* For now, redirect fields are set to 0 */
543 	/* Trap fields are not applicable to CM, hence set to 0 */
544 
545 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_clpinfo));
546 	IBTF_DPRINTF_L5(cmlog, "ibcm_init_classportinfo: done");
547 }
548 
549 /*
550  * ibcm_init():
551  * 	- call ibt_attach()
552  * 	- create AVL trees
553  *	- Attach HCA handlers that are already present before
554  *	CM got loaded.
555  *
556  * Arguments:	NONE
557  *
558  * Return values:
559  *	IBCM_SUCCESS - success
560  */
561 static ibcm_status_t
562 ibcm_init(void)
563 {
564 	ibt_status_t	status;
565 	kthread_t	*t;
566 
567 	IBTF_DPRINTF_L3(cmlog, "ibcm_init:");
568 
569 	ibcm_init_classportinfo();
570 
571 	if (ibcm_init_ids() != IBCM_SUCCESS) {
572 		IBTF_DPRINTF_L1(cmlog, "ibcm_init: "
573 		    "fatal error: vmem_create() failed");
574 		return (IBCM_FAILURE);
575 	}
576 	ibcm_init_locks();
577 
578 	if (ibcm_ar_init() != IBCM_SUCCESS) {
579 		IBTF_DPRINTF_L1(cmlog, "ibcm_init: "
580 		    "fatal error: ibcm_ar_init() failed");
581 		ibcm_fini_ids();
582 		ibcm_fini_locks();
583 		return (IBCM_FAILURE);
584 	}
585 	ibcm_rc_flow_control_init();
586 
587 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_taskq))
588 	ibcm_taskq = system_taskq;
589 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_taskq))
590 
591 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_timeout_list_flags))
592 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_timeout_thread_did))
593 
594 	/* Start the timeout list processing thread */
595 	ibcm_timeout_list_flags = 0;
596 	t = thread_create(NULL, 0, ibcm_process_tlist, 0, 0, &p0, TS_RUN,
597 	    ibcm_timeout_thread_pri);
598 	ibcm_timeout_thread_did = t->t_did;
599 
600 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_timeout_list_flags))
601 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_timeout_thread_did))
602 
603 	/*
604 	 * NOTE : if ibt_attach is done after ibcm_init_hcas, then some
605 	 * HCA DR events may be lost. CM could call re-init hca list
606 	 * again, but it is more complicated. Some HCA's DR's lost may
607 	 * be HCA detach, which makes hca list re-syncing and locking more
608 	 * complex
609 	 */
610 	status = ibt_attach(&ibcm_ibt_modinfo, NULL, NULL, &ibcm_ibt_handle);
611 	if (status != IBT_SUCCESS) {
612 		IBTF_DPRINTF_L2(cmlog, "ibcm_init(): ibt_attach failed %d",
613 		    status);
614 		(void) ibcm_ar_fini();
615 		ibcm_stop_timeout_thread();
616 		ibcm_fini_ids();
617 		ibcm_fini_locks();
618 		ibcm_rc_flow_control_fini();
619 		return (IBCM_FAILURE);
620 	}
621 
622 	/* Block all HCA attach/detach asyncs */
623 	mutex_enter(&ibcm_global_hca_lock);
624 
625 	ibcm_init_hcas();
626 	ibcm_finit_state = IBCM_FINIT_IDLE;
627 
628 	ibcm_path_cache_init();
629 
630 	/* Unblock any waiting HCA DR asyncs in CM */
631 	mutex_exit(&ibcm_global_hca_lock);
632 
633 	IBTF_DPRINTF_L4(cmlog, "ibcm_init: done");
634 	return (IBCM_SUCCESS);
635 }
636 
637 /* Allocates and initializes the "per hca" global data in CM */
638 static void
639 ibcm_init_hcas()
640 {
641 	uint_t	num_hcas = 0;
642 	ib_guid_t *guid_array;
643 	int i;
644 
645 	IBTF_DPRINTF_L5(cmlog, "ibcm_init_hcas:");
646 
647 	/* Get the number of HCAs */
648 	num_hcas = ibt_get_hca_list(&guid_array);
649 	IBTF_DPRINTF_L4(cmlog, "ibcm_init_hcas: ibt_get_hca_list() "
650 	    "returned %d hcas", num_hcas);
651 
652 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
653 
654 	for (i = 0; i < num_hcas; i++)
655 		ibcm_hca_attach(guid_array[i]);
656 
657 	if (num_hcas)
658 		ibt_free_hca_list(guid_array, num_hcas);
659 
660 	IBTF_DPRINTF_L5(cmlog, "ibcm_init_hcas: done");
661 }
662 
663 
664 /*
665  * ibcm_fini():
666  * 	- Deregister w/ ibt
667  * 	- Cleanup IBCM HCA listp
668  * 	- Destroy mutexes
669  *
670  * Arguments:	NONE
671  *
672  * Return values:
673  *	IBCM_SUCCESS - success
674  */
675 static ibcm_status_t
676 ibcm_fini(void)
677 {
678 	ibt_status_t	status;
679 
680 	IBTF_DPRINTF_L3(cmlog, "ibcm_fini:");
681 
682 	/*
683 	 * CM assumes that the all general clients got rid of all the
684 	 * established connections and service registrations, completed all
685 	 * pending SIDR operations before a call to ibcm_fini()
686 	 */
687 
688 	if (ibcm_ar_fini() != IBCM_SUCCESS) {
689 		IBTF_DPRINTF_L2(cmlog, "ibcm_fini: ibcm_ar_fini failed");
690 		return (IBCM_FAILURE);
691 	}
692 
693 	/* cleanup the svcinfo list */
694 	mutex_enter(&ibcm_svc_info_lock);
695 	if (avl_first(&ibcm_svc_avl_tree) != NULL) {
696 		IBTF_DPRINTF_L2(cmlog, "ibcm_fini: "
697 		    "ibcm_svc_avl_tree is not empty");
698 		mutex_exit(&ibcm_svc_info_lock);
699 		return (IBCM_FAILURE);
700 	}
701 	mutex_exit(&ibcm_svc_info_lock);
702 
703 	/* disables any new hca attach/detaches */
704 	mutex_enter(&ibcm_global_hca_lock);
705 
706 	ibcm_finit_state = IBCM_FINIT_BUSY;
707 
708 	if (ibcm_fini_hcas() != IBCM_SUCCESS) {
709 		IBTF_DPRINTF_L2(cmlog, "ibcm_fini: "
710 		    "some hca's still have client resources");
711 
712 		/* First, re-initialize the hcas */
713 		ibcm_init_hcas();
714 		/* and then enable the HCA asyncs */
715 		ibcm_finit_state = IBCM_FINIT_IDLE;
716 		mutex_exit(&ibcm_global_hca_lock);
717 		if (ibcm_ar_init() != IBCM_SUCCESS) {
718 			IBTF_DPRINTF_L1(cmlog, "ibcm_fini:ibcm_ar_init failed");
719 		}
720 		return (IBCM_FAILURE);
721 	}
722 
723 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_timeout_list_hdr))
724 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_ud_timeout_list_hdr))
725 
726 	ASSERT(ibcm_timeout_list_hdr == NULL);
727 	ASSERT(ibcm_ud_timeout_list_hdr == NULL);
728 
729 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_timeout_list_hdr))
730 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_ud_timeout_list_hdr))
731 
732 	/* Release any pending asyncs on ibcm_global_hca_lock */
733 	ibcm_finit_state = IBCM_FINIT_SUCCESS;
734 	mutex_exit(&ibcm_global_hca_lock);
735 
736 	ibcm_stop_timeout_thread();
737 
738 	/*
739 	 * Detach from IBTL. Waits until all pending asyncs are complete.
740 	 * Above cv_broadcast wakes up any waiting hca attach/detach asyncs
741 	 */
742 	status = ibt_detach(ibcm_ibt_handle);
743 
744 	/* if detach fails, CM didn't free up some resources, so assert */
745 	if (status != IBT_SUCCESS)
746 		IBTF_DPRINTF_L1(cmlog, "ibcm_fini: ibt_detach failed %d",
747 		    status);
748 
749 	ibcm_rc_flow_control_fini();
750 
751 	ibcm_path_cache_fini();
752 
753 	ibcm_fini_ids();
754 	ibcm_fini_locks();
755 	IBTF_DPRINTF_L3(cmlog, "ibcm_fini: done");
756 	return (IBCM_SUCCESS);
757 }
758 
759 /* This routine exit's the ibcm timeout thread  */
760 static void
761 ibcm_stop_timeout_thread()
762 {
763 	mutex_enter(&ibcm_timeout_list_lock);
764 
765 	/* Stop the timeout list processing thread */
766 	ibcm_timeout_list_flags =
767 	    ibcm_timeout_list_flags | IBCM_TIMEOUT_THREAD_EXIT;
768 
769 	/* Wake up, if the timeout thread is on a cv_wait */
770 	cv_signal(&ibcm_timeout_list_cv);
771 
772 	mutex_exit(&ibcm_timeout_list_lock);
773 	thread_join(ibcm_timeout_thread_did);
774 
775 	IBTF_DPRINTF_L5(cmlog, "ibcm_stop_timeout_thread: done");
776 }
777 
778 
779 /* Attempts to release all the hca's associated with CM */
780 static ibcm_status_t
781 ibcm_fini_hcas()
782 {
783 	ibcm_hca_info_t *hcap, *next;
784 
785 	IBTF_DPRINTF_L4(cmlog, "ibcm_fini_hcas:");
786 
787 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
788 
789 	hcap = ibcm_hca_listp;
790 	while (hcap != NULL) {
791 		next = hcap->hca_next;
792 		if (ibcm_hca_detach(hcap) != IBCM_SUCCESS) {
793 			ibcm_hca_listp = hcap;
794 			return (IBCM_FAILURE);
795 		}
796 		hcap = next;
797 	}
798 
799 	IBTF_DPRINTF_L4(cmlog, "ibcm_fini_hcas: SUCCEEDED");
800 	return (IBCM_SUCCESS);
801 }
802 
803 
804 /*
805  * ibcm_hca_attach():
806  *	Called as an asynchronous event to notify CM of an attach of HCA.
807  *	Here ibcm_hca_info_t is initialized and all fields are
808  *	filled in along with SA Access handles and IBMA handles.
809  *	Also called from ibcm_init to initialize ibcm_hca_info_t's for each
810  *	hca's
811  *
812  * Arguments: (WILL CHANGE BASED ON ASYNC EVENT CODE)
813  *	hca_guid	- HCA's guid
814  *
815  * Return values: NONE
816  */
817 static void
818 ibcm_hca_attach(ib_guid_t hcaguid)
819 {
820 	int			i;
821 	ibt_status_t		status;
822 	uint_t			nports = 0;
823 	ibcm_hca_info_t		*hcap;
824 	ibt_hca_attr_t		hca_attrs;
825 
826 	IBTF_DPRINTF_L3(cmlog, "ibcm_hca_attach: guid = 0x%llX", hcaguid);
827 
828 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
829 
830 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hcap))
831 
832 	status = ibt_query_hca_byguid(hcaguid, &hca_attrs);
833 	if (status != IBT_SUCCESS) {
834 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_attach: "
835 		    "ibt_query_hca_byguid failed = %d", status);
836 		return;
837 	}
838 	nports = hca_attrs.hca_nports;
839 
840 	IBTF_DPRINTF_L4(cmlog, "ibcm_hca_attach: num ports = %x", nports);
841 
842 	if ((hcap = ibcm_add_hca_entry(hcaguid, nports)) == NULL)
843 		return;
844 
845 	hcap->hca_guid = hcaguid;	/* Set GUID */
846 	hcap->hca_num_ports = nports;	/* Set number of ports */
847 
848 	if (ibcm_init_hca_ids(hcap) != IBCM_SUCCESS) {
849 		ibcm_delete_hca_entry(hcap);
850 		return;
851 	}
852 
853 	/* Store the static hca attribute data */
854 	hcap->hca_caps = hca_attrs.hca_flags;
855 	hcap->hca_vendor_id = hca_attrs.hca_vendor_id;
856 	hcap->hca_device_id = hca_attrs.hca_device_id;
857 	hcap->hca_ack_delay = hca_attrs.hca_local_ack_delay;
858 	hcap->hca_max_rdma_in_qp = hca_attrs.hca_max_rdma_in_qp;
859 	hcap->hca_max_rdma_out_qp = hca_attrs.hca_max_rdma_out_qp;
860 
861 	/* loop thru nports and initialize IBMF handles */
862 	for (i = 0; i < hcap->hca_num_ports; i++) {
863 		status = ibt_get_port_state_byguid(hcaguid, i + 1, NULL, NULL);
864 		if (status != IBT_SUCCESS) {
865 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_attach: "
866 			    "port_num %d state DOWN", i + 1);
867 		}
868 
869 		hcap->hca_port_info[i].port_hcap = hcap;
870 		hcap->hca_port_info[i].port_num = i+1;
871 
872 		if (ibcm_hca_init_port(hcap, i) != IBT_SUCCESS)
873 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_attach: "
874 			    "ibcm_hca_init_port failed %d port_num %d",
875 			    status, i+1);
876 	}
877 
878 	/* create the "active" CM AVL tree */
879 	avl_create(&hcap->hca_active_tree, ibcm_active_node_compare,
880 	    sizeof (ibcm_state_data_t),
881 	    offsetof(struct ibcm_state_data_s, avl_active_link));
882 
883 	/* create the "passive" CM AVL tree */
884 	avl_create(&hcap->hca_passive_tree, ibcm_passive_node_compare,
885 	    sizeof (ibcm_state_data_t),
886 	    offsetof(struct ibcm_state_data_s, avl_passive_link));
887 
888 	/* create the "passive comid" CM AVL tree */
889 	avl_create(&hcap->hca_passive_comid_tree,
890 	    ibcm_passive_comid_node_compare,
891 	    sizeof (ibcm_state_data_t),
892 	    offsetof(struct ibcm_state_data_s, avl_passive_comid_link));
893 
894 	/*
895 	 * Mark the state of the HCA to "attach" only at the end
896 	 * Now CM starts accepting incoming MADs and client API calls
897 	 */
898 	hcap->hca_state = IBCM_HCA_ACTIVE;
899 
900 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hcap))
901 
902 	IBTF_DPRINTF_L3(cmlog, "ibcm_hca_attach: ATTACH Done");
903 }
904 
905 /*
906  * ibcm_hca_detach():
907  *	Called as an asynchronous event to notify CM of a detach of HCA.
908  *	Here ibcm_hca_info_t is freed up and all fields that
909  *	were initialized earlier are cleaned up
910  *
911  * Arguments: (WILL CHANGE BASED ON ASYNC EVENT CODE)
912  *	hca_guid    - HCA's guid
913  *
914  * Return values:
915  *	IBCM_SUCCESS	- able to detach HCA
916  *	IBCM_FAILURE	- failed to detach HCA
917  */
918 static ibcm_status_t
919 ibcm_hca_detach(ibcm_hca_info_t *hcap)
920 {
921 	int		port_index, i;
922 	ibcm_status_t	status = IBCM_SUCCESS;
923 	clock_t		absolute_time;
924 
925 	IBTF_DPRINTF_L3(cmlog, "ibcm_hca_detach: hcap = 0x%p guid = 0x%llX",
926 	    hcap, hcap->hca_guid);
927 
928 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
929 
930 	/*
931 	 * Declare hca is going away to all CM clients. Wait until the
932 	 * access count becomes zero.
933 	 */
934 	hcap->hca_state = IBCM_HCA_NOT_ACTIVE;
935 
936 	/* wait on response CV */
937 	absolute_time = ddi_get_lbolt() +
938 	    drv_usectohz(ibcm_wait_for_acc_cnt_timeout);
939 
940 	while (hcap->hca_acc_cnt > 0)
941 		if (cv_timedwait(&ibcm_global_hca_cv, &ibcm_global_hca_lock,
942 		    absolute_time) == -1)
943 			break;
944 
945 	if (hcap->hca_acc_cnt != 0) {
946 		/* We got a timeout */
947 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: Aborting due"
948 		    " to timeout on hca_acc_cnt %u, \n Some CM Clients are "
949 		    "still active, looks like we need to wait some more time "
950 		    "(ibcm_wait_for_acc_cnt_timeout).", hcap->hca_acc_cnt);
951 		hcap->hca_state = IBCM_HCA_ACTIVE;
952 		return (IBCM_FAILURE);
953 	}
954 
955 	/*
956 	 * First make sure, there are no active users of ibma handles,
957 	 * and then de-register handles.
958 	 */
959 
960 	/* make sure that there are no "Service"s registered w/ this HCA. */
961 	if (hcap->hca_svc_cnt != 0) {
962 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: "
963 		    "Active services still there %d", hcap->hca_svc_cnt);
964 		hcap->hca_state = IBCM_HCA_ACTIVE;
965 		return (IBCM_FAILURE);
966 	}
967 
968 	if (ibcm_check_sidr_clean(hcap) != IBCM_SUCCESS) {
969 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach:"
970 		    "There are active SIDR operations");
971 		hcap->hca_state = IBCM_HCA_ACTIVE;
972 		return (IBCM_FAILURE);
973 	}
974 
975 	if (ibcm_check_avl_clean(hcap) != IBCM_SUCCESS) {
976 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: "
977 		    "There are active RC connections");
978 		hcap->hca_state = IBCM_HCA_ACTIVE;
979 		return (IBCM_FAILURE);
980 	}
981 
982 	/*
983 	 * Now, wait until all rc and sidr stateps go away
984 	 * All these stateps must be short lived ones, waiting to be cleaned
985 	 * up after some timeout value, based on the current state.
986 	 */
987 	IBTF_DPRINTF_L3(cmlog, "ibcm_hca_detach:hca_guid = 0x%llX res_cnt = %d",
988 	    hcap->hca_guid, hcap->hca_res_cnt);
989 
990 	while (hcap->hca_res_cnt > 0)
991 		cv_wait(&ibcm_global_hca_cv, &ibcm_global_hca_lock);
992 
993 	if (hcap->hca_res_cnt != 0) {
994 		/* We got a timeout waiting for hca_res_cnt to become 0 */
995 		IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: Aborting due"
996 		    " to timeout on res_cnt %d, \n Some CM connections are "
997 		    "still in transient state, looks like we need to wait "
998 		    "some more time (ibcm_wait_for_res_cnt_timeout).",
999 		    hcap->hca_res_cnt);
1000 		hcap->hca_state = IBCM_HCA_ACTIVE;
1001 		return (IBCM_FAILURE);
1002 	}
1003 
1004 	/* Re-assert the while loop step above */
1005 	ASSERT(hcap->hca_sidr_list == NULL);
1006 	avl_destroy(&hcap->hca_active_tree);
1007 	avl_destroy(&hcap->hca_passive_tree);
1008 	avl_destroy(&hcap->hca_passive_comid_tree);
1009 
1010 	/*
1011 	 * Unregister all ports from IBMA
1012 	 * If there is a failure, re-initialize any free'd ibma handles. This
1013 	 * is required to receive the incoming mads
1014 	 */
1015 	status = IBCM_SUCCESS;
1016 	for (port_index = 0; port_index < hcap->hca_num_ports; port_index++) {
1017 		if ((status = ibcm_hca_fini_port(hcap, port_index)) !=
1018 		    IBCM_SUCCESS) {
1019 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: "
1020 			    "Failed to free IBMA Handle for port_num %d",
1021 			    port_index + 1);
1022 			break;
1023 		}
1024 	}
1025 
1026 	/* If detach fails, re-initialize ibma handles for incoming mads */
1027 	if (status != IBCM_SUCCESS)  {
1028 		for (i = 0; i < port_index; i++) {
1029 			if (ibcm_hca_init_port(hcap, i) != IBT_SUCCESS)
1030 				IBTF_DPRINTF_L2(cmlog, "ibcm_hca_detach: "
1031 				    "Failed to re-allocate IBMA Handles for"
1032 				    " port_num %d", port_index + 1);
1033 		}
1034 		hcap->hca_state = IBCM_HCA_ACTIVE;
1035 		return (IBCM_FAILURE);
1036 	}
1037 
1038 	ibcm_fini_hca_ids(hcap);
1039 	ibcm_delete_hca_entry(hcap);
1040 
1041 	IBTF_DPRINTF_L3(cmlog, "ibcm_hca_detach: DETACH succeeded");
1042 	return (IBCM_SUCCESS);
1043 }
1044 
1045 /* Checks, if there are any active sidr state entries in the specified hca */
1046 static ibcm_status_t
1047 ibcm_check_sidr_clean(ibcm_hca_info_t *hcap)
1048 {
1049 	ibcm_ud_state_data_t	*usp;
1050 	uint32_t		transient_cnt = 0;
1051 
1052 	IBTF_DPRINTF_L5(cmlog, "ibcm_check_sidr_clean:");
1053 
1054 	rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER);
1055 	usp = hcap->hca_sidr_list;	/* Point to the list */
1056 	while (usp != NULL) {
1057 		mutex_enter(&usp->ud_state_mutex);
1058 		if ((usp->ud_state != IBCM_STATE_SIDR_REP_SENT) &&
1059 		    (usp->ud_state != IBCM_STATE_TIMED_OUT) &&
1060 		    (usp->ud_state != IBCM_STATE_DELETE)) {
1061 
1062 			IBTF_DPRINTF_L3(cmlog, "ibcm_check_sidr_clean:"
1063 			    "usp = %p not in transient state = %d", usp,
1064 			    usp->ud_state);
1065 
1066 			mutex_exit(&usp->ud_state_mutex);
1067 			rw_exit(&hcap->hca_sidr_list_lock);
1068 			return (IBCM_FAILURE);
1069 		} else {
1070 			mutex_exit(&usp->ud_state_mutex);
1071 			++transient_cnt;
1072 		}
1073 
1074 		usp = usp->ud_nextp;
1075 	}
1076 	rw_exit(&hcap->hca_sidr_list_lock);
1077 
1078 	IBTF_DPRINTF_L4(cmlog, "ibcm_check_sidr_clean: transient_cnt %d",
1079 	    transient_cnt);
1080 
1081 	return (IBCM_SUCCESS);
1082 }
1083 
1084 /* Checks, if there are any active rc state entries, in the specified hca */
1085 static ibcm_status_t
1086 ibcm_check_avl_clean(ibcm_hca_info_t *hcap)
1087 
1088 {
1089 	ibcm_state_data_t	*sp;
1090 	avl_tree_t		*avl_tree;
1091 	uint32_t		transient_cnt = 0;
1092 
1093 	IBTF_DPRINTF_L5(cmlog, "ibcm_check_avl_clean:");
1094 	/*
1095 	 * Both the trees ie., active and passive must reference to all
1096 	 * statep's, so let's use one
1097 	 */
1098 	avl_tree = &hcap->hca_active_tree;
1099 
1100 	rw_enter(&hcap->hca_state_rwlock, RW_WRITER);
1101 
1102 	for (sp = avl_first(avl_tree); sp != NULL;
1103 	    sp = avl_walk(avl_tree, sp, AVL_AFTER)) {
1104 		mutex_enter(&sp->state_mutex);
1105 		if ((sp->state != IBCM_STATE_TIMEWAIT) &&
1106 		    (sp->state != IBCM_STATE_REJ_SENT) &&
1107 		    (sp->state != IBCM_STATE_DELETE)) {
1108 			IBTF_DPRINTF_L3(cmlog, "ibcm_check_avl_clean: "
1109 			    "sp = %p not in transient state = %d", sp,
1110 			    sp->state);
1111 			mutex_exit(&sp->state_mutex);
1112 			rw_exit(&hcap->hca_state_rwlock);
1113 			return (IBCM_FAILURE);
1114 		} else {
1115 			mutex_exit(&sp->state_mutex);
1116 			++transient_cnt;
1117 		}
1118 	}
1119 
1120 	rw_exit(&hcap->hca_state_rwlock);
1121 
1122 	IBTF_DPRINTF_L4(cmlog, "ibcm_check_avl_clean: transient_cnt %d",
1123 	    transient_cnt);
1124 
1125 	return (IBCM_SUCCESS);
1126 }
1127 
1128 /* Adds a new entry into CM's global hca list, if hca_guid is not there yet */
1129 static ibcm_hca_info_t *
1130 ibcm_add_hca_entry(ib_guid_t hcaguid, uint_t nports)
1131 {
1132 	ibcm_hca_info_t	*hcap;
1133 
1134 	IBTF_DPRINTF_L5(cmlog, "ibcm_add_hca_entry: guid = 0x%llX",
1135 	    hcaguid);
1136 
1137 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
1138 
1139 	/*
1140 	 * Check if this hca_guid already in the list
1141 	 * If yes, then ignore this and return NULL
1142 	 */
1143 
1144 	hcap = ibcm_hca_listp;
1145 
1146 	/* search for this HCA */
1147 	while (hcap != NULL) {
1148 		if (hcap->hca_guid == hcaguid) {
1149 			/* already exists */
1150 			IBTF_DPRINTF_L2(cmlog, "ibcm_add_hca_entry: "
1151 			    "hcap %p guid 0x%llX, entry already exists !!",
1152 			    hcap, hcap->hca_guid);
1153 			return (NULL);
1154 		}
1155 		hcap = hcap->hca_next;
1156 	}
1157 
1158 	/* Allocate storage for the new HCA entry found */
1159 	hcap = kmem_zalloc(sizeof (ibcm_hca_info_t) +
1160 	    (nports - 1) * sizeof (ibcm_port_info_t), KM_SLEEP);
1161 
1162 	/* initialize RW lock */
1163 	rw_init(&hcap->hca_state_rwlock, NULL, RW_DRIVER, NULL);
1164 	/* initialize SIDR list lock */
1165 	rw_init(&hcap->hca_sidr_list_lock, NULL, RW_DRIVER, NULL);
1166 	/* Insert "hcap" into the global HCA list maintained by CM */
1167 	hcap->hca_next = ibcm_hca_listp;
1168 	ibcm_hca_listp = hcap;
1169 
1170 	IBTF_DPRINTF_L5(cmlog, "ibcm_add_hca_entry: done hcap = 0x%p", hcap);
1171 
1172 	return (hcap);
1173 
1174 }
1175 
1176 /* deletes the given ibcm_hca_info_t from CM's global hca list */
1177 void
1178 ibcm_delete_hca_entry(ibcm_hca_info_t *hcap)
1179 {
1180 	ibcm_hca_info_t	*headp, *prevp = NULL;
1181 
1182 	/* ibcm_hca_global_lock is held */
1183 	IBTF_DPRINTF_L5(cmlog, "ibcm_delete_hca_entry: guid = 0x%llX "
1184 	    "hcap = 0x%p", hcap->hca_guid, hcap);
1185 
1186 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
1187 
1188 	headp = ibcm_hca_listp;
1189 	while (headp != NULL) {
1190 		if (headp == hcap) {
1191 			IBTF_DPRINTF_L3(cmlog, "ibcm_delete_hca_entry: "
1192 			    "deleting hcap %p hcaguid %llX", hcap,
1193 			    hcap->hca_guid);
1194 			if (prevp) {
1195 				prevp->hca_next = headp->hca_next;
1196 			} else {
1197 				prevp = headp->hca_next;
1198 				ibcm_hca_listp = prevp;
1199 			}
1200 			rw_destroy(&hcap->hca_state_rwlock);
1201 			rw_destroy(&hcap->hca_sidr_list_lock);
1202 			kmem_free(hcap, sizeof (ibcm_hca_info_t) +
1203 			    (hcap->hca_num_ports - 1) *
1204 			    sizeof (ibcm_port_info_t));
1205 			return;
1206 		}
1207 
1208 		prevp = headp;
1209 		headp = headp->hca_next;
1210 	}
1211 }
1212 
1213 /*
1214  * ibcm_find_hca_entry:
1215  *	Given a HCA's GUID find out ibcm_hca_info_t entry for that HCA
1216  *	This entry can be then used to access AVL tree/SIDR list etc.
1217  *	If entry exists and in HCA ATTACH state, then hca's ref cnt is
1218  *	incremented and entry returned. Else NULL returned.
1219  *
1220  *	All functions that use ibcm_find_hca_entry and get a non-NULL
1221  *	return values must call ibcm_dec_hca_acc_cnt to decrement the
1222  *	respective hca ref cnt. There shouldn't be any usage of
1223  *	ibcm_hca_info_t * returned from ibcm_find_hca_entry,
1224  *	after decrementing the hca_acc_cnt
1225  *
1226  * INPUTS:
1227  *	hca_guid	- HCA's guid
1228  *
1229  * RETURN VALUE:
1230  *	hcap		- if a match is found, else NULL
1231  */
1232 ibcm_hca_info_t *
1233 ibcm_find_hca_entry(ib_guid_t hca_guid)
1234 {
1235 	ibcm_hca_info_t *hcap;
1236 
1237 	IBTF_DPRINTF_L5(cmlog, "ibcm_find_hca_entry: guid = 0x%llX", hca_guid);
1238 
1239 	mutex_enter(&ibcm_global_hca_lock);
1240 
1241 	hcap = ibcm_hca_listp;
1242 	/* search for this HCA */
1243 	while (hcap != NULL) {
1244 		if (hcap->hca_guid == hca_guid)
1245 			break;
1246 		hcap = hcap->hca_next;
1247 	}
1248 
1249 	/* if no hcap for the hca_guid, return NULL */
1250 	if (hcap == NULL) {
1251 		mutex_exit(&ibcm_global_hca_lock);
1252 		return (NULL);
1253 	}
1254 
1255 	/* return hcap, only if it valid to use */
1256 	if (hcap->hca_state == IBCM_HCA_ACTIVE) {
1257 		++(hcap->hca_acc_cnt);
1258 
1259 		IBTF_DPRINTF_L5(cmlog, "ibcm_find_hca_entry: "
1260 		    "found hcap = 0x%p hca_acc_cnt %u", hcap,
1261 		    hcap->hca_acc_cnt);
1262 
1263 		mutex_exit(&ibcm_global_hca_lock);
1264 		return (hcap);
1265 	} else {
1266 		mutex_exit(&ibcm_global_hca_lock);
1267 
1268 		IBTF_DPRINTF_L2(cmlog, "ibcm_find_hca_entry: "
1269 		    "found hcap = 0x%p not in active state", hcap);
1270 		return (NULL);
1271 	}
1272 }
1273 
1274 /*
1275  * Searches for ibcm_hca_info_t entry based on hca_guid, but doesn't increment
1276  * the hca's reference count. This function is used, where the calling context
1277  * is attempting to delete hcap itself and hence acc_cnt cannot be incremented
1278  * OR assumes that valid hcap must be available in ibcm's global hca list.
1279  */
1280 ibcm_hca_info_t *
1281 ibcm_find_hcap_entry(ib_guid_t hca_guid)
1282 {
1283 	ibcm_hca_info_t *hcap;
1284 
1285 	IBTF_DPRINTF_L5(cmlog, "ibcm_find_hcap_entry: guid = 0x%llX", hca_guid);
1286 
1287 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
1288 
1289 	hcap = ibcm_hca_listp;
1290 	/* search for this HCA */
1291 	while (hcap != NULL) {
1292 		if (hcap->hca_guid == hca_guid)
1293 			break;
1294 		hcap = hcap->hca_next;
1295 	}
1296 
1297 	if (hcap == NULL)
1298 		IBTF_DPRINTF_L2(cmlog, "ibcm_find_hcap_entry: No hcap found for"
1299 		    " hca_guid 0x%llX", hca_guid);
1300 	else
1301 		IBTF_DPRINTF_L5(cmlog, "ibcm_find_hcap_entry: hcap found for"
1302 		    " hca_guid 0x%llX", hca_guid);
1303 
1304 	return (hcap);
1305 }
1306 
1307 /* increment the hca's temporary reference count */
1308 ibcm_status_t
1309 ibcm_inc_hca_acc_cnt(ibcm_hca_info_t *hcap)
1310 {
1311 	mutex_enter(&ibcm_global_hca_lock);
1312 	if (hcap->hca_state == IBCM_HCA_ACTIVE) {
1313 		++(hcap->hca_acc_cnt);
1314 		IBTF_DPRINTF_L5(cmlog, "ibcm_inc_hca_acc_cnt: "
1315 		    "hcap = 0x%p  acc_cnt = %d ", hcap, hcap->hca_acc_cnt);
1316 		mutex_exit(&ibcm_global_hca_lock);
1317 		return (IBCM_SUCCESS);
1318 	} else {
1319 		IBTF_DPRINTF_L2(cmlog, "ibcm_inc_hca_acc_cnt: "
1320 		    "hcap INACTIVE 0x%p  acc_cnt = %d ", hcap,
1321 		    hcap->hca_acc_cnt);
1322 		mutex_exit(&ibcm_global_hca_lock);
1323 		return (IBCM_FAILURE);
1324 	}
1325 }
1326 
1327 /* decrement the hca's ref count, and wake up any waiting threads */
1328 void
1329 ibcm_dec_hca_acc_cnt(ibcm_hca_info_t *hcap)
1330 {
1331 	mutex_enter(&ibcm_global_hca_lock);
1332 	ASSERT(hcap->hca_acc_cnt > 0);
1333 	--(hcap->hca_acc_cnt);
1334 	IBTF_DPRINTF_L5(cmlog, "ibcm_dec_hca_acc_cnt: hcap = 0x%p "
1335 	    "acc_cnt = %d", hcap, hcap->hca_acc_cnt);
1336 	if ((hcap->hca_state == IBCM_HCA_NOT_ACTIVE) &&
1337 	    (hcap->hca_acc_cnt == 0)) {
1338 		IBTF_DPRINTF_L3(cmlog, "ibcm_dec_hca_acc_cnt: "
1339 		    "cv_broadcast for hcap = 0x%p", hcap);
1340 		cv_broadcast(&ibcm_global_hca_cv);
1341 	}
1342 	mutex_exit(&ibcm_global_hca_lock);
1343 }
1344 
1345 /* increment the hca's resource count */
1346 void
1347 ibcm_inc_hca_res_cnt(ibcm_hca_info_t *hcap)
1348 
1349 {
1350 	mutex_enter(&ibcm_global_hca_lock);
1351 	++(hcap->hca_res_cnt);
1352 	IBTF_DPRINTF_L5(cmlog, "ibcm_inc_hca_res_cnt: hcap = 0x%p "
1353 	    "ref_cnt = %d", hcap, hcap->hca_res_cnt);
1354 	mutex_exit(&ibcm_global_hca_lock);
1355 }
1356 
1357 /* decrement the hca's resource count, and wake up any waiting threads */
1358 void
1359 ibcm_dec_hca_res_cnt(ibcm_hca_info_t *hcap)
1360 {
1361 	mutex_enter(&ibcm_global_hca_lock);
1362 	ASSERT(hcap->hca_res_cnt > 0);
1363 	--(hcap->hca_res_cnt);
1364 	IBTF_DPRINTF_L5(cmlog, "ibcm_dec_hca_res_cnt: hcap = 0x%p "
1365 	    "ref_cnt = %d", hcap, hcap->hca_res_cnt);
1366 	if ((hcap->hca_state == IBCM_HCA_NOT_ACTIVE) &&
1367 	    (hcap->hca_res_cnt == 0)) {
1368 		IBTF_DPRINTF_L3(cmlog, "ibcm_dec_hca_res_cnt: "
1369 		    "cv_broadcast for hcap = 0x%p", hcap);
1370 		cv_broadcast(&ibcm_global_hca_cv);
1371 	}
1372 	mutex_exit(&ibcm_global_hca_lock);
1373 }
1374 
1375 /* increment the hca's service count */
1376 void
1377 ibcm_inc_hca_svc_cnt(ibcm_hca_info_t *hcap)
1378 
1379 {
1380 	mutex_enter(&ibcm_global_hca_lock);
1381 	++(hcap->hca_svc_cnt);
1382 	IBTF_DPRINTF_L5(cmlog, "ibcm_inc_hca_svc_cnt: hcap = 0x%p "
1383 	    "svc_cnt = %d", hcap, hcap->hca_svc_cnt);
1384 	mutex_exit(&ibcm_global_hca_lock);
1385 }
1386 
1387 /* decrement the hca's service count */
1388 void
1389 ibcm_dec_hca_svc_cnt(ibcm_hca_info_t *hcap)
1390 {
1391 	mutex_enter(&ibcm_global_hca_lock);
1392 	ASSERT(hcap->hca_svc_cnt > 0);
1393 	--(hcap->hca_svc_cnt);
1394 	IBTF_DPRINTF_L5(cmlog, "ibcm_dec_hca_svc_cnt: hcap = 0x%p "
1395 	    "svc_cnt = %d", hcap, hcap->hca_svc_cnt);
1396 	mutex_exit(&ibcm_global_hca_lock);
1397 }
1398 
1399 /*
1400  * The following code manages three classes of requests that CM makes to
1401  * the fabric.  Those three classes are SA_ACCESS, REQ/REP/RTU, and DREQ/DREP.
1402  * The main issue is that the fabric can become very busy, and the CM
1403  * protocols rely on responses being made based on a predefined timeout
1404  * value.  By managing how many simultaneous sessions are allowed, there
1405  * is observed extremely high reliability of CM protocol succeeding when
1406  * it should.
1407  *
1408  * SA_ACCESS and DREQ/DREP are managed at the thread level, whereby the
1409  * thread blocks until there are less than some number of threads doing
1410  * similar requests.
1411  *
1412  * REQ/REP/RTU requests beyond a given limit are added to a list,
1413  * allowing the thread to return immediately to its caller in the
1414  * case where the "mode" is IBT_NONBLOCKING.  This is the mode used
1415  * by uDAPL and seems to be an important feature/behavior.
1416  */
1417 
1418 static int
1419 ibcm_ok_to_start(struct ibcm_open_s *openp)
1420 {
1421 	return (openp->sends < openp->sends_hiwat &&
1422 	    openp->in_progress < openp->in_progress_max);
1423 }
1424 
1425 void
1426 ibcm_open_done(ibcm_state_data_t *statep)
1427 {
1428 	int run;
1429 	ibcm_state_data_t **linkp, *tmp;
1430 
1431 	ASSERT(MUTEX_HELD(&statep->state_mutex));
1432 	if (statep->open_flow == 1) {
1433 		statep->open_flow = 0;
1434 		mutex_enter(&ibcm_open.mutex);
1435 		if (statep->open_link == NULL) {
1436 			ibcm_open.in_progress--;
1437 			run = ibcm_ok_to_start(&ibcm_open);
1438 		} else {
1439 			ibcm_open.queued--;
1440 			linkp = &ibcm_open.head.open_link;
1441 			while (*linkp != statep)
1442 				linkp = &((*linkp)->open_link);
1443 			*linkp = statep->open_link;
1444 			statep->open_link = NULL;
1445 			/*
1446 			 * If we remove what tail pointed to, we need
1447 			 * to reassign tail (it is never NULL).
1448 			 * tail points to head for the empty list.
1449 			 */
1450 			if (ibcm_open.tail == statep) {
1451 				tmp = &ibcm_open.head;
1452 				while (tmp->open_link != &ibcm_open.head)
1453 					tmp = tmp->open_link;
1454 				ibcm_open.tail = tmp;
1455 			}
1456 			run = 0;
1457 		}
1458 		mutex_exit(&ibcm_open.mutex);
1459 		if (run)
1460 			ibcm_run_tlist_thread();
1461 	}
1462 }
1463 
1464 /* dtrace */
1465 void
1466 ibcm_open_wait(hrtime_t delta)
1467 {
1468 	if (delta > 1000000)
1469 		IBTF_DPRINTF_L2(cmlog, "ibcm_open_wait: flow more %lld", delta);
1470 }
1471 
1472 void
1473 ibcm_open_start(ibcm_state_data_t *statep)
1474 {
1475 	ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_REQ);
1476 
1477 	mutex_enter(&statep->state_mutex);
1478 	ibcm_open_wait(gethrtime() - statep->post_time);
1479 	mutex_exit(&statep->state_mutex);
1480 
1481 	ibcm_post_rc_mad(statep, statep->stored_msg, ibcm_post_req_complete,
1482 	    statep);
1483 
1484 	mutex_enter(&statep->state_mutex);
1485 	IBCM_REF_CNT_DECR(statep);
1486 	mutex_exit(&statep->state_mutex);
1487 }
1488 
1489 void
1490 ibcm_open_enqueue(ibcm_state_data_t *statep)
1491 {
1492 	int run;
1493 
1494 	mutex_enter(&statep->state_mutex);
1495 	statep->post_time = gethrtime();
1496 	mutex_exit(&statep->state_mutex);
1497 	mutex_enter(&ibcm_open.mutex);
1498 	if (ibcm_open.queued == 0 && ibcm_ok_to_start(&ibcm_open)) {
1499 		ibcm_open.in_progress++;
1500 		mutex_exit(&ibcm_open.mutex);
1501 		ibcm_open_start(statep);
1502 	} else {
1503 		ibcm_open.queued++;
1504 		statep->open_link = &ibcm_open.head;
1505 		ibcm_open.tail->open_link = statep;
1506 		ibcm_open.tail = statep;
1507 		run = ibcm_ok_to_start(&ibcm_open);
1508 		mutex_exit(&ibcm_open.mutex);
1509 		if (run)
1510 			ibcm_run_tlist_thread();
1511 	}
1512 }
1513 
1514 ibcm_state_data_t *
1515 ibcm_open_dequeue(void)
1516 {
1517 	ibcm_state_data_t *statep;
1518 
1519 	ASSERT(MUTEX_HELD(&ibcm_open.mutex));
1520 	ibcm_open.queued--;
1521 	ibcm_open.in_progress++;
1522 	statep = ibcm_open.head.open_link;
1523 	ibcm_open.head.open_link = statep->open_link;
1524 	statep->open_link = NULL;
1525 	/*
1526 	 * If we remove what tail pointed to, we need
1527 	 * to reassign tail (it is never NULL).
1528 	 * tail points to head for the empty list.
1529 	 */
1530 	if (ibcm_open.tail == statep)
1531 		ibcm_open.tail = &ibcm_open.head;
1532 	return (statep);
1533 }
1534 
1535 void
1536 ibcm_check_for_opens(void)
1537 {
1538 	ibcm_state_data_t 	*statep;
1539 
1540 	mutex_enter(&ibcm_open.mutex);
1541 
1542 	while (ibcm_open.queued > 0) {
1543 		if (ibcm_ok_to_start(&ibcm_open)) {
1544 			statep = ibcm_open_dequeue();
1545 			mutex_exit(&ibcm_open.mutex);
1546 
1547 			ibcm_open_start(statep);
1548 
1549 			mutex_enter(&ibcm_open.mutex);
1550 		} else {
1551 			break;
1552 		}
1553 	}
1554 	mutex_exit(&ibcm_open.mutex);
1555 }
1556 
1557 
1558 static void
1559 ibcm_flow_init(ibcm_flow_t *flow, uint_t simul_max)
1560 {
1561 	flow->list			= NULL;
1562 	flow->simul			= 0;
1563 	flow->waiters_per_chunk		= 4;
1564 	flow->simul_max			= simul_max;
1565 	flow->lowat			= simul_max - flow->waiters_per_chunk;
1566 	flow->lowat_default		= flow->lowat;
1567 	/* stats */
1568 	flow->total			= 0;
1569 }
1570 
1571 static void
1572 ibcm_rc_flow_control_init(void)
1573 {
1574 	mutex_init(&ibcm_open.mutex, NULL, MUTEX_DEFAULT, NULL);
1575 	mutex_enter(&ibcm_open.mutex);
1576 	ibcm_flow_init(&ibcm_close_flow, ibcm_close_simul_max);
1577 	ibcm_flow_init(&ibcm_lapr_flow, ibcm_lapr_simul_max);
1578 	ibcm_flow_init(&ibcm_saa_flow, ibcm_saa_simul_max);
1579 
1580 	ibcm_open.queued 		= 0;
1581 	ibcm_open.exit_deferred 	= 0;
1582 	ibcm_open.in_progress 		= 0;
1583 	ibcm_open.in_progress_max 	= 16;
1584 	ibcm_open.sends 		= 0;
1585 	ibcm_open.sends_max 		= 0;
1586 	ibcm_open.sends_lowat 		= 8;
1587 	ibcm_open.sends_hiwat 		= 16;
1588 	ibcm_open.tail 			= &ibcm_open.head;
1589 	ibcm_open.head.open_link 	= NULL;
1590 	mutex_exit(&ibcm_open.mutex);
1591 
1592 	mutex_init(&ibcm_close.mutex, NULL, MUTEX_DEFAULT, NULL);
1593 	mutex_enter(&ibcm_close.mutex);
1594 	ibcm_close.tail			= &ibcm_close.head;
1595 	ibcm_close.head.close_link 	= NULL;
1596 	mutex_exit(&ibcm_close.mutex);
1597 }
1598 
1599 static void
1600 ibcm_rc_flow_control_fini(void)
1601 {
1602 	mutex_destroy(&ibcm_open.mutex);
1603 	mutex_destroy(&ibcm_close.mutex);
1604 }
1605 
1606 static ibcm_flow1_t *
1607 ibcm_flow_find(ibcm_flow_t *flow)
1608 {
1609 	ibcm_flow1_t *flow1;
1610 	ibcm_flow1_t *f;
1611 
1612 	f = flow->list;
1613 	if (f) {	/* most likely code path */
1614 		while (f->link != NULL)
1615 			f = f->link;
1616 		if (f->waiters < flow->waiters_per_chunk)
1617 			return (f);
1618 	}
1619 
1620 	/* There was no flow1 list element ready for another waiter */
1621 	mutex_exit(&ibcm_open.mutex);
1622 	flow1 = kmem_alloc(sizeof (*flow1), KM_SLEEP);
1623 	mutex_enter(&ibcm_open.mutex);
1624 
1625 	f = flow->list;
1626 	if (f) {
1627 		while (f->link != NULL)
1628 			f = f->link;
1629 		if (f->waiters < flow->waiters_per_chunk) {
1630 			kmem_free(flow1, sizeof (*flow1));
1631 			return (f);
1632 		}
1633 		f->link = flow1;
1634 	} else {
1635 		flow->list = flow1;
1636 	}
1637 	cv_init(&flow1->cv, NULL, CV_DRIVER, NULL);
1638 	flow1->waiters = 0;
1639 	flow1->link = NULL;
1640 	return (flow1);
1641 }
1642 
1643 static void
1644 ibcm_flow_enter(ibcm_flow_t *flow)
1645 {
1646 	mutex_enter(&ibcm_open.mutex);
1647 	if (flow->list == NULL && flow->simul < flow->simul_max) {
1648 		flow->simul++;
1649 		flow->total++;
1650 		mutex_exit(&ibcm_open.mutex);
1651 	} else {
1652 		ibcm_flow1_t *flow1;
1653 
1654 		flow1 = ibcm_flow_find(flow);
1655 		flow1->waiters++;
1656 		cv_wait(&flow1->cv, &ibcm_open.mutex);
1657 		if (--flow1->waiters == 0) {
1658 			cv_destroy(&flow1->cv);
1659 			mutex_exit(&ibcm_open.mutex);
1660 			kmem_free(flow1, sizeof (*flow1));
1661 		} else
1662 			mutex_exit(&ibcm_open.mutex);
1663 	}
1664 }
1665 
1666 static void
1667 ibcm_flow_exit(ibcm_flow_t *flow)
1668 {
1669 	mutex_enter(&ibcm_open.mutex);
1670 	if (--flow->simul < flow->lowat) {
1671 		if (flow->lowat < flow->lowat_default)
1672 			flow->lowat++;
1673 		if (flow->list) {
1674 			ibcm_flow1_t *flow1;
1675 
1676 			flow1 = flow->list;
1677 			flow->list = flow1->link;	/* unlink */
1678 			flow1->link = NULL;		/* be clean */
1679 			flow->total += flow1->waiters;
1680 			flow->simul += flow1->waiters;
1681 			cv_broadcast(&flow1->cv);
1682 		}
1683 	}
1684 	mutex_exit(&ibcm_open.mutex);
1685 }
1686 
1687 void
1688 ibcm_flow_inc(void)
1689 {
1690 	mutex_enter(&ibcm_open.mutex);
1691 	if (++ibcm_open.sends > ibcm_open.sends_max) {
1692 		ibcm_open.sends_max = ibcm_open.sends;
1693 		IBTF_DPRINTF_L2(cmlog, "ibcm_flow_inc: sends max = %d",
1694 		    ibcm_open.sends_max);
1695 	}
1696 	mutex_exit(&ibcm_open.mutex);
1697 }
1698 
1699 static void
1700 ibcm_check_send_cmpltn_time(hrtime_t delta, char *event_msg)
1701 {
1702 	if (delta > 4000000LL) {
1703 		IBTF_DPRINTF_L2(cmlog, "ibcm_check_send_cmpltn_time: "
1704 		    "%s: %lldns", event_msg, delta);
1705 	}
1706 }
1707 
1708 void
1709 ibcm_flow_dec(hrtime_t time, char *mad_type)
1710 {
1711 	int flow_exit = 0;
1712 	int run = 0;
1713 
1714 	if (ibcm_dtrace)
1715 		ibcm_check_send_cmpltn_time(gethrtime() - time, mad_type);
1716 	mutex_enter(&ibcm_open.mutex);
1717 	ibcm_open.sends--;
1718 	if (ibcm_open.sends < ibcm_open.sends_lowat) {
1719 		run = ibcm_ok_to_start(&ibcm_open);
1720 		if (ibcm_open.exit_deferred) {
1721 			ibcm_open.exit_deferred--;
1722 			flow_exit = 1;
1723 		}
1724 	}
1725 	mutex_exit(&ibcm_open.mutex);
1726 	if (flow_exit)
1727 		ibcm_flow_exit(&ibcm_close_flow);
1728 	if (run)
1729 		ibcm_run_tlist_thread();
1730 }
1731 
1732 void
1733 ibcm_close_enqueue(ibcm_state_data_t *statep)
1734 {
1735 	mutex_enter(&ibcm_close.mutex);
1736 	statep->close_link = NULL;
1737 	ibcm_close.tail->close_link = statep;
1738 	ibcm_close.tail = statep;
1739 	mutex_exit(&ibcm_close.mutex);
1740 	ibcm_run_tlist_thread();
1741 }
1742 
1743 void
1744 ibcm_check_for_async_close()
1745 {
1746 	ibcm_state_data_t 	*statep;
1747 
1748 	mutex_enter(&ibcm_close.mutex);
1749 
1750 	while (ibcm_close.head.close_link) {
1751 		statep = ibcm_close.head.close_link;
1752 		ibcm_close.head.close_link = statep->close_link;
1753 		statep->close_link = NULL;
1754 		if (ibcm_close.tail == statep)
1755 			ibcm_close.tail = &ibcm_close.head;
1756 		mutex_exit(&ibcm_close.mutex);
1757 		ibcm_close_start(statep);
1758 		mutex_enter(&ibcm_close.mutex);
1759 	}
1760 	mutex_exit(&ibcm_close.mutex);
1761 }
1762 
1763 void
1764 ibcm_close_enter(void)
1765 {
1766 	ibcm_flow_enter(&ibcm_close_flow);
1767 }
1768 
1769 void
1770 ibcm_close_exit(void)
1771 {
1772 	int flow_exit;
1773 
1774 	mutex_enter(&ibcm_open.mutex);
1775 	if (ibcm_open.sends < ibcm_open.sends_lowat ||
1776 	    ibcm_open.exit_deferred >= 4)
1777 		flow_exit = 1;
1778 	else {
1779 		flow_exit = 0;
1780 		ibcm_open.exit_deferred++;
1781 	}
1782 	mutex_exit(&ibcm_open.mutex);
1783 	if (flow_exit)
1784 		ibcm_flow_exit(&ibcm_close_flow);
1785 }
1786 
1787 /*
1788  * This function needs to be called twice to finish our flow
1789  * control accounting when closing down a connection.  One
1790  * call has send_done set to 1, while the other has it set to 0.
1791  * Because of retries, this could get called more than once
1792  * with either 0 or 1, but additional calls have no effect.
1793  */
1794 void
1795 ibcm_close_done(ibcm_state_data_t *statep, int send_done)
1796 {
1797 	int flow_exit;
1798 
1799 	ASSERT(MUTEX_HELD(&statep->state_mutex));
1800 	if (statep->close_flow == 1) {
1801 		if (send_done)
1802 			statep->close_flow = 3;
1803 		else
1804 			statep->close_flow = 2;
1805 	} else if ((send_done && statep->close_flow == 2) ||
1806 	    (!send_done && statep->close_flow == 3)) {
1807 		statep->close_flow = 0;
1808 		mutex_enter(&ibcm_open.mutex);
1809 		if (ibcm_open.sends < ibcm_open.sends_lowat ||
1810 		    ibcm_open.exit_deferred >= 4)
1811 			flow_exit = 1;
1812 		else {
1813 			flow_exit = 0;
1814 			ibcm_open.exit_deferred++;
1815 		}
1816 		mutex_exit(&ibcm_open.mutex);
1817 		if (flow_exit)
1818 			ibcm_flow_exit(&ibcm_close_flow);
1819 	}
1820 }
1821 
1822 void
1823 ibcm_lapr_enter(void)
1824 {
1825 	ibcm_flow_enter(&ibcm_lapr_flow);
1826 }
1827 
1828 void
1829 ibcm_lapr_exit(void)
1830 {
1831 	ibcm_flow_exit(&ibcm_lapr_flow);
1832 }
1833 
1834 void
1835 ibcm_sa_access_enter()
1836 {
1837 	ibcm_flow_enter(&ibcm_saa_flow);
1838 }
1839 
1840 void
1841 ibcm_sa_access_exit()
1842 {
1843 	ibcm_flow_exit(&ibcm_saa_flow);
1844 }
1845 
1846 static void
1847 ibcm_sm_notice_handler(ibmf_saa_handle_t saa_handle,
1848     ibmf_saa_subnet_event_t saa_event_code,
1849     ibmf_saa_event_details_t *saa_event_details,
1850     void *callback_arg)
1851 {
1852 	ibcm_port_info_t	*portp = (ibcm_port_info_t *)callback_arg;
1853 	ibt_subnet_event_code_t code;
1854 	ibt_subnet_event_t	event;
1855 	uint8_t			event_status;
1856 
1857 	IBTF_DPRINTF_L3(cmlog, "ibcm_sm_notice_handler: saa_hdl %p, code = %d",
1858 	    saa_handle, saa_event_code);
1859 
1860 	mutex_enter(&ibcm_sm_notice_serialize_lock);
1861 
1862 	switch (saa_event_code) {
1863 	case IBMF_SAA_EVENT_MCG_CREATED:
1864 		code = IBT_SM_EVENT_MCG_CREATED;
1865 		break;
1866 	case IBMF_SAA_EVENT_MCG_DELETED:
1867 		code = IBT_SM_EVENT_MCG_DELETED;
1868 		break;
1869 	case IBMF_SAA_EVENT_GID_AVAILABLE:
1870 		code = IBT_SM_EVENT_GID_AVAIL;
1871 		ibcm_path_cache_purge();
1872 		break;
1873 	case IBMF_SAA_EVENT_GID_UNAVAILABLE:
1874 		code = IBT_SM_EVENT_GID_UNAVAIL;
1875 		ibcm_path_cache_purge();
1876 		break;
1877 	case IBMF_SAA_EVENT_SUBSCRIBER_STATUS_CHG:
1878 		event_status =
1879 		    saa_event_details->ie_producer_event_status_mask &
1880 		    IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM;
1881 		if (event_status == (portp->port_event_status &
1882 		    IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM)) {
1883 			mutex_exit(&ibcm_sm_notice_serialize_lock);
1884 			return;	/* no change */
1885 		}
1886 		portp->port_event_status = event_status;
1887 		if (event_status == IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM)
1888 			code = IBT_SM_EVENT_AVAILABLE;
1889 		else
1890 			code = IBT_SM_EVENT_UNAVAILABLE;
1891 		break;
1892 	default:
1893 		mutex_exit(&ibcm_sm_notice_serialize_lock);
1894 		return;
1895 	}
1896 
1897 	mutex_enter(&ibcm_global_hca_lock);
1898 
1899 	/* don't send the event if we're tearing down */
1900 	if (!IBCM_ACCESS_HCA_OK(portp->port_hcap)) {
1901 		mutex_exit(&ibcm_global_hca_lock);
1902 		mutex_exit(&ibcm_sm_notice_serialize_lock);
1903 		return;
1904 	}
1905 
1906 	++(portp->port_hcap->hca_acc_cnt);
1907 	mutex_exit(&ibcm_global_hca_lock);
1908 
1909 	event.sm_notice_gid = saa_event_details->ie_gid;
1910 	ibtl_cm_sm_notice_handler(portp->port_sgid0, code, &event);
1911 
1912 	mutex_exit(&ibcm_sm_notice_serialize_lock);
1913 
1914 	ibcm_dec_hca_acc_cnt(portp->port_hcap);
1915 }
1916 
1917 void
1918 ibt_register_subnet_notices(ibt_clnt_hdl_t ibt_hdl,
1919     ibt_sm_notice_handler_t sm_notice_handler, void *private)
1920 {
1921 	ibcm_port_info_t	*portp;
1922 	ibcm_hca_info_t		*hcap;
1923 	uint8_t			port;
1924 	int			num_failed_sgids;
1925 	ibtl_cm_sm_init_fail_t	*ifail;
1926 	ib_gid_t		*sgidp;
1927 
1928 	IBTF_DPRINTF_L3(cmlog, "ibt_register_subnet_notices: ibt_hdl = %p",
1929 	    ibt_hdl);
1930 
1931 	mutex_enter(&ibcm_sm_notice_serialize_lock);
1932 
1933 	ibtl_cm_set_sm_notice_handler(ibt_hdl, sm_notice_handler, private);
1934 	if (sm_notice_handler == NULL) {
1935 		mutex_exit(&ibcm_sm_notice_serialize_lock);
1936 		return;
1937 	}
1938 
1939 	/* for each port, if service is not available, make a call */
1940 	mutex_enter(&ibcm_global_hca_lock);
1941 	num_failed_sgids = 0;
1942 	hcap = ibcm_hca_listp;
1943 	while (hcap != NULL) {
1944 		portp = hcap->hca_port_info;
1945 		for (port = 0; port < hcap->hca_num_ports; port++) {
1946 			if (!(portp->port_event_status &
1947 			    IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM))
1948 				num_failed_sgids++;
1949 			portp++;
1950 		}
1951 		hcap = hcap->hca_next;
1952 	}
1953 	if (num_failed_sgids != 0) {
1954 		ifail = kmem_alloc(sizeof (*ifail) +
1955 		    (num_failed_sgids - 1) * sizeof (ib_gid_t), KM_SLEEP);
1956 		ifail->smf_num_sgids = num_failed_sgids;
1957 		ifail->smf_ibt_hdl = ibt_hdl;
1958 		sgidp = &ifail->smf_sgid[0];
1959 		hcap = ibcm_hca_listp;
1960 		while (hcap != NULL) {
1961 			portp = hcap->hca_port_info;
1962 			for (port = 0; port < hcap->hca_num_ports; port++) {
1963 				if (!(portp->port_event_status &
1964 				    IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM))
1965 					*sgidp++ = portp->port_sgid0;
1966 				portp++;
1967 			}
1968 			hcap = hcap->hca_next;
1969 		}
1970 	}
1971 	mutex_exit(&ibcm_global_hca_lock);
1972 
1973 	if (num_failed_sgids != 0) {
1974 		ibtl_cm_sm_notice_init_failure(ifail);
1975 		kmem_free(ifail, sizeof (*ifail) +
1976 		    (num_failed_sgids - 1) * sizeof (ib_gid_t));
1977 	}
1978 	mutex_exit(&ibcm_sm_notice_serialize_lock);
1979 }
1980 
1981 /* The following is run from a taskq because we've seen the stack overflow. */
1982 static void
1983 ibcm_init_saa(void *arg)
1984 {
1985 	ibcm_port_info_t		*portp = (ibcm_port_info_t *)arg;
1986 	int				status;
1987 	ib_guid_t			port_guid;
1988 	ibmf_saa_subnet_event_args_t	event_args;
1989 
1990 	port_guid = portp->port_sgid0.gid_guid;
1991 
1992 	IBTF_DPRINTF_L3(cmlog, "ibcm_init_saa: port guid %llX", port_guid);
1993 
1994 	event_args.is_event_callback_arg = portp;
1995 	event_args.is_event_callback = ibcm_sm_notice_handler;
1996 
1997 	if ((status = ibmf_sa_session_open(port_guid, 0, &event_args,
1998 	    IBMF_VERSION, 0, &portp->port_ibmf_saa_hdl)) != IBMF_SUCCESS) {
1999 		IBTF_DPRINTF_L2(cmlog, "ibcm_init_saa: "
2000 		    "ibmf_sa_session_open failed for port guid %llX "
2001 		    "status = %d", port_guid, status);
2002 	} else {
2003 		IBTF_DPRINTF_L2(cmlog, "ibcm_init_saa: "
2004 		    "registered sa_hdl 0x%p for port guid %llX",
2005 		    portp->port_ibmf_saa_hdl, port_guid);
2006 	}
2007 
2008 	mutex_enter(&ibcm_sa_open_lock);
2009 	portp->port_saa_open_in_progress = 0;
2010 	cv_broadcast(&ibcm_sa_open_cv);
2011 	mutex_exit(&ibcm_sa_open_lock);
2012 }
2013 
2014 void
2015 ibcm_init_saa_handle(ibcm_hca_info_t *hcap, uint8_t port)
2016 {
2017 	ibmf_saa_handle_t	saa_handle;
2018 	uint8_t			port_index = port - 1;
2019 	ibcm_port_info_t	*portp = &hcap->hca_port_info[port_index];
2020 	ibt_status_t		ibt_status;
2021 
2022 	if (port_index >= hcap->hca_num_ports)
2023 		return;
2024 
2025 	mutex_enter(&ibcm_sa_open_lock);
2026 	if (portp->port_saa_open_in_progress) {
2027 		mutex_exit(&ibcm_sa_open_lock);
2028 		return;
2029 	}
2030 
2031 	saa_handle = portp->port_ibmf_saa_hdl;
2032 	if (saa_handle != NULL) {
2033 		mutex_exit(&ibcm_sa_open_lock);
2034 		return;
2035 	}
2036 
2037 	portp->port_saa_open_in_progress = 1;
2038 	mutex_exit(&ibcm_sa_open_lock);
2039 
2040 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(portp->port_event_status))
2041 
2042 	/* The assumption is that we're getting event notifications */
2043 	portp->port_event_status = IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM;
2044 
2045 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(portp->port_event_status))
2046 
2047 	ibt_status = ibt_get_port_state_byguid(portp->port_hcap->hca_guid,
2048 	    portp->port_num, &portp->port_sgid0, NULL);
2049 	if (ibt_status != IBT_SUCCESS) {
2050 		IBTF_DPRINTF_L2(cmlog, "ibcm_init_saa_handle: "
2051 		    "ibt_get_port_state_byguid failed for guid %llX "
2052 		    "with status %d", portp->port_hcap->hca_guid, ibt_status);
2053 		mutex_enter(&ibcm_sa_open_lock);
2054 		portp->port_saa_open_in_progress = 0;
2055 		cv_broadcast(&ibcm_sa_open_cv);
2056 		mutex_exit(&ibcm_sa_open_lock);
2057 		return;
2058 	}
2059 	/* if the port is UP, try sa_session_open */
2060 	(void) taskq_dispatch(ibcm_taskq, ibcm_init_saa, portp, TQ_SLEEP);
2061 }
2062 
2063 
2064 ibmf_saa_handle_t
2065 ibcm_get_saa_handle(ibcm_hca_info_t *hcap, uint8_t port)
2066 {
2067 	ibmf_saa_handle_t	saa_handle;
2068 	uint8_t			port_index = port - 1;
2069 	ibcm_port_info_t	*portp = &hcap->hca_port_info[port_index];
2070 	ibt_status_t		ibt_status;
2071 
2072 	if (port_index >= hcap->hca_num_ports)
2073 		return (NULL);
2074 
2075 	mutex_enter(&ibcm_sa_open_lock);
2076 	while (portp->port_saa_open_in_progress) {
2077 		cv_wait(&ibcm_sa_open_cv, &ibcm_sa_open_lock);
2078 	}
2079 
2080 	saa_handle = portp->port_ibmf_saa_hdl;
2081 	if (saa_handle != NULL) {
2082 		mutex_exit(&ibcm_sa_open_lock);
2083 		return (saa_handle);
2084 	}
2085 
2086 	portp->port_saa_open_in_progress = 1;
2087 	mutex_exit(&ibcm_sa_open_lock);
2088 
2089 	ibt_status = ibt_get_port_state_byguid(portp->port_hcap->hca_guid,
2090 	    portp->port_num, &portp->port_sgid0, NULL);
2091 	if (ibt_status != IBT_SUCCESS) {
2092 		IBTF_DPRINTF_L2(cmlog, "ibcm_get_saa_handle: "
2093 		    "ibt_get_port_state_byguid failed for guid %llX "
2094 		    "with status %d", portp->port_hcap->hca_guid, ibt_status);
2095 		mutex_enter(&ibcm_sa_open_lock);
2096 		portp->port_saa_open_in_progress = 0;
2097 		cv_broadcast(&ibcm_sa_open_cv);
2098 		mutex_exit(&ibcm_sa_open_lock);
2099 		return (NULL);
2100 	}
2101 	/* if the port is UP, try sa_session_open */
2102 	(void) taskq_dispatch(ibcm_taskq, ibcm_init_saa, portp, TQ_SLEEP);
2103 
2104 	mutex_enter(&ibcm_sa_open_lock);
2105 	while (portp->port_saa_open_in_progress) {
2106 		cv_wait(&ibcm_sa_open_cv, &ibcm_sa_open_lock);
2107 	}
2108 	saa_handle = portp->port_ibmf_saa_hdl;
2109 	mutex_exit(&ibcm_sa_open_lock);
2110 	return (saa_handle);
2111 }
2112 
2113 
2114 /*
2115  * ibcm_hca_init_port():
2116  * 	- Register port with IBMA
2117  *
2118  * Arguments:
2119  *	hcap		- HCA's guid
2120  *	port_index	- port number minus 1
2121  *
2122  * Return values:
2123  *	IBCM_SUCCESS - success
2124  */
2125 ibt_status_t
2126 ibcm_hca_init_port(ibcm_hca_info_t *hcap, uint8_t port_index)
2127 {
2128 	int			status;
2129 	ibmf_register_info_t	*ibmf_reg;
2130 
2131 	IBTF_DPRINTF_L4(cmlog, "ibcm_hca_init_port: hcap = 0x%p port_num %d",
2132 	    hcap, port_index + 1);
2133 
2134 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
2135 
2136 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(hcap->hca_port_info))
2137 
2138 	if (hcap->hca_port_info[port_index].port_ibmf_hdl == NULL) {
2139 		/* Register with IBMF */
2140 		ibmf_reg = &hcap->hca_port_info[port_index].port_ibmf_reg;
2141 		ibmf_reg->ir_ci_guid = hcap->hca_guid;
2142 		ibmf_reg->ir_port_num = port_index + 1;
2143 		ibmf_reg->ir_client_class = COMM_MGT_MANAGER_AGENT;
2144 
2145 		/*
2146 		 * register with management framework
2147 		 */
2148 		status = ibmf_register(ibmf_reg, IBMF_VERSION,
2149 		    IBMF_REG_FLAG_NO_OFFLOAD, NULL, NULL,
2150 		    &(hcap->hca_port_info[port_index].port_ibmf_hdl),
2151 		    &(hcap->hca_port_info[port_index].port_ibmf_caps));
2152 
2153 		if (status != IBMF_SUCCESS) {
2154 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_init_port: "
2155 			    "ibmf_register failed for port_num %x, "
2156 			    "status = %x", port_index + 1, status);
2157 			return (ibcm_ibmf_analyze_error(status));
2158 		}
2159 
2160 		hcap->hca_port_info[port_index].port_qp1.qp_cm =
2161 		    IBMF_QP_HANDLE_DEFAULT;
2162 		hcap->hca_port_info[port_index].port_qp1.qp_port =
2163 		    &(hcap->hca_port_info[port_index]);
2164 
2165 		/*
2166 		 * Register the read callback with IBMF.
2167 		 * Since we just did an ibmf_register, handle is
2168 		 * valid and ibcm_recv_cb() is valid so we can
2169 		 * safely assert for success of ibmf_setup_recv_cb()
2170 		 *
2171 		 * Depending on the "state" of the HCA,
2172 		 * CM may drop incoming packets
2173 		 */
2174 		status = ibmf_setup_async_cb(
2175 		    hcap->hca_port_info[port_index].port_ibmf_hdl,
2176 		    IBMF_QP_HANDLE_DEFAULT, ibcm_recv_cb,
2177 		    &(hcap->hca_port_info[port_index].port_qp1), 0);
2178 		ASSERT(status == IBMF_SUCCESS);
2179 
2180 		IBTF_DPRINTF_L5(cmlog, "ibcm_hca_init_port: "
2181 		    "IBMF hdl[%x] = 0x%p", port_index,
2182 		    hcap->hca_port_info[port_index].port_ibmf_hdl);
2183 
2184 		/* Attempt to get the saa_handle for this port */
2185 		ibcm_init_saa_handle(hcap, port_index + 1);
2186 	}
2187 
2188 	return (IBT_SUCCESS);
2189 }
2190 
2191 /*
2192  * useful, to re attempt to initialize port ibma handles from elsewhere in
2193  * cm code
2194  */
2195 ibt_status_t
2196 ibcm_hca_reinit_port(ibcm_hca_info_t *hcap, uint8_t port_index)
2197 {
2198 	ibt_status_t	status;
2199 
2200 	IBTF_DPRINTF_L5(cmlog, "ibcm_hca_reinit_port: hcap 0x%p port_num %d",
2201 	    hcap, port_index + 1);
2202 
2203 	mutex_enter(&ibcm_global_hca_lock);
2204 	status = ibcm_hca_init_port(hcap, port_index);
2205 	mutex_exit(&ibcm_global_hca_lock);
2206 	return (status);
2207 }
2208 
2209 
2210 /*
2211  * ibcm_hca_fini_port():
2212  * 	- Deregister port with IBMA
2213  *
2214  * Arguments:
2215  *	hcap		- HCA's guid
2216  *	port_index	- port number minus 1
2217  *
2218  * Return values:
2219  *	IBCM_SUCCESS - success
2220  */
2221 static ibcm_status_t
2222 ibcm_hca_fini_port(ibcm_hca_info_t *hcap, uint8_t port_index)
2223 {
2224 	int			ibmf_status;
2225 	ibcm_status_t		ibcm_status;
2226 
2227 	IBTF_DPRINTF_L4(cmlog, "ibcm_hca_fini_port: hcap = 0x%p port_num %d ",
2228 	    hcap, port_index + 1);
2229 
2230 	ASSERT(MUTEX_HELD(&ibcm_global_hca_lock));
2231 
2232 	if (hcap->hca_port_info[port_index].port_ibmf_saa_hdl != NULL) {
2233 		IBTF_DPRINTF_L5(cmlog, "ibcm_hca_fini_port: "
2234 		    "ibmf_sa_session_close IBMF SAA hdl %p",
2235 		    hcap->hca_port_info[port_index].port_ibmf_saa_hdl);
2236 
2237 		ibmf_status = ibmf_sa_session_close(
2238 		    &hcap->hca_port_info[port_index].port_ibmf_saa_hdl, 0);
2239 		if (ibmf_status != IBMF_SUCCESS) {
2240 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_fini_port: "
2241 			    "ibmf_sa_session_close of port %d returned %x",
2242 			    port_index + 1, ibmf_status);
2243 			return (IBCM_FAILURE);
2244 		}
2245 	}
2246 
2247 	if (hcap->hca_port_info[port_index].port_ibmf_hdl != NULL) {
2248 		IBTF_DPRINTF_L5(cmlog, "ibcm_hca_fini_port: "
2249 		    "ibmf_unregister IBMF Hdl %p",
2250 		    hcap->hca_port_info[port_index].port_ibmf_hdl);
2251 
2252 		/* clean-up all the ibmf qp's allocated on this port */
2253 		ibcm_status = ibcm_free_allqps(hcap, port_index + 1);
2254 
2255 		if (ibcm_status != IBCM_SUCCESS) {
2256 
2257 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_fini_port: "
2258 			    "ibcm_free_allqps failed for port_num %d",
2259 			    port_index + 1);
2260 			return (IBCM_FAILURE);
2261 		}
2262 
2263 		/* Tear down the receive callback */
2264 		ibmf_status = ibmf_tear_down_async_cb(
2265 		    hcap->hca_port_info[port_index].port_ibmf_hdl,
2266 		    IBMF_QP_HANDLE_DEFAULT, 0);
2267 
2268 		if (ibmf_status != IBMF_SUCCESS) {
2269 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_fini_port: "
2270 			    "ibmf_tear_down_async_cb failed %d port_num %d",
2271 			    ibmf_status, port_index + 1);
2272 			return (IBCM_FAILURE);
2273 		}
2274 
2275 		/* Now, unregister with IBMF */
2276 		ibmf_status = ibmf_unregister(
2277 		    &hcap->hca_port_info[port_index].port_ibmf_hdl, 0);
2278 		IBTF_DPRINTF_L4(cmlog, "ibcm_hca_fini_port: "
2279 		    "ibmf_unregister of port_num %x returned %x",
2280 		    port_index + 1, ibmf_status);
2281 
2282 		if (ibmf_status == IBMF_SUCCESS)
2283 			hcap->hca_port_info[port_index].port_ibmf_hdl = NULL;
2284 		else {
2285 			IBTF_DPRINTF_L2(cmlog, "ibcm_hca_fini_port: "
2286 			    "ibmf_unregister failed %d port_num %d",
2287 			    ibmf_status, port_index + 1);
2288 			return (IBCM_FAILURE);
2289 		}
2290 	}
2291 	return (IBCM_SUCCESS);
2292 }
2293 
2294 /*
2295  * ibcm_comm_est_handler():
2296  *	Check if the given channel is in ESTABLISHED state or not
2297  *
2298  * Arguments:
2299  *	eventp	- A pointer to an ibt_async_event_t struct
2300  *
2301  * Return values: NONE
2302  */
2303 static void
2304 ibcm_comm_est_handler(ibt_async_event_t *eventp)
2305 {
2306 	ibcm_state_data_t	*statep;
2307 
2308 	IBTF_DPRINTF_L4(cmlog, "ibcm_comm_est_handler:");
2309 
2310 	/* Both QP and EEC handles can't be NULL */
2311 	if (eventp->ev_chan_hdl == NULL) {
2312 		IBTF_DPRINTF_L2(cmlog, "ibcm_comm_est_handler: "
2313 		    "both QP and EEC handles are NULL");
2314 		return;
2315 	}
2316 
2317 	/* get the "statep" from qp/eec handles */
2318 	IBCM_GET_CHAN_PRIVATE(eventp->ev_chan_hdl, statep);
2319 	if (statep == NULL) {
2320 		IBTF_DPRINTF_L2(cmlog, "ibcm_comm_est_handler: statep is NULL");
2321 		return;
2322 	}
2323 
2324 	mutex_enter(&statep->state_mutex);
2325 
2326 	IBCM_RELEASE_CHAN_PRIVATE(eventp->ev_chan_hdl);
2327 
2328 	IBTF_DPRINTF_L4(cmlog, "ibcm_comm_est_handler: statep = %p", statep);
2329 
2330 	IBCM_REF_CNT_INCR(statep);
2331 
2332 	if ((statep->state == IBCM_STATE_REP_SENT) ||
2333 	    (statep->state == IBCM_STATE_MRA_REP_RCVD)) {
2334 		timeout_id_t	timer_val = statep->timerid;
2335 
2336 		statep->state = IBCM_STATE_TRANSIENT_ESTABLISHED;
2337 
2338 		if (timer_val) {
2339 			statep->timerid = 0;
2340 			mutex_exit(&statep->state_mutex);
2341 			(void) untimeout(timer_val);
2342 		} else
2343 			mutex_exit(&statep->state_mutex);
2344 
2345 		/* CM doesn't have RTU message here */
2346 		ibcm_cep_state_rtu(statep, NULL);
2347 
2348 	} else {
2349 		if (statep->state == IBCM_STATE_ESTABLISHED ||
2350 		    statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) {
2351 			IBTF_DPRINTF_L4(cmlog, "ibcm_comm_est_handler: "
2352 			    "Channel already in ESTABLISHED state");
2353 		} else {
2354 			/* An unexpected behavior from remote */
2355 			IBTF_DPRINTF_L2(cmlog, "ibcm_comm_est_handler: "
2356 			    "Unexpected in state = %d", statep->state);
2357 		}
2358 		mutex_exit(&statep->state_mutex);
2359 
2360 		ibcm_insert_trace(statep, IBCM_TRACE_INCOMING_COMEST);
2361 	}
2362 
2363 	mutex_enter(&statep->state_mutex);
2364 	IBCM_REF_CNT_DECR(statep);
2365 	mutex_exit(&statep->state_mutex);
2366 }
2367 
2368 
2369 /*
2370  * ibcm_async_handler():
2371  *	CM's Async Handler
2372  *	(Handles ATTACH, DETACH, COM_EST events)
2373  *
2374  * Arguments:
2375  *	eventp	- A pointer to an ibt_async_event_t struct
2376  *
2377  * Return values: None
2378  *
2379  * NOTE : CM assumes that all HCA DR events are delivered sequentially
2380  * i.e., until ibcm_async_handler  completes for a given HCA DR, framework
2381  * shall not invoke ibcm_async_handler with another DR event for the same
2382  * HCA
2383  */
2384 /* ARGSUSED */
2385 void
2386 ibcm_async_handler(void *clnt_hdl, ibt_hca_hdl_t hca_hdl,
2387     ibt_async_code_t code, ibt_async_event_t *eventp)
2388 {
2389 	ibcm_hca_info_t		*hcap;
2390 	ibcm_port_up_t		*pup;
2391 
2392 	IBTF_DPRINTF_L3(cmlog, "ibcm_async_handler: "
2393 	    "clnt_hdl = %p, code = 0x%x, eventp = 0x%p",
2394 	    clnt_hdl, code, eventp);
2395 
2396 	mutex_enter(&ibcm_global_hca_lock);
2397 
2398 	/* If fini is going to complete successfully, then return */
2399 	if (ibcm_finit_state != IBCM_FINIT_IDLE) {
2400 
2401 		/*
2402 		 * This finit state implies one of the following:
2403 		 * Init either didn't start or didn't complete OR
2404 		 * Fini is about to return SUCCESS and release the global lock.
2405 		 * In all these cases, it is safe to ignore the async.
2406 		 */
2407 
2408 		IBTF_DPRINTF_L2(cmlog, "ibcm_async_handler: ignoring event %x, "
2409 		    "as either init didn't complete or fini about to succeed",
2410 		    code);
2411 		mutex_exit(&ibcm_global_hca_lock);
2412 		return;
2413 	}
2414 
2415 	switch (code) {
2416 	case IBT_EVENT_PORT_UP:
2417 		mutex_exit(&ibcm_global_hca_lock);
2418 		pup = kmem_alloc(sizeof (ibcm_port_up_t), KM_SLEEP);
2419 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pup))
2420 		pup->pup_hca_guid = eventp->ev_hca_guid;
2421 		pup->pup_port = eventp->ev_port;
2422 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pup))
2423 		(void) taskq_dispatch(ibcm_taskq,
2424 		    ibcm_service_record_rewrite_task, pup, TQ_SLEEP);
2425 		ibcm_path_cache_purge();
2426 		return;
2427 
2428 	case IBT_HCA_ATTACH_EVENT:
2429 
2430 		/* eventp->ev_hcaguid is the HCA GUID of interest */
2431 		ibcm_hca_attach(eventp->ev_hca_guid);
2432 		break;
2433 
2434 	case IBT_HCA_DETACH_EVENT:
2435 
2436 		/* eventp->ev_hca_guid is the HCA GUID of interest */
2437 		if ((hcap = ibcm_find_hcap_entry(eventp->ev_hca_guid)) ==
2438 		    NULL) {
2439 			IBTF_DPRINTF_L2(cmlog, "ibcm_async_handler:"
2440 			    " hca %llX doesn't exist", eventp->ev_hca_guid);
2441 			break;
2442 		}
2443 
2444 		(void) ibcm_hca_detach(hcap);
2445 		break;
2446 
2447 	case IBT_EVENT_COM_EST_QP:
2448 		/* eventp->ev_qp_hdl is the ibt_qp_hdl_t of interest */
2449 	case IBT_EVENT_COM_EST_EEC:
2450 		/* eventp->ev_eec_hdl is the ibt_eec_hdl_t of interest */
2451 		ibcm_comm_est_handler(eventp);
2452 		break;
2453 	default:
2454 		break;
2455 	}
2456 
2457 	/* Unblock, any blocked fini/init operations */
2458 	mutex_exit(&ibcm_global_hca_lock);
2459 }
2460