1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
83 #define ISCSI_BHS_OPCODE_INTERNAL	0x3e
88 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
95     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
101     &maxtags, 0, "Max number of requests queued by initiator");
109 	} while (0)
113 		if (debug > 0) {					\
117 	} while (0)
123 			    __func__, S->cs_initiator_addr,		\
124 			    S->cs_initiator_name, ## __VA_ARGS__);	\
126 	} while (0)
130 		if (debug > 0) {					\
132 			    S->cs_initiator_addr,			\
133 			    S->cs_initiator_name, ## __VA_ARGS__);	\
135 	} while (0)
137 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
138 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
139 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
141 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
142 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
150     ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
151 #define	PRIV_REQUEST(io)		PRIV(io)->request
152 #define	PRIV_EXPDATASN(io)		PRIV(io)->expdatasn
153 #define	PRIV_R2TSN(io)			PRIV(io)->r2tsn
172 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
174 		    struct cfiscsi_session *cs, union ctl_io *io,
177 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
185 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
203 	return (icl_pdu_new(request->ip_conn, flags));  in cfiscsi_pdu_new_response()
210 	struct cfiscsi_session *cs;  in cfiscsi_pdu_update_cmdsn()  local
213 	cs = PDU_SESSION(request);  in cfiscsi_pdu_update_cmdsn()
216 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.  in cfiscsi_pdu_update_cmdsn()
218 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed  in cfiscsi_pdu_update_cmdsn()
221 	cs->cs_timeout = 0;  in cfiscsi_pdu_update_cmdsn()
227 	if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)  in cfiscsi_pdu_update_cmdsn()
231 	 * Data-Out PDUs don't contain CmdSN.  in cfiscsi_pdu_update_cmdsn()
233 	if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)  in cfiscsi_pdu_update_cmdsn()
238 	 * (initiator -> target) PDUs.  in cfiscsi_pdu_update_cmdsn()
240 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;  in cfiscsi_pdu_update_cmdsn()
241 	curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);  in cfiscsi_pdu_update_cmdsn()
247 		if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))  in cfiscsi_pdu_update_cmdsn()
252 	 * The target MUST silently ignore any non-immediate command outside  in cfiscsi_pdu_update_cmdsn()
256 	    ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {  in cfiscsi_pdu_update_cmdsn()
257 		CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "  in cfiscsi_pdu_update_cmdsn()
264 	 * CmdSN means lost PDUs.  Since we don't support PDU retransmission --  in cfiscsi_pdu_update_cmdsn()
267 	CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "  in cfiscsi_pdu_update_cmdsn()
270 	cfiscsi_session_terminate(cs);  in cfiscsi_pdu_update_cmdsn()
277 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle()  local
280 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle()
294 	switch (request->ip_bhs->bhs_opcode &  in cfiscsi_pdu_handle()
312 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "  in cfiscsi_pdu_handle()
313 		    "opcode 0x%x; dropping connection",  in cfiscsi_pdu_handle()
314 		    request->ip_bhs->bhs_opcode);  in cfiscsi_pdu_handle()
316 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle()
325 	struct cfiscsi_session *cs;  in cfiscsi_receive_callback()  local
327 	cs = PDU_SESSION(request);  in cfiscsi_receive_callback()
328 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {  in cfiscsi_receive_callback()
329 		if (cs->cs_login_pdu == NULL)  in cfiscsi_receive_callback()
330 			cs->cs_login_pdu = request;  in cfiscsi_receive_callback()
333 		cv_signal(&cs->cs_login_cv);  in cfiscsi_receive_callback()
344 	struct cfiscsi_session *cs;  in cfiscsi_error_callback()  local
346 	cs = CONN_SESSION(ic);  in cfiscsi_error_callback()
348 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");  in cfiscsi_error_callback()
349 	cfiscsi_session_terminate(cs);  in cfiscsi_error_callback()
355 	struct cfiscsi_session *cs;  in cfiscsi_pdu_prepare()  local
360 	cs = PDU_SESSION(response);  in cfiscsi_pdu_prepare()
362 	CFISCSI_SESSION_LOCK_ASSERT(cs);  in cfiscsi_pdu_prepare()
366 	 * (target -> initiator) PDUs.  in cfiscsi_pdu_prepare()
368 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;  in cfiscsi_pdu_prepare()
374 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)  in cfiscsi_pdu_prepare()
378 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,  in cfiscsi_pdu_prepare()
381 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&   in cfiscsi_pdu_prepare()
382 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)  in cfiscsi_pdu_prepare()
386 	 * See the comment below - StatSN is not meaningful and must  in cfiscsi_pdu_prepare()
389 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&  in cfiscsi_pdu_prepare()
390 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)  in cfiscsi_pdu_prepare()
397 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||  in cfiscsi_pdu_prepare()
398 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))  in cfiscsi_pdu_prepare()
399 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);  in cfiscsi_pdu_prepare()
400 	cmdsn = cs->cs_cmdsn;  in cfiscsi_pdu_prepare()
401 	bhssr->bhssr_expcmdsn = htonl(cmdsn);  in cfiscsi_pdu_prepare()
402 	bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +  in cfiscsi_pdu_prepare()
403 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));  in cfiscsi_pdu_prepare()
406 		cs->cs_statsn++;  in cfiscsi_pdu_prepare()
408 	return (0);  in cfiscsi_pdu_prepare()
414 	struct cfiscsi_session *cs;  in cfiscsi_pdu_queue()  local
416 	cs = PDU_SESSION(response);  in cfiscsi_pdu_queue()
418 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_pdu_queue()
421 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_pdu_queue()
427 	struct cfiscsi_session *cs = PDU_SESSION(response);  in cfiscsi_pdu_queue_cb()  local
429 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_pdu_queue_cb()
432 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_pdu_queue_cb()
438 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle_nop_out()  local
446 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle_nop_out()
447 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;  in cfiscsi_pdu_handle_nop_out()
449 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {  in cfiscsi_pdu_handle_nop_out()
459 	if (datasize > 0) {  in cfiscsi_pdu_handle_nop_out()
462 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "  in cfiscsi_pdu_handle_nop_out()
465 			cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_nop_out()
468 		icl_pdu_get_data(request, 0, data, datasize);  in cfiscsi_pdu_handle_nop_out()
473 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "  in cfiscsi_pdu_handle_nop_out()
477 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_nop_out()
480 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;  in cfiscsi_pdu_handle_nop_out()
481 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;  in cfiscsi_pdu_handle_nop_out()
482 	bhsni->bhsni_flags = 0x80;  in cfiscsi_pdu_handle_nop_out()
483 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;  in cfiscsi_pdu_handle_nop_out()
484 	bhsni->bhsni_target_transfer_tag = 0xffffffff;  in cfiscsi_pdu_handle_nop_out()
485 	if (datasize > 0) {  in cfiscsi_pdu_handle_nop_out()
487 		if (error != 0) {  in cfiscsi_pdu_handle_nop_out()
488 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "  in cfiscsi_pdu_handle_nop_out()
493 			cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_nop_out()
507 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle_scsi_command()  local
511 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle_scsi_command()
512 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;  in cfiscsi_pdu_handle_scsi_command()
513 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",  in cfiscsi_pdu_handle_scsi_command()
514 	//    bhssc->bhssc_initiator_task_tag);  in cfiscsi_pdu_handle_scsi_command()
516 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {  in cfiscsi_pdu_handle_scsi_command()
517 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "  in cfiscsi_pdu_handle_scsi_command()
520 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_scsi_command()
523 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);  in cfiscsi_pdu_handle_scsi_command()
526 	io->io_hdr.io_type = CTL_IO_SCSI;  in cfiscsi_pdu_handle_scsi_command()
527 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;  in cfiscsi_pdu_handle_scsi_command()
528 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;  in cfiscsi_pdu_handle_scsi_command()
529 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));  in cfiscsi_pdu_handle_scsi_command()
530 	io->scsiio.priority = (bhssc->bhssc_pri & BHSSC_PRI_MASK) >>  in cfiscsi_pdu_handle_scsi_command()
532 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;  in cfiscsi_pdu_handle_scsi_command()
533 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {  in cfiscsi_pdu_handle_scsi_command()
535 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;  in cfiscsi_pdu_handle_scsi_command()
538 		io->scsiio.tag_type = CTL_TAG_SIMPLE;  in cfiscsi_pdu_handle_scsi_command()
541         	io->scsiio.tag_type = CTL_TAG_ORDERED;  in cfiscsi_pdu_handle_scsi_command()
544         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;  in cfiscsi_pdu_handle_scsi_command()
547 		io->scsiio.tag_type = CTL_TAG_ACA;  in cfiscsi_pdu_handle_scsi_command()
550 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;  in cfiscsi_pdu_handle_scsi_command()
551 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",  in cfiscsi_pdu_handle_scsi_command()
552 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);  in cfiscsi_pdu_handle_scsi_command()
555 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */  in cfiscsi_pdu_handle_scsi_command()
556 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));  in cfiscsi_pdu_handle_scsi_command()
557 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_pdu_handle_scsi_command()
560 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "  in cfiscsi_pdu_handle_scsi_command()
563 		refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_pdu_handle_scsi_command()
565 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_scsi_command()
575 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle_task_request()  local
579 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle_task_request()
580 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;  in cfiscsi_pdu_handle_task_request()
581 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);  in cfiscsi_pdu_handle_task_request()
584 	io->io_hdr.io_type = CTL_IO_TASK;  in cfiscsi_pdu_handle_task_request()
585 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;  in cfiscsi_pdu_handle_task_request()
586 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;  in cfiscsi_pdu_handle_task_request()
587 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));  in cfiscsi_pdu_handle_task_request()
588 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */  in cfiscsi_pdu_handle_task_request()
590 	switch (bhstmr->bhstmr_function & ~0x80) {  in cfiscsi_pdu_handle_task_request()
592 #if 0  in cfiscsi_pdu_handle_task_request()
593 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");  in cfiscsi_pdu_handle_task_request()
595 		io->taskio.task_action = CTL_TASK_ABORT_TASK;  in cfiscsi_pdu_handle_task_request()
596 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;  in cfiscsi_pdu_handle_task_request()
599 #if 0  in cfiscsi_pdu_handle_task_request()
600 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");  in cfiscsi_pdu_handle_task_request()
602 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;  in cfiscsi_pdu_handle_task_request()
605 #if 0  in cfiscsi_pdu_handle_task_request()
606 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");  in cfiscsi_pdu_handle_task_request()
608 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;  in cfiscsi_pdu_handle_task_request()
611 #if 0  in cfiscsi_pdu_handle_task_request()
612 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");  in cfiscsi_pdu_handle_task_request()
614 		io->taskio.task_action = CTL_TASK_LUN_RESET;  in cfiscsi_pdu_handle_task_request()
617 #if 0  in cfiscsi_pdu_handle_task_request()
618 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");  in cfiscsi_pdu_handle_task_request()
620 		io->taskio.task_action = CTL_TASK_TARGET_RESET;  in cfiscsi_pdu_handle_task_request()
623 #if 0  in cfiscsi_pdu_handle_task_request()
624 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");  in cfiscsi_pdu_handle_task_request()
626 		io->taskio.task_action = CTL_TASK_TARGET_RESET;  in cfiscsi_pdu_handle_task_request()
629 #if 0  in cfiscsi_pdu_handle_task_request()
630 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");  in cfiscsi_pdu_handle_task_request()
632 		io->taskio.task_action = CTL_TASK_QUERY_TASK;  in cfiscsi_pdu_handle_task_request()
633 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;  in cfiscsi_pdu_handle_task_request()
636 #if 0  in cfiscsi_pdu_handle_task_request()
637 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");  in cfiscsi_pdu_handle_task_request()
639 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;  in cfiscsi_pdu_handle_task_request()
642 #if 0  in cfiscsi_pdu_handle_task_request()
643 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");  in cfiscsi_pdu_handle_task_request()
645 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;  in cfiscsi_pdu_handle_task_request()
648 #if 0  in cfiscsi_pdu_handle_task_request()
649 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");  in cfiscsi_pdu_handle_task_request()
651 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;  in cfiscsi_pdu_handle_task_request()
654 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",  in cfiscsi_pdu_handle_task_request()
655 		    bhstmr->bhstmr_function & ~0x80);  in cfiscsi_pdu_handle_task_request()
660 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "  in cfiscsi_pdu_handle_task_request()
663 			cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_task_request()
667 		    response->ip_bhs;  in cfiscsi_pdu_handle_task_request()
668 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;  in cfiscsi_pdu_handle_task_request()
669 		bhstmr2->bhstmr_flags = 0x80;  in cfiscsi_pdu_handle_task_request()
670 		bhstmr2->bhstmr_response =  in cfiscsi_pdu_handle_task_request()
672 		bhstmr2->bhstmr_initiator_task_tag =  in cfiscsi_pdu_handle_task_request()
673 		    bhstmr->bhstmr_initiator_task_tag;  in cfiscsi_pdu_handle_task_request()
679 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_pdu_handle_task_request()
682 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "  in cfiscsi_pdu_handle_task_request()
685 		refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_pdu_handle_task_request()
687 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_task_request()
695 	struct cfiscsi_session *cs;  in cfiscsi_handle_data_segment()  local
701 	cs = PDU_SESSION(request);  in cfiscsi_handle_data_segment()
703 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_handle_data_segment()
705 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_handle_data_segment()
707 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));  in cfiscsi_handle_data_segment()
710 	 * We're only using fields common for Data-Out and SCSI Command PDUs.  in cfiscsi_handle_data_segment()
712 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;  in cfiscsi_handle_data_segment()
714 	io = cdw->cdw_ctl_io;  in cfiscsi_handle_data_segment()
715 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,  in cfiscsi_handle_data_segment()
718 #if 0  in cfiscsi_handle_data_segment()
719 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",  in cfiscsi_handle_data_segment()
720 	    request->ip_data_len, io->scsiio.kern_total_len);  in cfiscsi_handle_data_segment()
723 	if (io->scsiio.kern_sg_entries > 0) {  in cfiscsi_handle_data_segment()
724 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;  in cfiscsi_handle_data_segment()
725 		ctl_sg_count = io->scsiio.kern_sg_entries;  in cfiscsi_handle_data_segment()
728 		ctl_sglist->addr = io->scsiio.kern_data_ptr;  in cfiscsi_handle_data_segment()
729 		ctl_sglist->len = io->scsiio.kern_data_len;  in cfiscsi_handle_data_segment()
733 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_handle_data_segment()
735 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);  in cfiscsi_handle_data_segment()
737 		buffer_offset = 0;  in cfiscsi_handle_data_segment()
742 	 * we're supposed to be at in the scatter-gather list.  in cfiscsi_handle_data_segment()
745 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||  in cfiscsi_handle_data_segment()
747 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {  in cfiscsi_handle_data_segment()
748 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "  in cfiscsi_handle_data_segment()
750 		    (size_t)io->scsiio.kern_rel_offset +  in cfiscsi_handle_data_segment()
751 		    (size_t)io->scsiio.ext_data_filled);  in cfiscsi_handle_data_segment()
752 		ctl_set_data_phase_error(&io->scsiio);  in cfiscsi_handle_data_segment()
753 		cfiscsi_session_terminate(cs);  in cfiscsi_handle_data_segment()
762 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -  in cfiscsi_handle_data_segment()
772 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,  in cfiscsi_handle_data_segment()
773 		    ("cdw->cdw_sg_index >= ctl_sg_count"));  in cfiscsi_handle_data_segment()
774 		if (cdw->cdw_sg_len == 0) {  in cfiscsi_handle_data_segment()
775 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;  in cfiscsi_handle_data_segment()
776 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;  in cfiscsi_handle_data_segment()
779 		copy_len = len - off;  in cfiscsi_handle_data_segment()
780 		if (copy_len > cdw->cdw_sg_len)  in cfiscsi_handle_data_segment()
781 			copy_len = cdw->cdw_sg_len;  in cfiscsi_handle_data_segment()
783 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);  in cfiscsi_handle_data_segment()
784 		cdw->cdw_sg_addr += copy_len;  in cfiscsi_handle_data_segment()
785 		cdw->cdw_sg_len -= copy_len;  in cfiscsi_handle_data_segment()
787 		io->scsiio.ext_data_filled += copy_len;  in cfiscsi_handle_data_segment()
788 		io->scsiio.kern_data_resid -= copy_len;  in cfiscsi_handle_data_segment()
790 		if (cdw->cdw_sg_len == 0) {  in cfiscsi_handle_data_segment()
794 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {  in cfiscsi_handle_data_segment()
800 			cdw->cdw_sg_index++;  in cfiscsi_handle_data_segment()
820 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_handle_data_segment()
824 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "  in cfiscsi_handle_data_segment()
827 		ctl_set_data_phase_error(&io->scsiio);  in cfiscsi_handle_data_segment()
828 		cfiscsi_session_terminate(cs);  in cfiscsi_handle_data_segment()
832 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&  in cfiscsi_handle_data_segment()
833 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {  in cfiscsi_handle_data_segment()
834 		CFISCSI_SESSION_WARN(cs, "got the final packet without "  in cfiscsi_handle_data_segment()
835 		    "the F flag; flags = 0x%x; dropping connection",  in cfiscsi_handle_data_segment()
836 		    bhsdo->bhsdo_flags);  in cfiscsi_handle_data_segment()
837 		ctl_set_data_phase_error(&io->scsiio);  in cfiscsi_handle_data_segment()
838 		cfiscsi_session_terminate(cs);  in cfiscsi_handle_data_segment()
842 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&  in cfiscsi_handle_data_segment()
843 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {  in cfiscsi_handle_data_segment()
844 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_handle_data_segment()
846 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "  in cfiscsi_handle_data_segment()
849 			    (size_t)io->scsiio.ext_data_filled,  in cfiscsi_handle_data_segment()
850 			    cdw->cdw_r2t_end);  in cfiscsi_handle_data_segment()
851 			ctl_set_data_phase_error(&io->scsiio);  in cfiscsi_handle_data_segment()
852 			cfiscsi_session_terminate(cs);  in cfiscsi_handle_data_segment()
863 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {  in cfiscsi_handle_data_segment()
864 #if 0  in cfiscsi_handle_data_segment()
865 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "  in cfiscsi_handle_data_segment()
866 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);  in cfiscsi_handle_data_segment()
879 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle_data_out()  local
884 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle_data_out()
885 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;  in cfiscsi_pdu_handle_data_out()
887 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_pdu_handle_data_out()
888 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {  in cfiscsi_pdu_handle_data_out()
889 #if 0  in cfiscsi_pdu_handle_data_out()
890 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "  in cfiscsi_pdu_handle_data_out()
891 		    "ttt 0x%x, itt 0x%x",  in cfiscsi_pdu_handle_data_out()
892 		    bhsdo->bhsdo_target_transfer_tag,  in cfiscsi_pdu_handle_data_out()
893 		    bhsdo->bhsdo_initiator_task_tag,  in cfiscsi_pdu_handle_data_out()
894 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));  in cfiscsi_pdu_handle_data_out()
896 		if (bhsdo->bhsdo_target_transfer_tag ==  in cfiscsi_pdu_handle_data_out()
897 		    cdw->cdw_target_transfer_tag)  in cfiscsi_pdu_handle_data_out()
900 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_pdu_handle_data_out()
902 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "  in cfiscsi_pdu_handle_data_out()
903 		    "0x%x, not found; dropping connection",  in cfiscsi_pdu_handle_data_out()
904 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);  in cfiscsi_pdu_handle_data_out()
906 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_data_out()
910 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {  in cfiscsi_pdu_handle_data_out()
911 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "  in cfiscsi_pdu_handle_data_out()
913 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);  in cfiscsi_pdu_handle_data_out()
915 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_data_out()
918 	cdw->cdw_datasn += request->ip_additional_pdus + 1;  in cfiscsi_pdu_handle_data_out()
920 	io = cdw->cdw_ctl_io;  in cfiscsi_pdu_handle_data_out()
921 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,  in cfiscsi_pdu_handle_data_out()
926 		CFISCSI_SESSION_LOCK(cs);  in cfiscsi_pdu_handle_data_out()
927 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);  in cfiscsi_pdu_handle_data_out()
928 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_pdu_handle_data_out()
929 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||  in cfiscsi_pdu_handle_data_out()
930 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);  in cfiscsi_pdu_handle_data_out()
931 		cfiscsi_data_wait_free(cs, cdw);  in cfiscsi_pdu_handle_data_out()
932 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;  in cfiscsi_pdu_handle_data_out()
948 	struct cfiscsi_session *cs;  in cfiscsi_pdu_handle_logout_request()  local
950 	cs = PDU_SESSION(request);  in cfiscsi_pdu_handle_logout_request()
951 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;  in cfiscsi_pdu_handle_logout_request()
952 	switch (bhslr->bhslr_reason & 0x7f) {  in cfiscsi_pdu_handle_logout_request()
957 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");  in cfiscsi_pdu_handle_logout_request()
959 			cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_logout_request()
962 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;  in cfiscsi_pdu_handle_logout_request()
963 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;  in cfiscsi_pdu_handle_logout_request()
964 		bhslr2->bhslr_flags = 0x80;  in cfiscsi_pdu_handle_logout_request()
965 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;  in cfiscsi_pdu_handle_logout_request()
966 		bhslr2->bhslr_initiator_task_tag =  in cfiscsi_pdu_handle_logout_request()
967 		    bhslr->bhslr_initiator_task_tag;  in cfiscsi_pdu_handle_logout_request()
970 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_logout_request()
975 			CFISCSI_SESSION_WARN(cs,  in cfiscsi_pdu_handle_logout_request()
978 			cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_logout_request()
981 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;  in cfiscsi_pdu_handle_logout_request()
982 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;  in cfiscsi_pdu_handle_logout_request()
983 		bhslr2->bhslr_flags = 0x80;  in cfiscsi_pdu_handle_logout_request()
984 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;  in cfiscsi_pdu_handle_logout_request()
985 		bhslr2->bhslr_initiator_task_tag =  in cfiscsi_pdu_handle_logout_request()
986 		    bhslr->bhslr_initiator_task_tag;  in cfiscsi_pdu_handle_logout_request()
991 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",  in cfiscsi_pdu_handle_logout_request()
992 		    bhslr->bhslr_reason);  in cfiscsi_pdu_handle_logout_request()
994 		cfiscsi_session_terminate(cs);  in cfiscsi_pdu_handle_logout_request()
1004 	struct cfiscsi_session *cs;  in cfiscsi_callout()  local
1006 	cs = context;  in cfiscsi_callout()
1008 	if (cs->cs_terminating)   in cfiscsi_callout()
1011 	callout_schedule(&cs->cs_callout, 1 * hz);  in cfiscsi_callout()
1013 	atomic_add_int(&cs->cs_timeout, 1);  in cfiscsi_callout()
1016 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {  in cfiscsi_callout()
1017 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {  in cfiscsi_callout()
1018 			CFISCSI_SESSION_WARN(cs, "login timed out after "  in cfiscsi_callout()
1019 			    "%d seconds; dropping connection", cs->cs_timeout);  in cfiscsi_callout()
1020 			cfiscsi_session_terminate(cs);  in cfiscsi_callout()
1026 	if (ping_timeout <= 0) {  in cfiscsi_callout()
1028 		 * Pings are disabled.  Don't send NOP-In in this case;  in cfiscsi_callout()
1031 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid  in cfiscsi_callout()
1035 		cs->cs_timeout = 0;  in cfiscsi_callout()
1039 	if (cs->cs_timeout >= ping_timeout) {  in cfiscsi_callout()
1040 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "  in cfiscsi_callout()
1042 		cfiscsi_session_terminate(cs);  in cfiscsi_callout()
1047 	 * If the ping was reset less than one second ago - which means  in cfiscsi_callout()
1048 	 * that we've received some PDU during the last second - assume  in cfiscsi_callout()
1049 	 * the traffic flows correctly and don't bother sending a NOP-Out.  in cfiscsi_callout()
1051 	 * (It's 2 - one for one second, and one for incrementing is_timeout  in cfiscsi_callout()
1054 	if (cs->cs_timeout < 2)  in cfiscsi_callout()
1057 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);  in cfiscsi_callout()
1059 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");  in cfiscsi_callout()
1062 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;  in cfiscsi_callout()
1063 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;  in cfiscsi_callout()
1064 	bhsni->bhsni_flags = 0x80;  in cfiscsi_callout()
1065 	bhsni->bhsni_initiator_task_tag = 0xffffffff;  in cfiscsi_callout()
1071 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,  in cfiscsi_data_wait_new()  argument
1079 		CFISCSI_SESSION_WARN(cs,  in cfiscsi_data_wait_new()
1084 	error = icl_conn_transfer_setup(cs->cs_conn, PRIV_REQUEST(io), io,  in cfiscsi_data_wait_new()
1085 	    target_transfer_tagp, &cdw->cdw_icl_prv);  in cfiscsi_data_wait_new()
1086 	if (error != 0) {  in cfiscsi_data_wait_new()
1087 		CFISCSI_SESSION_WARN(cs,  in cfiscsi_data_wait_new()
1093 	cdw->cdw_ctl_io = io;  in cfiscsi_data_wait_new()
1094 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;  in cfiscsi_data_wait_new()
1095 	cdw->cdw_initiator_task_tag = initiator_task_tag;  in cfiscsi_data_wait_new()
1101 cfiscsi_data_wait_free(struct cfiscsi_session *cs,  in cfiscsi_data_wait_free()  argument
1105 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);  in cfiscsi_data_wait_free()
1110 cfiscsi_data_wait_abort(struct cfiscsi_session *cs,  in cfiscsi_data_wait_abort()  argument
1120 	MPASS(status != 0);  in cfiscsi_data_wait_abort()
1121 	cdw_io = cdw->cdw_ctl_io;  in cfiscsi_data_wait_abort()
1122 	cdw_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;  in cfiscsi_data_wait_abort()
1123 	cdw_io->scsiio.io_hdr.port_status = status;  in cfiscsi_data_wait_abort()
1124 	cfiscsi_data_wait_free(cs, cdw);  in cfiscsi_data_wait_abort()
1129 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)  in cfiscsi_session_terminate_tasks()  argument
1136 	if (cs->cs_target == NULL)  in cfiscsi_session_terminate_tasks()
1138 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);  in cfiscsi_session_terminate_tasks()
1139 	ip->ip_bhs->bhs_opcode = ISCSI_BHS_OPCODE_INTERNAL;  in cfiscsi_session_terminate_tasks()
1140 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);  in cfiscsi_session_terminate_tasks()
1143 	io->io_hdr.io_type = CTL_IO_TASK;  in cfiscsi_session_terminate_tasks()
1144 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;  in cfiscsi_session_terminate_tasks()
1145 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;  in cfiscsi_session_terminate_tasks()
1146 	io->io_hdr.nexus.targ_lun = 0;  in cfiscsi_session_terminate_tasks()
1147 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */  in cfiscsi_session_terminate_tasks()
1148 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;  in cfiscsi_session_terminate_tasks()
1149 	wait = cs->cs_outstanding_ctl_pdus;  in cfiscsi_session_terminate_tasks()
1150 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_session_terminate_tasks()
1153 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d", error);  in cfiscsi_session_terminate_tasks()
1154 		refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_session_terminate_tasks()
1159 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_session_terminate_tasks()
1160 	cs->cs_terminating_tasks = true;  in cfiscsi_session_terminate_tasks()
1161 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {  in cfiscsi_session_terminate_tasks()
1162 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);  in cfiscsi_session_terminate_tasks()
1163 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_session_terminate_tasks()
1164 		cfiscsi_data_wait_abort(cs, cdw, 42);  in cfiscsi_session_terminate_tasks()
1165 		CFISCSI_SESSION_LOCK(cs);  in cfiscsi_session_terminate_tasks()
1167 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_session_terminate_tasks()
1172 	if (wait > 0)  in cfiscsi_session_terminate_tasks()
1173 		CFISCSI_SESSION_WARN(cs,  in cfiscsi_session_terminate_tasks()
1176 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_session_terminate_tasks()
1177 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_session_terminate_tasks()
1178 		if (last != 0)  in cfiscsi_session_terminate_tasks()
1180 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),  in cfiscsi_session_terminate_tasks()
1181 		    0, "cfiscsi_terminate", hz / 100);  in cfiscsi_session_terminate_tasks()
1183 	if (wait > 0)  in cfiscsi_session_terminate_tasks()
1184 		CFISCSI_SESSION_WARN(cs, "tasks terminated");  in cfiscsi_session_terminate_tasks()
1190 	struct cfiscsi_session *cs;  in cfiscsi_maintenance_thread()  local
1192 	cs = arg;  in cfiscsi_maintenance_thread()
1195 		CFISCSI_SESSION_LOCK(cs);  in cfiscsi_maintenance_thread()
1196 		if (cs->cs_terminating == false || cs->cs_handoff_in_progress)  in cfiscsi_maintenance_thread()
1197 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);  in cfiscsi_maintenance_thread()
1198 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_maintenance_thread()
1200 		if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {  in cfiscsi_maintenance_thread()
1207 			callout_drain(&cs->cs_callout);  in cfiscsi_maintenance_thread()
1208 			icl_conn_close(cs->cs_conn);  in cfiscsi_maintenance_thread()
1214 			cfiscsi_session_terminate_tasks(cs);  in cfiscsi_maintenance_thread()
1215 			cfiscsi_session_delete(cs);  in cfiscsi_maintenance_thread()
1219 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");  in cfiscsi_maintenance_thread()
1224 cfiscsi_session_terminate(struct cfiscsi_session *cs)  in cfiscsi_session_terminate()  argument
1227 	cs->cs_terminating = true;  in cfiscsi_session_terminate()
1228 	cv_signal(&cs->cs_maintenance_cv);  in cfiscsi_session_terminate()
1230 	cv_signal(&cs->cs_login_cv);  in cfiscsi_session_terminate()
1235 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)  in cfiscsi_session_register_initiator()  argument
1241 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));  in cfiscsi_session_register_initiator()
1243 	ct = cs->cs_target;  in cfiscsi_session_register_initiator()
1244 	name = strdup(cs->cs_initiator_id, M_CTL);  in cfiscsi_session_register_initiator()
1245 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);  in cfiscsi_session_register_initiator()
1246 	if (i < 0) {  in cfiscsi_session_register_initiator()
1247 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",  in cfiscsi_session_register_initiator()
1249 		cs->cs_ctl_initid = -1;  in cfiscsi_session_register_initiator()
1252 	cs->cs_ctl_initid = i;  in cfiscsi_session_register_initiator()
1253 #if 0  in cfiscsi_session_register_initiator()
1254 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);  in cfiscsi_session_register_initiator()
1257 	return (0);  in cfiscsi_session_register_initiator()
1261 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)  in cfiscsi_session_unregister_initiator()  argument
1265 	if (cs->cs_ctl_initid == -1)  in cfiscsi_session_unregister_initiator()
1268 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);  in cfiscsi_session_unregister_initiator()
1269 	if (error != 0) {  in cfiscsi_session_unregister_initiator()
1270 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",  in cfiscsi_session_unregister_initiator()
1273 	cs->cs_ctl_initid = -1;  in cfiscsi_session_unregister_initiator()
1279 	struct cfiscsi_session *cs;  in cfiscsi_session_new()  local
1282 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);  in cfiscsi_session_new()
1283 	if (cs == NULL) {  in cfiscsi_session_new()
1287 	cs->cs_ctl_initid = -1;  in cfiscsi_session_new()
1289 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);  in cfiscsi_session_new()
1290 	TAILQ_INIT(&cs->cs_waiting_for_data_out);  in cfiscsi_session_new()
1291 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);  in cfiscsi_session_new()
1292 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");  in cfiscsi_session_new()
1294 	cv_init(&cs->cs_login_cv, "cfiscsi_login");  in cfiscsi_session_new()
1302 	cs->cs_handoff_in_progress = true;  in cfiscsi_session_new()
1304 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);  in cfiscsi_session_new()
1305 	if (cs->cs_conn == NULL) {  in cfiscsi_session_new()
1306 		free(cs, M_CFISCSI);  in cfiscsi_session_new()
1309 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;  in cfiscsi_session_new()
1310 	cs->cs_conn->ic_error = cfiscsi_error_callback;  in cfiscsi_session_new()
1311 	cs->cs_conn->ic_prv0 = cs;  in cfiscsi_session_new()
1313 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");  in cfiscsi_session_new()
1314 	if (error != 0) {  in cfiscsi_session_new()
1315 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);  in cfiscsi_session_new()
1316 		free(cs, M_CFISCSI);  in cfiscsi_session_new()
1320 	mtx_lock(&softc->lock);  in cfiscsi_session_new()
1321 	cs->cs_id = ++softc->last_session_id;  in cfiscsi_session_new()
1322 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);  in cfiscsi_session_new()
1323 	mtx_unlock(&softc->lock);  in cfiscsi_session_new()
1328 	callout_init(&cs->cs_callout, 1);  in cfiscsi_session_new()
1329 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);  in cfiscsi_session_new()
1331 	return (cs);  in cfiscsi_session_new()
1335 cfiscsi_session_delete(struct cfiscsi_session *cs)  in cfiscsi_session_delete()  argument
1341 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,  in cfiscsi_session_delete()
1343 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),  in cfiscsi_session_delete()
1344 	    ("destroying session with non-empty queue"));  in cfiscsi_session_delete()
1346 	mtx_lock(&softc->lock);  in cfiscsi_session_delete()
1347 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);  in cfiscsi_session_delete()
1348 	mtx_unlock(&softc->lock);  in cfiscsi_session_delete()
1350 	cfiscsi_session_unregister_initiator(cs);  in cfiscsi_session_delete()
1351 	if (cs->cs_target != NULL)  in cfiscsi_session_delete()
1352 		cfiscsi_target_release(cs->cs_target);  in cfiscsi_session_delete()
1353 	icl_conn_close(cs->cs_conn);  in cfiscsi_session_delete()
1354 	icl_conn_free(cs->cs_conn);  in cfiscsi_session_delete()
1355 	free(cs, M_CFISCSI);  in cfiscsi_session_delete()
1356 	cv_signal(&softc->sessions_cv);  in cfiscsi_session_delete()
1366 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);  in cfiscsi_init()
1368 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");  in cfiscsi_init()
1370 	cv_init(&softc->accept_cv, "cfiscsi_accept");  in cfiscsi_init()
1372 	TAILQ_INIT(&softc->sessions);  in cfiscsi_init()
1373 	TAILQ_INIT(&softc->targets);  in cfiscsi_init()
1377 	    UMA_ALIGN_PTR, 0);  in cfiscsi_init()
1379 	return (0);  in cfiscsi_init()
1387 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))  in cfiscsi_shutdown()
1392 	cv_destroy(&softc->accept_cv);  in cfiscsi_shutdown()
1394 	cv_destroy(&softc->sessions_cv);  in cfiscsi_shutdown()
1395 	mtx_destroy(&softc->lock);  in cfiscsi_shutdown()
1396 	return (0);  in cfiscsi_shutdown()
1403 	struct cfiscsi_session *cs;  in cfiscsi_accept()  local
1405 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);  in cfiscsi_accept()
1406 	if (cs == NULL) {  in cfiscsi_accept()
1411 	icl_conn_handoff_sock(cs->cs_conn, so);  in cfiscsi_accept()
1412 	cs->cs_initiator_sa = sa;  in cfiscsi_accept()
1413 	cs->cs_portal_id = portal_id;  in cfiscsi_accept()
1414 	cs->cs_handoff_in_progress = false;  in cfiscsi_accept()
1415 	cs->cs_waiting_for_ctld = true;  in cfiscsi_accept()
1418 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_accept()
1423 	if (cs->cs_terminating)  in cfiscsi_accept()
1424 		cfiscsi_session_terminate(cs);  in cfiscsi_accept()
1425 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_accept()
1437 	softc = ct->ct_softc;  in cfiscsi_online()
1439 	mtx_lock(&softc->lock);  in cfiscsi_online()
1440 	if (ct->ct_online) {  in cfiscsi_online()
1441 		mtx_unlock(&softc->lock);  in cfiscsi_online()
1444 	ct->ct_online = 1;  in cfiscsi_online()
1445 	online = softc->online++;  in cfiscsi_online()
1446 	mtx_unlock(&softc->lock);  in cfiscsi_online()
1447 	if (online > 0)  in cfiscsi_online()
1451 	if (softc->listener != NULL)  in cfiscsi_online()
1452 		icl_listen_free(softc->listener);  in cfiscsi_online()
1453 	softc->listener = icl_listen_new(cfiscsi_accept);  in cfiscsi_online()
1462 	struct cfiscsi_session *cs;  in cfiscsi_offline()  local
1466 	softc = ct->ct_softc;  in cfiscsi_offline()
1468 	mtx_lock(&softc->lock);  in cfiscsi_offline()
1469 	if (!ct->ct_online) {  in cfiscsi_offline()
1470 		mtx_unlock(&softc->lock);  in cfiscsi_offline()
1473 	ct->ct_online = 0;  in cfiscsi_offline()
1474 	online = --softc->online;  in cfiscsi_offline()
1477 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {  in cfiscsi_offline()
1478 			if (cs->cs_target == ct)  in cfiscsi_offline()
1479 				cfiscsi_session_terminate(cs);  in cfiscsi_offline()
1481 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {  in cfiscsi_offline()
1482 			if (cs->cs_target == ct)  in cfiscsi_offline()
1485 		if (cs != NULL) {  in cfiscsi_offline()
1486 			error = cv_wait_sig(&softc->sessions_cv, &softc->lock);  in cfiscsi_offline()
1487 			if (error != 0) {  in cfiscsi_offline()
1488 				CFISCSI_SESSION_DEBUG(cs,  in cfiscsi_offline()
1493 	} while (cs != NULL && ct->ct_online == 0);  in cfiscsi_offline()
1494 	mtx_unlock(&softc->lock);  in cfiscsi_offline()
1495 	if (online > 0)  in cfiscsi_offline()
1499 	icl_listen_free(softc->listener);  in cfiscsi_offline()
1500 	softc->listener = NULL;  in cfiscsi_offline()
1511 	    ct->ct_state);  in cfiscsi_info()
1519 	struct cfiscsi_session *cs, *cs2;  in cfiscsi_ioctl_handoff()  local
1524 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);  in cfiscsi_ioctl_handoff()
1528 	    cihp->initiator_name, cihp->initiator_addr,  in cfiscsi_ioctl_handoff()
1529 	    cihp->target_name);  in cfiscsi_ioctl_handoff()
1531 	ct = cfiscsi_target_find(softc, cihp->target_name,  in cfiscsi_ioctl_handoff()
1532 	    cihp->portal_group_tag);  in cfiscsi_ioctl_handoff()
1534 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1535 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1541 	if (cihp->socket > 0 && cihp->connection_id > 0) {  in cfiscsi_ioctl_handoff()
1542 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1544 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1548 	if (cihp->socket == 0) {  in cfiscsi_ioctl_handoff()
1550 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {  in cfiscsi_ioctl_handoff()
1551 			if (cs->cs_id == cihp->connection_id)  in cfiscsi_ioctl_handoff()
1554 		if (cs == NULL) {  in cfiscsi_ioctl_handoff()
1556 			snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1558 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1565 		cs = cfiscsi_session_new(softc, cihp->offload);  in cfiscsi_ioctl_handoff()
1566 		if (cs == NULL) {  in cfiscsi_ioctl_handoff()
1567 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1568 			snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1580 	 * the -1 below.  in cfiscsi_ioctl_handoff()
1582 	cs->cs_cmdsn = cihp->cmdsn;  in cfiscsi_ioctl_handoff()
1583 	cs->cs_statsn = cihp->statsn;  in cfiscsi_ioctl_handoff()
1584 	cs->cs_conn->ic_max_recv_data_segment_length =  in cfiscsi_ioctl_handoff()
1585 	    cihp->max_recv_data_segment_length;  in cfiscsi_ioctl_handoff()
1586 	cs->cs_conn->ic_max_send_data_segment_length =  in cfiscsi_ioctl_handoff()
1587 	    cihp->max_send_data_segment_length;  in cfiscsi_ioctl_handoff()
1588 	cs->cs_max_burst_length = cihp->max_burst_length;  in cfiscsi_ioctl_handoff()
1589 	cs->cs_first_burst_length = cihp->first_burst_length;  in cfiscsi_ioctl_handoff()
1590 	cs->cs_immediate_data = !!cihp->immediate_data;  in cfiscsi_ioctl_handoff()
1591 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)  in cfiscsi_ioctl_handoff()
1592 		cs->cs_conn->ic_header_crc32c = true;  in cfiscsi_ioctl_handoff()
1593 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)  in cfiscsi_ioctl_handoff()
1594 		cs->cs_conn->ic_data_crc32c = true;  in cfiscsi_ioctl_handoff()
1596 	strlcpy(cs->cs_initiator_name,  in cfiscsi_ioctl_handoff()
1597 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));  in cfiscsi_ioctl_handoff()
1598 	strlcpy(cs->cs_initiator_addr,  in cfiscsi_ioctl_handoff()
1599 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));  in cfiscsi_ioctl_handoff()
1600 	strlcpy(cs->cs_initiator_alias,  in cfiscsi_ioctl_handoff()
1601 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));  in cfiscsi_ioctl_handoff()
1602 	memcpy(cs->cs_initiator_isid,  in cfiscsi_ioctl_handoff()
1603 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));  in cfiscsi_ioctl_handoff()
1604 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),  in cfiscsi_ioctl_handoff()
1605 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,  in cfiscsi_ioctl_handoff()
1606 	    cihp->initiator_isid[0], cihp->initiator_isid[1],  in cfiscsi_ioctl_handoff()
1607 	    cihp->initiator_isid[2], cihp->initiator_isid[3],  in cfiscsi_ioctl_handoff()
1608 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);  in cfiscsi_ioctl_handoff()
1610 	mtx_lock(&softc->lock);  in cfiscsi_ioctl_handoff()
1611 	if (ct->ct_online == 0) {  in cfiscsi_ioctl_handoff()
1612 		mtx_unlock(&softc->lock);  in cfiscsi_ioctl_handoff()
1613 		CFISCSI_SESSION_LOCK(cs);  in cfiscsi_ioctl_handoff()
1614 		cs->cs_handoff_in_progress = false;  in cfiscsi_ioctl_handoff()
1615 		cfiscsi_session_terminate(cs);  in cfiscsi_ioctl_handoff()
1616 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_handoff()
1618 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1619 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1623 	cs->cs_target = ct;  in cfiscsi_ioctl_handoff()
1624 	mtx_unlock(&softc->lock);  in cfiscsi_ioctl_handoff()
1627 	if (!cs->cs_terminating) {  in cfiscsi_ioctl_handoff()
1628 		mtx_lock(&softc->lock);  in cfiscsi_ioctl_handoff()
1629 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {  in cfiscsi_ioctl_handoff()
1630 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&  in cfiscsi_ioctl_handoff()
1631 			    cs->cs_target == cs2->cs_target &&  in cfiscsi_ioctl_handoff()
1632 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {  in cfiscsi_ioctl_handoff()
1633 				if (strcmp(cs->cs_initiator_addr,  in cfiscsi_ioctl_handoff()
1634 				    cs2->cs_initiator_addr) != 0) {  in cfiscsi_ioctl_handoff()
1638 					    cs->cs_initiator_addr);  in cfiscsi_ioctl_handoff()
1644 				mtx_unlock(&softc->lock);  in cfiscsi_ioctl_handoff()
1649 		mtx_unlock(&softc->lock);  in cfiscsi_ioctl_handoff()
1655 	cfiscsi_session_register_initiator(cs);  in cfiscsi_ioctl_handoff()
1658 	if (cihp->socket > 0) {  in cfiscsi_ioctl_handoff()
1660 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);  in cfiscsi_ioctl_handoff()
1661 		if (error != 0) {  in cfiscsi_ioctl_handoff()
1662 			CFISCSI_SESSION_LOCK(cs);  in cfiscsi_ioctl_handoff()
1663 			cs->cs_handoff_in_progress = false;  in cfiscsi_ioctl_handoff()
1664 			cfiscsi_session_terminate(cs);  in cfiscsi_ioctl_handoff()
1665 			CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_handoff()
1666 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_handoff()
1667 			snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_handoff()
1677 	cs->cs_login_phase = false;  in cfiscsi_ioctl_handoff()
1683 	if (cs->cs_login_pdu != NULL) {  in cfiscsi_ioctl_handoff()
1684 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");  in cfiscsi_ioctl_handoff()
1685 		cfiscsi_pdu_handle(cs->cs_login_pdu);  in cfiscsi_ioctl_handoff()
1686 		cs->cs_login_pdu = NULL;  in cfiscsi_ioctl_handoff()
1690 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_ioctl_handoff()
1691 	cs->cs_handoff_in_progress = false;  in cfiscsi_ioctl_handoff()
1696 	if (cs->cs_terminating)  in cfiscsi_ioctl_handoff()
1697 		cfiscsi_session_terminate(cs);  in cfiscsi_ioctl_handoff()
1698 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_handoff()
1700 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_handoff()
1707 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_list()  local
1712 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);  in cfiscsi_ioctl_list()
1715 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);  in cfiscsi_ioctl_list()
1717 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_list()
1718 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_list()
1720 		    cilp->alloc_len);  in cfiscsi_ioctl_list()
1725 	mtx_lock(&softc->lock);  in cfiscsi_ioctl_list()
1726 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {  in cfiscsi_ioctl_list()
1727 		if (cs->cs_target == NULL)  in cfiscsi_ioctl_list()
1746 		    cs->cs_id,  in cfiscsi_ioctl_list()
1747 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,  in cfiscsi_ioctl_list()
1748 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,  in cfiscsi_ioctl_list()
1749 		    cs->cs_target->ct_tag,  in cfiscsi_ioctl_list()
1750 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",  in cfiscsi_ioctl_list()
1751 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",  in cfiscsi_ioctl_list()
1752 		    cs->cs_conn->ic_max_recv_data_segment_length,  in cfiscsi_ioctl_list()
1753 		    cs->cs_conn->ic_max_send_data_segment_length,  in cfiscsi_ioctl_list()
1754 		    cs->cs_max_burst_length,  in cfiscsi_ioctl_list()
1755 		    cs->cs_first_burst_length,  in cfiscsi_ioctl_list()
1756 		    cs->cs_immediate_data,  in cfiscsi_ioctl_list()
1757 		    cs->cs_conn->ic_iser,  in cfiscsi_ioctl_list()
1758 		    cs->cs_conn->ic_offload);  in cfiscsi_ioctl_list()
1759 		if (error != 0)  in cfiscsi_ioctl_list()
1762 	mtx_unlock(&softc->lock);  in cfiscsi_ioctl_list()
1764 	if (error != 0) {  in cfiscsi_ioctl_list()
1766 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;  in cfiscsi_ioctl_list()
1767 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_list()
1768 		    "Out of space, %d bytes is too small", cilp->alloc_len);  in cfiscsi_ioctl_list()
1773 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);  in cfiscsi_ioctl_list()
1774 	if (error != 0) {  in cfiscsi_ioctl_list()
1776 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_list()
1778 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_list()
1781 	cilp->fill_len = sbuf_len(sb) + 1;  in cfiscsi_ioctl_list()
1782 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_list()
1792 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_logout()  local
1794 	int found = 0;  in cfiscsi_ioctl_logout()
1796 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);  in cfiscsi_ioctl_logout()
1799 	mtx_lock(&softc->lock);  in cfiscsi_ioctl_logout()
1800 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {  in cfiscsi_ioctl_logout()
1801 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&  in cfiscsi_ioctl_logout()
1802 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&  in cfiscsi_ioctl_logout()
1803 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)  in cfiscsi_ioctl_logout()
1806 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);  in cfiscsi_ioctl_logout()
1808 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_logout()
1809 			snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_logout()
1811 			mtx_unlock(&softc->lock);  in cfiscsi_ioctl_logout()
1815 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;  in cfiscsi_ioctl_logout()
1816 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;  in cfiscsi_ioctl_logout()
1817 		bhsam->bhsam_flags = 0x80;  in cfiscsi_ioctl_logout()
1818 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;  in cfiscsi_ioctl_logout()
1819 		bhsam->bhsam_parameter3 = htons(10);  in cfiscsi_ioctl_logout()
1823 	mtx_unlock(&softc->lock);  in cfiscsi_ioctl_logout()
1825 	if (found == 0) {  in cfiscsi_ioctl_logout()
1826 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;  in cfiscsi_ioctl_logout()
1827 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_logout()
1832 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_logout()
1841 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_terminate()  local
1843 	int found = 0;  in cfiscsi_ioctl_terminate()
1845 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);  in cfiscsi_ioctl_terminate()
1848 	mtx_lock(&softc->lock);  in cfiscsi_ioctl_terminate()
1849 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {  in cfiscsi_ioctl_terminate()
1850 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&  in cfiscsi_ioctl_terminate()
1851 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&  in cfiscsi_ioctl_terminate()
1852 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)  in cfiscsi_ioctl_terminate()
1855 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);  in cfiscsi_ioctl_terminate()
1862 			    response->ip_bhs;  in cfiscsi_ioctl_terminate()
1863 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;  in cfiscsi_ioctl_terminate()
1864 			bhsam->bhsam_flags = 0x80;  in cfiscsi_ioctl_terminate()
1865 			bhsam->bhsam_0xffffffff = 0xffffffff;  in cfiscsi_ioctl_terminate()
1866 			bhsam->bhsam_async_event =  in cfiscsi_ioctl_terminate()
1870 		cfiscsi_session_terminate(cs);  in cfiscsi_ioctl_terminate()
1873 	mtx_unlock(&softc->lock);  in cfiscsi_ioctl_terminate()
1875 	if (found == 0) {  in cfiscsi_ioctl_terminate()
1876 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;  in cfiscsi_ioctl_terminate()
1877 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_terminate()
1882 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_terminate()
1892 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);  in cfiscsi_ioctl_limits()
1894 	error = icl_limits(cilp->offload, false, cilp->socket, &idl);  in cfiscsi_ioctl_limits()
1895 	if (error != 0) {  in cfiscsi_ioctl_limits()
1896 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_limits()
1897 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_limits()
1903 	cilp->max_recv_data_segment_length =  in cfiscsi_ioctl_limits()
1905 	cilp->max_send_data_segment_length =  in cfiscsi_ioctl_limits()
1907 	cilp->max_burst_length = idl.idl_max_burst_length;  in cfiscsi_ioctl_limits()
1908 	cilp->first_burst_length = idl.idl_first_burst_length;  in cfiscsi_ioctl_limits()
1910 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_limits()
1921 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);  in cfiscsi_ioctl_listen()
1925 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");  in cfiscsi_ioctl_listen()
1926 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_listen()
1930 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);  in cfiscsi_ioctl_listen()
1931 	if (error != 0) {  in cfiscsi_ioctl_listen()
1933 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");  in cfiscsi_ioctl_listen()
1934 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_listen()
1938 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,  in cfiscsi_ioctl_listen()
1939 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);  in cfiscsi_ioctl_listen()
1940 	if (error != 0) {  in cfiscsi_ioctl_listen()
1943 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_listen()
1945 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_listen()
1949 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_listen()
1956 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_accept()  local
1959 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);  in cfiscsi_ioctl_accept()
1963 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {  in cfiscsi_ioctl_accept()
1964 			if (cs->cs_waiting_for_ctld)  in cfiscsi_ioctl_accept()
1967 		if (cs != NULL)  in cfiscsi_ioctl_accept()
1970 		if (error != 0) {  in cfiscsi_ioctl_accept()
1972 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");  in cfiscsi_ioctl_accept()
1973 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_accept()
1979 	cs->cs_waiting_for_ctld = false;  in cfiscsi_ioctl_accept()
1980 	cs->cs_login_phase = true;  in cfiscsi_ioctl_accept()
1982 	ciap->connection_id = cs->cs_id;  in cfiscsi_ioctl_accept()
1983 	ciap->portal_id = cs->cs_portal_id;  in cfiscsi_ioctl_accept()
1984 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;  in cfiscsi_ioctl_accept()
1985 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,  in cfiscsi_ioctl_accept()
1986 	    cs->cs_initiator_sa->sa_len);  in cfiscsi_ioctl_accept()
1987 	if (error != 0) {  in cfiscsi_ioctl_accept()
1988 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_accept()
1990 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_accept()
1994 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_accept()
2001 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_send()  local
2007 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);  in cfiscsi_ioctl_send()
2010 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {  in cfiscsi_ioctl_send()
2011 		if (cs->cs_id == cisp->connection_id)  in cfiscsi_ioctl_send()
2014 	if (cs == NULL) {  in cfiscsi_ioctl_send()
2016 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");  in cfiscsi_ioctl_send()
2017 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_send()
2022 #if 0  in cfiscsi_ioctl_send()
2023 	if (cs->cs_login_phase == false)  in cfiscsi_ioctl_send()
2027 	if (cs->cs_terminating) {  in cfiscsi_ioctl_send()
2028 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");  in cfiscsi_ioctl_send()
2029 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_send()
2033 	datalen = cisp->data_segment_len;  in cfiscsi_ioctl_send()
2039 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");  in cfiscsi_ioctl_send()
2040 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_send()
2043 	if (datalen > 0) {  in cfiscsi_ioctl_send()
2045 		error = copyin(cisp->data_segment, data, datalen);  in cfiscsi_ioctl_send()
2046 		if (error != 0) {  in cfiscsi_ioctl_send()
2048 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);  in cfiscsi_ioctl_send()
2049 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_send()
2054 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);  in cfiscsi_ioctl_send()
2055 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));  in cfiscsi_ioctl_send()
2056 	if (datalen > 0) {  in cfiscsi_ioctl_send()
2060 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_ioctl_send()
2062 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_send()
2063 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_send()
2070 	struct cfiscsi_session *cs;  in cfiscsi_ioctl_receive()  local
2075 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);  in cfiscsi_ioctl_receive()
2078 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {  in cfiscsi_ioctl_receive()
2079 		if (cs->cs_id == cirp->connection_id)  in cfiscsi_ioctl_receive()
2082 	if (cs == NULL) {  in cfiscsi_ioctl_receive()
2084 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_receive()
2086 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_receive()
2091 #if 0  in cfiscsi_ioctl_receive()
2092 	if (is->is_login_phase == false)  in cfiscsi_ioctl_receive()
2096 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_ioctl_receive()
2097 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {  in cfiscsi_ioctl_receive()
2098 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);  in cfiscsi_ioctl_receive()
2099 		if (error != 0) {  in cfiscsi_ioctl_receive()
2100 			CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_receive()
2101 			snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_receive()
2103 			ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_receive()
2108 	if (cs->cs_terminating) {  in cfiscsi_ioctl_receive()
2109 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_receive()
2110 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_receive()
2112 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_receive()
2115 	ip = cs->cs_login_pdu;  in cfiscsi_ioctl_receive()
2116 	cs->cs_login_pdu = NULL;  in cfiscsi_ioctl_receive()
2117 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_ioctl_receive()
2119 	if (ip->ip_data_len > cirp->data_segment_len) {  in cfiscsi_ioctl_receive()
2121 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl_receive()
2123 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl_receive()
2127 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));  in cfiscsi_ioctl_receive()
2128 	if (ip->ip_data_len > 0) {  in cfiscsi_ioctl_receive()
2129 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);  in cfiscsi_ioctl_receive()
2130 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);  in cfiscsi_ioctl_receive()
2131 		copyout(data, cirp->data_segment, ip->ip_data_len);  in cfiscsi_ioctl_receive()
2136 	ci->status = CTL_ISCSI_OK;  in cfiscsi_ioctl_receive()
2151 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);  in cfiscsi_ioctl_port_create()
2153 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_create()
2154 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_create()
2159 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",  in cfiscsi_ioctl_port_create()
2162 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_create()
2163 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_create()
2168 	alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);  in cfiscsi_ioctl_port_create()
2170 	tag = strtoul(val, NULL, 0);  in cfiscsi_ioctl_port_create()
2173 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_create()
2174 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_create()
2178 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {  in cfiscsi_ioctl_port_create()
2179 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_create()
2180 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_create()
2186 	port = &ct->ct_port;  in cfiscsi_ioctl_port_create()
2188 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)  in cfiscsi_ioctl_port_create()
2191 	port->frontend = &cfiscsi_frontend;  in cfiscsi_ioctl_port_create()
2192 	port->port_type = CTL_PORT_ISCSI;  in cfiscsi_ioctl_port_create()
2194 	port->num_requested_ctl_io = 4096;  in cfiscsi_ioctl_port_create()
2195 	port->port_name = "iscsi";  in cfiscsi_ioctl_port_create()
2196 	port->physical_port = (int)tag;  in cfiscsi_ioctl_port_create()
2197 	port->virtual_port = ct->ct_target_id;  in cfiscsi_ioctl_port_create()
2198 	port->port_online = cfiscsi_online;  in cfiscsi_ioctl_port_create()
2199 	port->port_offline = cfiscsi_offline;  in cfiscsi_ioctl_port_create()
2200 	port->port_info = cfiscsi_info;  in cfiscsi_ioctl_port_create()
2201 	port->onoff_arg = ct;  in cfiscsi_ioctl_port_create()
2202 	port->fe_datamove = cfiscsi_datamove;  in cfiscsi_ioctl_port_create()
2203 	port->fe_done = cfiscsi_done;  in cfiscsi_ioctl_port_create()
2204 	port->targ_port = -1;  in cfiscsi_ioctl_port_create()
2205 	port->options = nvlist_clone(req->args_nvl);  in cfiscsi_ioctl_port_create()
2208 	idlen = strlen(target) + strlen(",t,0x0001") + 1;  in cfiscsi_ioctl_port_create()
2211 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,  in cfiscsi_ioctl_port_create()
2213 	port->port_devid->len = len;  in cfiscsi_ioctl_port_create()
2214 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;  in cfiscsi_ioctl_port_create()
2215 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;  in cfiscsi_ioctl_port_create()
2216 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |  in cfiscsi_ioctl_port_create()
2218 	desc->length = idlen;  in cfiscsi_ioctl_port_create()
2219 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);  in cfiscsi_ioctl_port_create()
2225 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,  in cfiscsi_ioctl_port_create()
2227 	port->target_devid->len = len;  in cfiscsi_ioctl_port_create()
2228 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;  in cfiscsi_ioctl_port_create()
2229 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;  in cfiscsi_ioctl_port_create()
2230 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |  in cfiscsi_ioctl_port_create()
2232 	desc->length = idlen;  in cfiscsi_ioctl_port_create()
2233 	strlcpy(desc->identifier, target, idlen);  in cfiscsi_ioctl_port_create()
2236 	if (retval != 0) {  in cfiscsi_ioctl_port_create()
2237 		free(port->port_devid, M_CFISCSI);  in cfiscsi_ioctl_port_create()
2238 		free(port->target_devid, M_CFISCSI);  in cfiscsi_ioctl_port_create()
2240 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_create()
2241 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_create()
2246 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;  in cfiscsi_ioctl_port_create()
2247 	req->status = CTL_LUN_OK;  in cfiscsi_ioctl_port_create()
2248 	req->result_nvl = nvlist_create(0);  in cfiscsi_ioctl_port_create()
2249 	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);  in cfiscsi_ioctl_port_create()
2259 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);  in cfiscsi_ioctl_port_remove()
2261 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_remove()
2262 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_remove()
2267 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",  in cfiscsi_ioctl_port_remove()
2270 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_remove()
2271 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_remove()
2276 	tag = strtoul(val, NULL, 0);  in cfiscsi_ioctl_port_remove()
2279 		req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl_port_remove()
2280 		snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl_port_remove()
2285 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;  in cfiscsi_ioctl_port_remove()
2286 	ctl_port_offline(&ct->ct_port);  in cfiscsi_ioctl_port_remove()
2289 	req->status = CTL_LUN_OK;  in cfiscsi_ioctl_port_remove()
2301 		switch (req->reqtype) {  in cfiscsi_ioctl()
2309 			req->status = CTL_LUN_ERROR;  in cfiscsi_ioctl()
2310 			snprintf(req->error_str, sizeof(req->error_str),  in cfiscsi_ioctl()
2311 			    "Unsupported request type %d", req->reqtype);  in cfiscsi_ioctl()
2313 		return (0);  in cfiscsi_ioctl()
2320 	switch (ci->type) {  in cfiscsi_ioctl()
2354 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl()
2355 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl()
2361 		ci->status = CTL_ISCSI_ERROR;  in cfiscsi_ioctl()
2362 		snprintf(ci->error_str, sizeof(ci->error_str),  in cfiscsi_ioctl()
2363 		    "%s: invalid iSCSI request type %d", __func__, ci->type);  in cfiscsi_ioctl()
2367 	return (0);  in cfiscsi_ioctl()
2374 	refcount_acquire(&ct->ct_refcount);  in cfiscsi_target_hold()
2382 	softc = ct->ct_softc;  in cfiscsi_target_release()
2383 	mtx_lock(&softc->lock);  in cfiscsi_target_release()
2384 	if (refcount_release(&ct->ct_refcount)) {  in cfiscsi_target_release()
2385 		TAILQ_REMOVE(&softc->targets, ct, ct_next);  in cfiscsi_target_release()
2386 		mtx_unlock(&softc->lock);  in cfiscsi_target_release()
2387 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {  in cfiscsi_target_release()
2388 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;  in cfiscsi_target_release()
2389 			if (ctl_port_deregister(&ct->ct_port) != 0)  in cfiscsi_target_release()
2397 	mtx_unlock(&softc->lock);  in cfiscsi_target_release()
2405 	mtx_lock(&softc->lock);  in cfiscsi_target_find()
2406 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {  in cfiscsi_target_find()
2407 		if (ct->ct_tag != tag ||  in cfiscsi_target_find()
2408 		    strcmp(name, ct->ct_name) != 0 ||  in cfiscsi_target_find()
2409 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)  in cfiscsi_target_find()
2412 		mtx_unlock(&softc->lock);  in cfiscsi_target_find()
2415 	mtx_unlock(&softc->lock);  in cfiscsi_target_find()
2426 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)  in cfiscsi_target_find_or_create()
2431 	mtx_lock(&softc->lock);  in cfiscsi_target_find_or_create()
2432 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {  in cfiscsi_target_find_or_create()
2433 		if (ct->ct_tag != tag ||  in cfiscsi_target_find_or_create()
2434 		    strcmp(name, ct->ct_name) != 0 ||  in cfiscsi_target_find_or_create()
2435 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)  in cfiscsi_target_find_or_create()
2438 		mtx_unlock(&softc->lock);  in cfiscsi_target_find_or_create()
2443 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));  in cfiscsi_target_find_or_create()
2445 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));  in cfiscsi_target_find_or_create()
2446 	newct->ct_tag = tag;  in cfiscsi_target_find_or_create()
2447 	refcount_init(&newct->ct_refcount, 1);  in cfiscsi_target_find_or_create()
2448 	newct->ct_softc = softc;  in cfiscsi_target_find_or_create()
2449 	if (TAILQ_EMPTY(&softc->targets))  in cfiscsi_target_find_or_create()
2450 		softc->last_target_id = 0;  in cfiscsi_target_find_or_create()
2451 	newct->ct_target_id = ++softc->last_target_id;  in cfiscsi_target_find_or_create()
2452 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);  in cfiscsi_target_find_or_create()
2453 	mtx_unlock(&softc->lock);  in cfiscsi_target_find_or_create()
2462 	if (error != 0)  in cfiscsi_pdu_done()
2464 	((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);  in cfiscsi_pdu_done()
2470 	struct cfiscsi_session *cs;  in cfiscsi_datamove_in()  local
2482 	cs = PDU_SESSION(request);  in cfiscsi_datamove_in()
2484 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;  in cfiscsi_datamove_in()
2485 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_datamove_in()
2487 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));  in cfiscsi_datamove_in()
2489 	if (io->scsiio.kern_sg_entries > 0) {  in cfiscsi_datamove_in()
2490 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;  in cfiscsi_datamove_in()
2491 		ctl_sg_count = io->scsiio.kern_sg_entries;  in cfiscsi_datamove_in()
2494 		ctl_sglist->addr = io->scsiio.kern_data_ptr;  in cfiscsi_datamove_in()
2495 		ctl_sglist->len = io->scsiio.kern_data_len;  in cfiscsi_datamove_in()
2501 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones  in cfiscsi_datamove_in()
2504 	buffer_offset = io->scsiio.kern_rel_offset;  in cfiscsi_datamove_in()
2510 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);  in cfiscsi_datamove_in()
2513 	 * If the transfer is outside of expected length -- we are done.  in cfiscsi_datamove_in()
2516 #if 0  in cfiscsi_datamove_in()
2517 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "  in cfiscsi_datamove_in()
2524 	if (io->scsiio.kern_data_ref != NULL)  in cfiscsi_datamove_in()
2529 	i = 0;  in cfiscsi_datamove_in()
2531 	sg_len = 0;  in cfiscsi_datamove_in()
2534 	if (cs->cs_conn->ic_hw_isomax != 0)  in cfiscsi_datamove_in()
2535 		max_send_data_segment_length = cs->cs_conn->ic_hw_isomax;  in cfiscsi_datamove_in()
2538 		    cs->cs_conn->ic_max_send_data_segment_length;  in cfiscsi_datamove_in()
2543 				CFISCSI_SESSION_WARN(cs, "failed to "  in cfiscsi_datamove_in()
2545 				ctl_set_busy(&io->scsiio);  in cfiscsi_datamove_in()
2547 				cfiscsi_session_terminate(cs);  in cfiscsi_datamove_in()
2550 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;  in cfiscsi_datamove_in()
2551 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;  in cfiscsi_datamove_in()
2552 			bhsdi->bhsdi_initiator_task_tag =  in cfiscsi_datamove_in()
2553 			    bhssc->bhssc_initiator_task_tag;  in cfiscsi_datamove_in()
2554 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;  in cfiscsi_datamove_in()
2555 			bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io));  in cfiscsi_datamove_in()
2556 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);  in cfiscsi_datamove_in()
2560 		if (sg_len == 0) {  in cfiscsi_datamove_in()
2563 			KASSERT(sg_len > 0, ("sg_len <= 0"));  in cfiscsi_datamove_in()
2571 		KASSERT(response->ip_data_len < max_send_data_segment_length,  in cfiscsi_datamove_in()
2573 		    response->ip_data_len, max_send_data_segment_length));  in cfiscsi_datamove_in()
2574 		if (response->ip_data_len + len > max_send_data_segment_length) {  in cfiscsi_datamove_in()
2575 			len = max_send_data_segment_length - response->ip_data_len;  in cfiscsi_datamove_in()
2583 		KASSERT(buffer_offset + response->ip_data_len < expected_len,  in cfiscsi_datamove_in()
2585 		    buffer_offset, response->ip_data_len, expected_len));  in cfiscsi_datamove_in()
2586 		if (buffer_offset + response->ip_data_len + len > expected_len) {  in cfiscsi_datamove_in()
2587 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "  in cfiscsi_datamove_in()
2589 			    buffer_offset + response->ip_data_len + len, expected_len);  in cfiscsi_datamove_in()
2590 			len = expected_len - (buffer_offset + response->ip_data_len);  in cfiscsi_datamove_in()
2596 		    M_NOWAIT | (cb ? ICL_NOCOPY : 0));  in cfiscsi_datamove_in()
2597 		if (error != 0) {  in cfiscsi_datamove_in()
2598 			CFISCSI_SESSION_WARN(cs, "failed to "  in cfiscsi_datamove_in()
2601 			ctl_set_busy(&io->scsiio);  in cfiscsi_datamove_in()
2603 			cfiscsi_session_terminate(cs);  in cfiscsi_datamove_in()
2607 		sg_len -= len;  in cfiscsi_datamove_in()
2608 		io->scsiio.kern_data_resid -= len;  in cfiscsi_datamove_in()
2610 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,  in cfiscsi_datamove_in()
2612 		    buffer_offset, response->ip_data_len, expected_len));  in cfiscsi_datamove_in()
2613 		if (buffer_offset + response->ip_data_len == expected_len) {  in cfiscsi_datamove_in()
2620 		if (sg_len == 0) {  in cfiscsi_datamove_in()
2622 			 * End of scatter-gather segment;  in cfiscsi_datamove_in()
2625 			if (i == ctl_sg_count - 1) {  in cfiscsi_datamove_in()
2634 		if (response->ip_data_len == max_send_data_segment_length) {  in cfiscsi_datamove_in()
2638 			 * for kern_data_resid == 0 instead; there  in cfiscsi_datamove_in()
2639 			 * may be several Data-In PDUs for the final  in cfiscsi_datamove_in()
2643 			buffer_offset += response->ip_data_len;  in cfiscsi_datamove_in()
2644 			if (buffer_offset == io->scsiio.kern_total_len ||  in cfiscsi_datamove_in()
2646 				buffer_offset -= response->ip_data_len;  in cfiscsi_datamove_in()
2649 			PRIV_EXPDATASN(io) += howmany(response->ip_data_len,  in cfiscsi_datamove_in()
2650 			    cs->cs_conn->ic_max_send_data_segment_length);  in cfiscsi_datamove_in()
2652 				response->ip_prv0 = io->scsiio.kern_data_ref;  in cfiscsi_datamove_in()
2653 				response->ip_prv1 = io->scsiio.kern_data_arg;  in cfiscsi_datamove_in()
2654 				io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);  in cfiscsi_datamove_in()
2662 		buffer_offset += response->ip_data_len;  in cfiscsi_datamove_in()
2663 		if (buffer_offset == io->scsiio.kern_total_len ||  in cfiscsi_datamove_in()
2665 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;  in cfiscsi_datamove_in()
2666 			if (io->io_hdr.status == CTL_SUCCESS) {  in cfiscsi_datamove_in()
2667 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;  in cfiscsi_datamove_in()
2668 				if (io->scsiio.kern_total_len <  in cfiscsi_datamove_in()
2669 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {  in cfiscsi_datamove_in()
2670 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;  in cfiscsi_datamove_in()
2671 					bhsdi->bhsdi_residual_count =  in cfiscsi_datamove_in()
2672 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -  in cfiscsi_datamove_in()
2673 					    io->scsiio.kern_total_len);  in cfiscsi_datamove_in()
2674 				} else if (io->scsiio.kern_total_len >  in cfiscsi_datamove_in()
2675 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {  in cfiscsi_datamove_in()
2676 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;  in cfiscsi_datamove_in()
2677 					bhsdi->bhsdi_residual_count =  in cfiscsi_datamove_in()
2678 					    htonl(io->scsiio.kern_total_len -  in cfiscsi_datamove_in()
2679 					    ntohl(bhssc->bhssc_expected_data_transfer_length));  in cfiscsi_datamove_in()
2681 				bhsdi->bhsdi_status = io->scsiio.scsi_status;  in cfiscsi_datamove_in()
2682 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;  in cfiscsi_datamove_in()
2685 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));  in cfiscsi_datamove_in()
2686 		PRIV_EXPDATASN(io) += howmany(response->ip_data_len,  in cfiscsi_datamove_in()
2687 		    cs->cs_conn->ic_max_send_data_segment_length);  in cfiscsi_datamove_in()
2689 			response->ip_prv0 = io->scsiio.kern_data_ref;  in cfiscsi_datamove_in()
2690 			response->ip_prv1 = io->scsiio.kern_data_arg;  in cfiscsi_datamove_in()
2691 			io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);  in cfiscsi_datamove_in()
2702 	struct cfiscsi_session *cs;  in cfiscsi_datamove_out()  local
2713 	cs = PDU_SESSION(request);  in cfiscsi_datamove_out()
2715 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;  in cfiscsi_datamove_out()
2716 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_datamove_out()
2718 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));  in cfiscsi_datamove_out()
2723 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);  in cfiscsi_datamove_out()
2724 	if (io->scsiio.kern_rel_offset >= expected_len) {  in cfiscsi_datamove_out()
2729 	datamove_len = MIN(io->scsiio.kern_data_len,  in cfiscsi_datamove_out()
2730 	    expected_len - io->scsiio.kern_rel_offset);  in cfiscsi_datamove_out()
2733 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);  in cfiscsi_datamove_out()
2734 	if (target_transfer_tag == 0xffffffff) {  in cfiscsi_datamove_out()
2736 		    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);  in cfiscsi_datamove_out()
2738 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,  in cfiscsi_datamove_out()
2741 		CFISCSI_SESSION_WARN(cs, "failed to "  in cfiscsi_datamove_out()
2743 		ctl_set_busy(&io->scsiio);  in cfiscsi_datamove_out()
2745 		cfiscsi_session_terminate(cs);  in cfiscsi_datamove_out()
2748 #if 0  in cfiscsi_datamove_out()
2749 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "  in cfiscsi_datamove_out()
2750 	    "task tag 0x%x, target transfer tag 0x%x",  in cfiscsi_datamove_out()
2751 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);  in cfiscsi_datamove_out()
2754 	cdw->cdw_ctl_io = io;  in cfiscsi_datamove_out()
2755 	cdw->cdw_target_transfer_tag = target_transfer_tag;  in cfiscsi_datamove_out()
2756 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;  in cfiscsi_datamove_out()
2757 	cdw->cdw_r2t_end = datamove_len;  in cfiscsi_datamove_out()
2758 	cdw->cdw_datasn = 0;  in cfiscsi_datamove_out()
2761 	if (io->scsiio.kern_sg_entries > 0) {  in cfiscsi_datamove_out()
2762 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;  in cfiscsi_datamove_out()
2765 		ctl_sglist->addr = io->scsiio.kern_data_ptr;  in cfiscsi_datamove_out()
2766 		ctl_sglist->len = datamove_len;  in cfiscsi_datamove_out()
2768 	cdw->cdw_sg_index = 0;  in cfiscsi_datamove_out()
2769 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;  in cfiscsi_datamove_out()
2770 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;  in cfiscsi_datamove_out()
2771 	r2t_off = io->scsiio.ext_data_filled;  in cfiscsi_datamove_out()
2772 	while (r2t_off > 0) {  in cfiscsi_datamove_out()
2773 		if (r2t_off >= cdw->cdw_sg_len) {  in cfiscsi_datamove_out()
2774 			r2t_off -= cdw->cdw_sg_len;  in cfiscsi_datamove_out()
2775 			cdw->cdw_sg_index++;  in cfiscsi_datamove_out()
2776 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;  in cfiscsi_datamove_out()
2777 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;  in cfiscsi_datamove_out()
2780 		cdw->cdw_sg_addr += r2t_off;  in cfiscsi_datamove_out()
2781 		cdw->cdw_sg_len -= r2t_off;  in cfiscsi_datamove_out()
2782 		r2t_off = 0;  in cfiscsi_datamove_out()
2785 	if (cs->cs_immediate_data &&  in cfiscsi_datamove_out()
2786 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <  in cfiscsi_datamove_out()
2790 			cfiscsi_data_wait_free(cs, cdw);  in cfiscsi_datamove_out()
2796 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;  in cfiscsi_datamove_out()
2797 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,  in cfiscsi_datamove_out()
2798 	    cs->cs_max_burst_length);  in cfiscsi_datamove_out()
2799 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;  in cfiscsi_datamove_out()
2801 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_datamove_out()
2802 	if (cs->cs_terminating_tasks) {  in cfiscsi_datamove_out()
2803 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_datamove_out()
2804 		KASSERT((io->io_hdr.flags & CTL_FLAG_ABORT) != 0,  in cfiscsi_datamove_out()
2806 		    __func__, io, cs));  in cfiscsi_datamove_out()
2807 		CFISCSI_SESSION_WARN(cs, "aborting data_wait for aborted I/O");  in cfiscsi_datamove_out()
2808 		cfiscsi_data_wait_abort(cs, cdw, 44);  in cfiscsi_datamove_out()
2811 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);  in cfiscsi_datamove_out()
2812 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_datamove_out()
2820 		CFISCSI_SESSION_WARN(cs, "failed to "  in cfiscsi_datamove_out()
2822 		ctl_set_busy(&io->scsiio);  in cfiscsi_datamove_out()
2824 		cfiscsi_session_terminate(cs);  in cfiscsi_datamove_out()
2827 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;  in cfiscsi_datamove_out()
2828 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;  in cfiscsi_datamove_out()
2829 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;  in cfiscsi_datamove_out()
2830 	bhsr2t->bhsr2t_flags = 0x80;  in cfiscsi_datamove_out()
2831 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;  in cfiscsi_datamove_out()
2832 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;  in cfiscsi_datamove_out()
2833 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;  in cfiscsi_datamove_out()
2839 	bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);  in cfiscsi_datamove_out()
2842 	 * i.e. for the first call of datamove(), it will be 0,  in cfiscsi_datamove_out()
2849 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);  in cfiscsi_datamove_out()
2855 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);  in cfiscsi_datamove_out()
2863 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)  in cfiscsi_datamove()
2867 		io->scsiio.ext_data_filled = 0;  in cfiscsi_datamove()
2880 	struct cfiscsi_session *cs;  in cfiscsi_scsi_command_done()  local
2885 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;  in cfiscsi_scsi_command_done()
2886 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_scsi_command_done()
2888 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));  in cfiscsi_scsi_command_done()
2890 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",  in cfiscsi_scsi_command_done()
2891 	//    bhssc->bhssc_initiator_task_tag);  in cfiscsi_scsi_command_done()
2894 	cs = PDU_SESSION(request);  in cfiscsi_scsi_command_done()
2895 	CFISCSI_SESSION_LOCK(cs);  in cfiscsi_scsi_command_done()
2896 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)  in cfiscsi_scsi_command_done()
2897 		KASSERT(bhssc->bhssc_initiator_task_tag !=  in cfiscsi_scsi_command_done()
2898 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));  in cfiscsi_scsi_command_done()
2899 	CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_scsi_command_done()
2906 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&  in cfiscsi_scsi_command_done()
2907 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||  in cfiscsi_scsi_command_done()
2908 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {  in cfiscsi_scsi_command_done()
2915 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;  in cfiscsi_scsi_command_done()
2916 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;  in cfiscsi_scsi_command_done()
2917 	bhssr->bhssr_flags = 0x80;  in cfiscsi_scsi_command_done()
2922 	if (io->scsiio.kern_total_len <  in cfiscsi_scsi_command_done()
2923 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {  in cfiscsi_scsi_command_done()
2924 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;  in cfiscsi_scsi_command_done()
2925 		bhssr->bhssr_residual_count =  in cfiscsi_scsi_command_done()
2926 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -  in cfiscsi_scsi_command_done()
2927 		    io->scsiio.kern_total_len);  in cfiscsi_scsi_command_done()
2928 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",  in cfiscsi_scsi_command_done()
2929 		//    ntohl(bhssr->bhssr_residual_count));  in cfiscsi_scsi_command_done()
2930 	} else if (io->scsiio.kern_total_len >  in cfiscsi_scsi_command_done()
2931 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {  in cfiscsi_scsi_command_done()
2932 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;  in cfiscsi_scsi_command_done()
2933 		bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -  in cfiscsi_scsi_command_done()
2934 		    ntohl(bhssc->bhssc_expected_data_transfer_length));  in cfiscsi_scsi_command_done()
2935 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",  in cfiscsi_scsi_command_done()
2936 		//    ntohl(bhssr->bhssr_residual_count));  in cfiscsi_scsi_command_done()
2938 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;  in cfiscsi_scsi_command_done()
2939 	bhssr->bhssr_status = io->scsiio.scsi_status;  in cfiscsi_scsi_command_done()
2940 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;  in cfiscsi_scsi_command_done()
2941 	bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));  in cfiscsi_scsi_command_done()
2943 	if (io->scsiio.sense_len > 0) {  in cfiscsi_scsi_command_done()
2944 #if 0  in cfiscsi_scsi_command_done()
2945 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",  in cfiscsi_scsi_command_done()
2946 		    io->scsiio.sense_len);  in cfiscsi_scsi_command_done()
2948 		sense_length = htons(io->scsiio.sense_len);  in cfiscsi_scsi_command_done()
2952 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);  in cfiscsi_scsi_command_done()
2967 	struct cfiscsi_session *cs, *tcs;  in cfiscsi_task_management_done()  local
2969 	int cold_reset = 0;  in cfiscsi_task_management_done()
2972 	cs = PDU_SESSION(request);  in cfiscsi_task_management_done()
2973 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;  in cfiscsi_task_management_done()
2974 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==  in cfiscsi_task_management_done()
2976 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));  in cfiscsi_task_management_done()
2978 #if 0  in cfiscsi_task_management_done()
2979 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",  in cfiscsi_task_management_done()
2980 	    bhstmr->bhstmr_initiator_task_tag,  in cfiscsi_task_management_done()
2981 	    bhstmr->bhstmr_referenced_task_tag);  in cfiscsi_task_management_done()
2984 	if ((bhstmr->bhstmr_function & ~0x80) ==  in cfiscsi_task_management_done()
2987 		 * Make sure we no longer wait for Data-Out for this command.  in cfiscsi_task_management_done()
2989 		CFISCSI_SESSION_LOCK(cs);  in cfiscsi_task_management_done()
2991 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {  in cfiscsi_task_management_done()
2992 			if (bhstmr->bhstmr_referenced_task_tag !=  in cfiscsi_task_management_done()
2993 			    cdw->cdw_initiator_task_tag)  in cfiscsi_task_management_done()
2996 #if 0  in cfiscsi_task_management_done()
2997 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "  in cfiscsi_task_management_done()
2998 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);  in cfiscsi_task_management_done()
3000 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,  in cfiscsi_task_management_done()
3002 			cfiscsi_data_wait_abort(cs, cdw, 43);  in cfiscsi_task_management_done()
3004 		CFISCSI_SESSION_UNLOCK(cs);  in cfiscsi_task_management_done()
3006 	if ((bhstmr->bhstmr_function & ~0x80) ==  in cfiscsi_task_management_done()
3008 	    io->io_hdr.status == CTL_SUCCESS)  in cfiscsi_task_management_done()
3013 	    response->ip_bhs;  in cfiscsi_task_management_done()
3014 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;  in cfiscsi_task_management_done()
3015 	bhstmr2->bhstmr_flags = 0x80;  in cfiscsi_task_management_done()
3016 	switch (io->taskio.task_status) {  in cfiscsi_task_management_done()
3018 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;  in cfiscsi_task_management_done()
3021 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;  in cfiscsi_task_management_done()
3024 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;  in cfiscsi_task_management_done()
3028 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;  in cfiscsi_task_management_done()
3031 	memcpy(bhstmr2->bhstmr_additional_reponse_information,  in cfiscsi_task_management_done()
3032 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));  in cfiscsi_task_management_done()
3033 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;  in cfiscsi_task_management_done()
3040 		softc = cs->cs_target->ct_softc;  in cfiscsi_task_management_done()
3041 		mtx_lock(&softc->lock);  in cfiscsi_task_management_done()
3042 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {  in cfiscsi_task_management_done()
3043 			if (tcs->cs_target == cs->cs_target)  in cfiscsi_task_management_done()
3046 		mtx_unlock(&softc->lock);  in cfiscsi_task_management_done()
3054 	struct cfiscsi_session *cs;  in cfiscsi_done()  local
3056 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),  in cfiscsi_done()
3057 		("invalid CTL status %#x", io->io_hdr.status));  in cfiscsi_done()
3060 	cs = PDU_SESSION(request);  in cfiscsi_done()
3062 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {  in cfiscsi_done()
3074 		cs->cs_tasks_aborted = true;  in cfiscsi_done()
3075 		refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_done()
3076 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));  in cfiscsi_done()
3080 		panic("cfiscsi_done called with wrong opcode 0x%x",  in cfiscsi_done()
3081 		    request->ip_bhs->bhs_opcode);  in cfiscsi_done()
3084 	refcount_release(&cs->cs_outstanding_ctl_pdus);  in cfiscsi_done()