14814a0a4SEdward Tomasz Napierala /*- 24814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 34814a0a4SEdward Tomasz Napierala * 44814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 54814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 64814a0a4SEdward Tomasz Napierala * are met: 74814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 84814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 94814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 104814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 114814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 124814a0a4SEdward Tomasz Napierala * 134814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 144814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 154814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 164814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 174814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 184814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 194814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 204814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 214814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 224814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 234814a0a4SEdward Tomasz Napierala * SUCH DAMAGE. 244814a0a4SEdward Tomasz Napierala */ 254814a0a4SEdward Tomasz Napierala 264814a0a4SEdward Tomasz Napierala #include "icl_iser.h" 274814a0a4SEdward Tomasz Napierala 284814a0a4SEdward Tomasz Napierala static MALLOC_DEFINE(M_ISER_VERBS, "iser_verbs", "iser verbs backend"); 294814a0a4SEdward Tomasz Napierala static int iser_cq_poll_limit = 512; 304814a0a4SEdward Tomasz Napierala 314814a0a4SEdward Tomasz Napierala static void 324814a0a4SEdward Tomasz Napierala iser_cq_event_callback(struct ib_event *cause, void *context) 334814a0a4SEdward Tomasz Napierala { 344814a0a4SEdward Tomasz Napierala ISER_ERR("got cq event %d", cause->event); 354814a0a4SEdward Tomasz Napierala } 364814a0a4SEdward Tomasz Napierala 374814a0a4SEdward Tomasz Napierala static void 384814a0a4SEdward Tomasz Napierala iser_qp_event_callback(struct ib_event *cause, void *context) 394814a0a4SEdward Tomasz Napierala { 404814a0a4SEdward Tomasz Napierala ISER_ERR("got qp event %d", cause->event); 414814a0a4SEdward Tomasz Napierala } 424814a0a4SEdward Tomasz Napierala 434814a0a4SEdward Tomasz Napierala static void 444814a0a4SEdward Tomasz Napierala iser_event_handler(struct ib_event_handler *handler, 454814a0a4SEdward Tomasz Napierala struct ib_event *event) 464814a0a4SEdward Tomasz Napierala { 474814a0a4SEdward Tomasz Napierala ISER_ERR("async event %d on device %s port %d", 484814a0a4SEdward Tomasz Napierala event->event, event->device->name, 494814a0a4SEdward Tomasz Napierala event->element.port_num); 504814a0a4SEdward Tomasz Napierala } 514814a0a4SEdward Tomasz Napierala 524814a0a4SEdward Tomasz Napierala /** 534814a0a4SEdward Tomasz Napierala * is_iser_tx_desc - Indicate if the completion wr_id 544814a0a4SEdward Tomasz Napierala * is a TX descriptor or not. 554814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection 564814a0a4SEdward Tomasz Napierala * @wr_id: completion WR identifier 574814a0a4SEdward Tomasz Napierala * 584814a0a4SEdward Tomasz Napierala * Since we cannot rely on wc opcode in FLUSH errors 594814a0a4SEdward Tomasz Napierala * we must work around it by checking if the wr_id address 604814a0a4SEdward Tomasz Napierala * falls in the iser connection rx_descs buffer. If so 614814a0a4SEdward Tomasz Napierala * it is an RX descriptor, otherwize it is a TX. 624814a0a4SEdward Tomasz Napierala */ 634814a0a4SEdward Tomasz Napierala static inline bool 644814a0a4SEdward Tomasz Napierala is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id) 654814a0a4SEdward Tomasz Napierala { 664814a0a4SEdward Tomasz Napierala void *start = iser_conn->rx_descs; 674814a0a4SEdward Tomasz Napierala u64 len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs); 684814a0a4SEdward Tomasz Napierala void *end = (void *)((uintptr_t)start + (uintptr_t)len); 694814a0a4SEdward Tomasz Napierala 704814a0a4SEdward Tomasz Napierala if (start) { 714814a0a4SEdward Tomasz Napierala if (wr_id >= start && wr_id < end) 724814a0a4SEdward Tomasz Napierala return false; 734814a0a4SEdward Tomasz Napierala } else { 744814a0a4SEdward Tomasz Napierala return ((uintptr_t)wr_id != (uintptr_t)iser_conn->login_resp_buf); 754814a0a4SEdward Tomasz Napierala } 764814a0a4SEdward Tomasz Napierala 774814a0a4SEdward Tomasz Napierala return true; 784814a0a4SEdward Tomasz Napierala } 794814a0a4SEdward Tomasz Napierala 804814a0a4SEdward Tomasz Napierala /** 814814a0a4SEdward Tomasz Napierala * iser_handle_comp_error() - Handle error completion 824814a0a4SEdward Tomasz Napierala * @ib_conn: connection RDMA resources 834814a0a4SEdward Tomasz Napierala * @wc: work completion 844814a0a4SEdward Tomasz Napierala * 854814a0a4SEdward Tomasz Napierala * Notes: Update post_recv_buf_count in case of recv error completion. 864814a0a4SEdward Tomasz Napierala * For non-FLUSH error completion we should also notify iscsi layer that 874814a0a4SEdward Tomasz Napierala * connection is failed (in case we passed bind stage). 884814a0a4SEdward Tomasz Napierala */ 894814a0a4SEdward Tomasz Napierala static void 904814a0a4SEdward Tomasz Napierala iser_handle_comp_error(struct ib_conn *ib_conn, 914814a0a4SEdward Tomasz Napierala struct ib_wc *wc) 924814a0a4SEdward Tomasz Napierala { 934814a0a4SEdward Tomasz Napierala void *wr_id = (void *)(uintptr_t)wc->wr_id; 944814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 954814a0a4SEdward Tomasz Napierala ib_conn); 964814a0a4SEdward Tomasz Napierala 974814a0a4SEdward Tomasz Napierala if (is_iser_tx_desc(iser_conn, wr_id)) { 984814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got send comp error", iser_conn); 994814a0a4SEdward Tomasz Napierala } else { 1004814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got recv comp error", iser_conn); 1014814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 1024814a0a4SEdward Tomasz Napierala } 1034814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) 1044814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 1054814a0a4SEdward Tomasz Napierala } 1064814a0a4SEdward Tomasz Napierala 1074814a0a4SEdward Tomasz Napierala /** 1084814a0a4SEdward Tomasz Napierala * iser_handle_wc - handle a single work completion 1094814a0a4SEdward Tomasz Napierala * @wc: work completion 1104814a0a4SEdward Tomasz Napierala * 1114814a0a4SEdward Tomasz Napierala * Soft-IRQ context, work completion can be either 1124814a0a4SEdward Tomasz Napierala * SEND or RECV, and can turn out successful or 1134814a0a4SEdward Tomasz Napierala * with error (or flush error). 1144814a0a4SEdward Tomasz Napierala */ 1154814a0a4SEdward Tomasz Napierala static void iser_handle_wc(struct ib_wc *wc) 1164814a0a4SEdward Tomasz Napierala { 1174814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 1184814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc; 1194814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 1204814a0a4SEdward Tomasz Napierala 1214814a0a4SEdward Tomasz Napierala ib_conn = wc->qp->qp_context; 1224814a0a4SEdward Tomasz Napierala if (likely(wc->status == IB_WC_SUCCESS)) { 1234814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_RECV) { 1244814a0a4SEdward Tomasz Napierala rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 1254814a0a4SEdward Tomasz Napierala iser_rcv_completion(rx_desc, wc->byte_len, 1264814a0a4SEdward Tomasz Napierala ib_conn); 1274814a0a4SEdward Tomasz Napierala } else 1284814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_SEND) { 1294814a0a4SEdward Tomasz Napierala tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 1304814a0a4SEdward Tomasz Napierala iser_snd_completion(tx_desc, ib_conn); 1314814a0a4SEdward Tomasz Napierala } else { 1324814a0a4SEdward Tomasz Napierala ISER_ERR("Unknown wc opcode %d", wc->opcode); 1334814a0a4SEdward Tomasz Napierala } 1344814a0a4SEdward Tomasz Napierala } else { 1354814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 1364814a0a4SEdward Tomasz Napierala ib_conn); 1374814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) { 138e199d806SEdward Tomasz Napierala ISER_ERR("conn %p wr id %llx status %d vend_err %x", 139e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id, 140e199d806SEdward Tomasz Napierala wc->status, wc->vendor_err); 1414814a0a4SEdward Tomasz Napierala } else { 142e199d806SEdward Tomasz Napierala ISER_DBG("flush error: conn %p wr id %llx", 143e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id); 1444814a0a4SEdward Tomasz Napierala } 1454814a0a4SEdward Tomasz Napierala 1464814a0a4SEdward Tomasz Napierala if (wc->wr_id == ISER_BEACON_WRID) { 1474814a0a4SEdward Tomasz Napierala /* all flush errors were consumed */ 1484814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 1494814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got ISER_BEACON_WRID", iser_conn); 1504814a0a4SEdward Tomasz Napierala cv_signal(&ib_conn->beacon.flush_cv); 1514814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 1524814a0a4SEdward Tomasz Napierala } else { 1534814a0a4SEdward Tomasz Napierala iser_handle_comp_error(ib_conn, wc); 1544814a0a4SEdward Tomasz Napierala } 1554814a0a4SEdward Tomasz Napierala } 1564814a0a4SEdward Tomasz Napierala } 1574814a0a4SEdward Tomasz Napierala 1584814a0a4SEdward Tomasz Napierala static void 1594814a0a4SEdward Tomasz Napierala iser_cq_tasklet_fn(void *data, int pending) 1604814a0a4SEdward Tomasz Napierala { 1614814a0a4SEdward Tomasz Napierala struct iser_comp *comp = (struct iser_comp *)data; 1624814a0a4SEdward Tomasz Napierala struct ib_cq *cq = comp->cq; 1634814a0a4SEdward Tomasz Napierala struct ib_wc *const wcs = comp->wcs; 1644814a0a4SEdward Tomasz Napierala int completed = 0; 1654814a0a4SEdward Tomasz Napierala int i; 1664814a0a4SEdward Tomasz Napierala int n; 1674814a0a4SEdward Tomasz Napierala 1684814a0a4SEdward Tomasz Napierala while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 1694814a0a4SEdward Tomasz Napierala for (i = 0; i < n; i++) 1704814a0a4SEdward Tomasz Napierala iser_handle_wc(&wcs[i]); 1714814a0a4SEdward Tomasz Napierala 1724814a0a4SEdward Tomasz Napierala completed += n; 1734814a0a4SEdward Tomasz Napierala if (completed >= iser_cq_poll_limit) 1744814a0a4SEdward Tomasz Napierala break; 1754814a0a4SEdward Tomasz Napierala } 1764814a0a4SEdward Tomasz Napierala 1774814a0a4SEdward Tomasz Napierala /* 1784814a0a4SEdward Tomasz Napierala * It is assumed here that arming CQ only once its empty 1794814a0a4SEdward Tomasz Napierala * would not cause interrupts to be missed. 1804814a0a4SEdward Tomasz Napierala */ 1814814a0a4SEdward Tomasz Napierala ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 1824814a0a4SEdward Tomasz Napierala } 1834814a0a4SEdward Tomasz Napierala 1844814a0a4SEdward Tomasz Napierala static void 1854814a0a4SEdward Tomasz Napierala iser_cq_callback(struct ib_cq *cq, void *cq_context) 1864814a0a4SEdward Tomasz Napierala { 1874814a0a4SEdward Tomasz Napierala struct iser_comp *comp = cq_context; 1884814a0a4SEdward Tomasz Napierala 189f83f8891SEdward Tomasz Napierala taskqueue_enqueue(comp->tq, &comp->task); 1904814a0a4SEdward Tomasz Napierala } 1914814a0a4SEdward Tomasz Napierala 1924814a0a4SEdward Tomasz Napierala /** 1934814a0a4SEdward Tomasz Napierala * iser_create_device_ib_res - creates Protection Domain (PD), Completion 1944814a0a4SEdward Tomasz Napierala * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with 1954814a0a4SEdward Tomasz Napierala * the adapator. 1964814a0a4SEdward Tomasz Napierala * 1974814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 1984814a0a4SEdward Tomasz Napierala */ 1994814a0a4SEdward Tomasz Napierala static int 2004814a0a4SEdward Tomasz Napierala iser_create_device_ib_res(struct iser_device *device) 2014814a0a4SEdward Tomasz Napierala { 20241dbd9ddSHans Petter Selasky struct ib_device *ib_dev = device->ib_device; 20341dbd9ddSHans Petter Selasky int i, max_cqe; 2044814a0a4SEdward Tomasz Napierala 20541dbd9ddSHans Petter Selasky if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { 2064814a0a4SEdward Tomasz Napierala ISER_ERR("device %s doesn't support Fastreg, " 2074814a0a4SEdward Tomasz Napierala "can't register memory", device->ib_device->name); 2084814a0a4SEdward Tomasz Napierala return (1); 2094814a0a4SEdward Tomasz Napierala } 2104814a0a4SEdward Tomasz Napierala 2114814a0a4SEdward Tomasz Napierala device->comps_used = min(mp_ncpus, device->ib_device->num_comp_vectors); 2124814a0a4SEdward Tomasz Napierala 2134814a0a4SEdward Tomasz Napierala device->comps = malloc(device->comps_used * sizeof(*device->comps), 2144814a0a4SEdward Tomasz Napierala M_ISER_VERBS, M_WAITOK | M_ZERO); 2154814a0a4SEdward Tomasz Napierala 21641dbd9ddSHans Petter Selasky max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe); 2174814a0a4SEdward Tomasz Napierala 2184814a0a4SEdward Tomasz Napierala ISER_DBG("using %d CQs, device %s supports %d vectors max_cqe %d", 2194814a0a4SEdward Tomasz Napierala device->comps_used, device->ib_device->name, 2204814a0a4SEdward Tomasz Napierala device->ib_device->num_comp_vectors, max_cqe); 2214814a0a4SEdward Tomasz Napierala 22241dbd9ddSHans Petter Selasky device->pd = ib_alloc_pd(device->ib_device, IB_PD_UNSAFE_GLOBAL_RKEY); 2234814a0a4SEdward Tomasz Napierala if (IS_ERR(device->pd)) 2244814a0a4SEdward Tomasz Napierala goto pd_err; 2254814a0a4SEdward Tomasz Napierala 2264814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2274814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 22841dbd9ddSHans Petter Selasky struct ib_cq_init_attr cq_attr = { 22941dbd9ddSHans Petter Selasky .cqe = max_cqe, 23041dbd9ddSHans Petter Selasky .comp_vector = i, 23141dbd9ddSHans Petter Selasky }; 2324814a0a4SEdward Tomasz Napierala 2334814a0a4SEdward Tomasz Napierala comp->device = device; 2344814a0a4SEdward Tomasz Napierala comp->cq = ib_create_cq(device->ib_device, 2354814a0a4SEdward Tomasz Napierala iser_cq_callback, 2364814a0a4SEdward Tomasz Napierala iser_cq_event_callback, 2374814a0a4SEdward Tomasz Napierala (void *)comp, 23841dbd9ddSHans Petter Selasky &cq_attr); 2394814a0a4SEdward Tomasz Napierala if (IS_ERR(comp->cq)) { 2404814a0a4SEdward Tomasz Napierala comp->cq = NULL; 2414814a0a4SEdward Tomasz Napierala goto cq_err; 2424814a0a4SEdward Tomasz Napierala } 2434814a0a4SEdward Tomasz Napierala 2444814a0a4SEdward Tomasz Napierala if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP)) 2454814a0a4SEdward Tomasz Napierala goto cq_err; 2464814a0a4SEdward Tomasz Napierala 2474814a0a4SEdward Tomasz Napierala TASK_INIT(&comp->task, 0, iser_cq_tasklet_fn, comp); 2484814a0a4SEdward Tomasz Napierala comp->tq = taskqueue_create_fast("iser_taskq", M_NOWAIT, 2494814a0a4SEdward Tomasz Napierala taskqueue_thread_enqueue, &comp->tq); 2504814a0a4SEdward Tomasz Napierala if (!comp->tq) 2514814a0a4SEdward Tomasz Napierala goto tq_err; 2524814a0a4SEdward Tomasz Napierala taskqueue_start_threads(&comp->tq, 1, PI_NET, "iser taskq"); 2534814a0a4SEdward Tomasz Napierala } 2544814a0a4SEdward Tomasz Napierala 25541dbd9ddSHans Petter Selasky device->mr = device->pd->__internal_mr; 2564814a0a4SEdward Tomasz Napierala if (IS_ERR(device->mr)) 2574814a0a4SEdward Tomasz Napierala goto tq_err; 2584814a0a4SEdward Tomasz Napierala 2594814a0a4SEdward Tomasz Napierala INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device, 2604814a0a4SEdward Tomasz Napierala iser_event_handler); 2614814a0a4SEdward Tomasz Napierala if (ib_register_event_handler(&device->event_handler)) 2628d73c9baSHans Petter Selasky goto tq_err; 2634814a0a4SEdward Tomasz Napierala 2644814a0a4SEdward Tomasz Napierala return (0); 2654814a0a4SEdward Tomasz Napierala 2664814a0a4SEdward Tomasz Napierala tq_err: 2674814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2684814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 2694814a0a4SEdward Tomasz Napierala if (comp->tq) 2704814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 2714814a0a4SEdward Tomasz Napierala } 2724814a0a4SEdward Tomasz Napierala cq_err: 2734814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2744814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 2754814a0a4SEdward Tomasz Napierala if (comp->cq) 2764814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 2774814a0a4SEdward Tomasz Napierala } 2784814a0a4SEdward Tomasz Napierala ib_dealloc_pd(device->pd); 2794814a0a4SEdward Tomasz Napierala pd_err: 2804814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 2814814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate an IB resource"); 2824814a0a4SEdward Tomasz Napierala return (1); 2834814a0a4SEdward Tomasz Napierala } 2844814a0a4SEdward Tomasz Napierala 2854814a0a4SEdward Tomasz Napierala /** 2864814a0a4SEdward Tomasz Napierala * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR, 2874814a0a4SEdward Tomasz Napierala * CQ and PD created with the device associated with the adapator. 2884814a0a4SEdward Tomasz Napierala */ 2894814a0a4SEdward Tomasz Napierala static void 2904814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(struct iser_device *device) 2914814a0a4SEdward Tomasz Napierala { 2924814a0a4SEdward Tomasz Napierala int i; 2934814a0a4SEdward Tomasz Napierala 2944814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2954814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 2964814a0a4SEdward Tomasz Napierala 2974814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 2984814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 2994814a0a4SEdward Tomasz Napierala comp->cq = NULL; 3004814a0a4SEdward Tomasz Napierala } 3014814a0a4SEdward Tomasz Napierala 3024814a0a4SEdward Tomasz Napierala (void)ib_unregister_event_handler(&device->event_handler); 3034814a0a4SEdward Tomasz Napierala (void)ib_dealloc_pd(device->pd); 3044814a0a4SEdward Tomasz Napierala 3054814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 3064814a0a4SEdward Tomasz Napierala device->comps = NULL; 3074814a0a4SEdward Tomasz Napierala 3084814a0a4SEdward Tomasz Napierala device->mr = NULL; 3094814a0a4SEdward Tomasz Napierala device->pd = NULL; 3104814a0a4SEdward Tomasz Napierala } 3114814a0a4SEdward Tomasz Napierala 3124814a0a4SEdward Tomasz Napierala static int 3134814a0a4SEdward Tomasz Napierala iser_alloc_reg_res(struct ib_device *ib_device, 3144814a0a4SEdward Tomasz Napierala struct ib_pd *pd, 3154814a0a4SEdward Tomasz Napierala struct iser_reg_resources *res) 3164814a0a4SEdward Tomasz Napierala { 3174814a0a4SEdward Tomasz Napierala int ret; 3184814a0a4SEdward Tomasz Napierala 31941dbd9ddSHans Petter Selasky res->mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, ISCSI_ISER_SG_TABLESIZE + 1); 3204814a0a4SEdward Tomasz Napierala if (IS_ERR(res->mr)) { 3214814a0a4SEdward Tomasz Napierala ret = -PTR_ERR(res->mr); 3224814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate fast reg mr err=%d", ret); 32341dbd9ddSHans Petter Selasky return (ret); 3244814a0a4SEdward Tomasz Napierala } 3254814a0a4SEdward Tomasz Napierala res->mr_valid = 1; 3264814a0a4SEdward Tomasz Napierala 3274814a0a4SEdward Tomasz Napierala return (0); 3284814a0a4SEdward Tomasz Napierala } 3294814a0a4SEdward Tomasz Napierala 3304814a0a4SEdward Tomasz Napierala static void 3314814a0a4SEdward Tomasz Napierala iser_free_reg_res(struct iser_reg_resources *rsc) 3324814a0a4SEdward Tomasz Napierala { 3334814a0a4SEdward Tomasz Napierala ib_dereg_mr(rsc->mr); 3344814a0a4SEdward Tomasz Napierala } 3354814a0a4SEdward Tomasz Napierala 3364814a0a4SEdward Tomasz Napierala static struct fast_reg_descriptor * 3374814a0a4SEdward Tomasz Napierala iser_create_fastreg_desc(struct ib_device *ib_device, struct ib_pd *pd) 3384814a0a4SEdward Tomasz Napierala { 3394814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 3404814a0a4SEdward Tomasz Napierala int ret; 3414814a0a4SEdward Tomasz Napierala 3424814a0a4SEdward Tomasz Napierala desc = malloc(sizeof(*desc), M_ISER_VERBS, M_WAITOK | M_ZERO); 3434814a0a4SEdward Tomasz Napierala ret = iser_alloc_reg_res(ib_device, pd, &desc->rsc); 3444814a0a4SEdward Tomasz Napierala if (ret) { 3454814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate reg_resources"); 3464814a0a4SEdward Tomasz Napierala goto err; 3474814a0a4SEdward Tomasz Napierala } 3484814a0a4SEdward Tomasz Napierala 3494814a0a4SEdward Tomasz Napierala return (desc); 3504814a0a4SEdward Tomasz Napierala err: 3514814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 3524814a0a4SEdward Tomasz Napierala return (NULL); 3534814a0a4SEdward Tomasz Napierala } 3544814a0a4SEdward Tomasz Napierala 3554814a0a4SEdward Tomasz Napierala /** 3564814a0a4SEdward Tomasz Napierala * iser_create_fmr_pool - Creates FMR pool and page_vector 3574814a0a4SEdward Tomasz Napierala * 3584814a0a4SEdward Tomasz Napierala * returns 0 on success, or errno code on failure 3594814a0a4SEdward Tomasz Napierala */ 3604814a0a4SEdward Tomasz Napierala int 3614814a0a4SEdward Tomasz Napierala iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max) 3624814a0a4SEdward Tomasz Napierala { 3634814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 3644814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 3654814a0a4SEdward Tomasz Napierala int i; 3664814a0a4SEdward Tomasz Napierala 3674814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ib_conn->fastreg.pool); 3684814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size = 0; 3694814a0a4SEdward Tomasz Napierala for (i = 0; i < cmds_max; i++) { 3704814a0a4SEdward Tomasz Napierala desc = iser_create_fastreg_desc(device->ib_device, device->pd); 3714814a0a4SEdward Tomasz Napierala if (!desc) { 3724814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to create fastreg descriptor"); 3734814a0a4SEdward Tomasz Napierala goto err; 3744814a0a4SEdward Tomasz Napierala } 3754814a0a4SEdward Tomasz Napierala 3764814a0a4SEdward Tomasz Napierala list_add_tail(&desc->list, &ib_conn->fastreg.pool); 3774814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size++; 3784814a0a4SEdward Tomasz Napierala } 3794814a0a4SEdward Tomasz Napierala 3804814a0a4SEdward Tomasz Napierala return (0); 3814814a0a4SEdward Tomasz Napierala 3824814a0a4SEdward Tomasz Napierala err: 3834814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(ib_conn); 3844814a0a4SEdward Tomasz Napierala return (ENOMEM); 3854814a0a4SEdward Tomasz Napierala } 3864814a0a4SEdward Tomasz Napierala 3874814a0a4SEdward Tomasz Napierala /** 3884814a0a4SEdward Tomasz Napierala * iser_free_fmr_pool - releases the FMR pool and page vec 3894814a0a4SEdward Tomasz Napierala */ 3904814a0a4SEdward Tomasz Napierala void 3914814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(struct ib_conn *ib_conn) 3924814a0a4SEdward Tomasz Napierala { 3934814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc, *tmp; 3944814a0a4SEdward Tomasz Napierala int i = 0; 3954814a0a4SEdward Tomasz Napierala 3964814a0a4SEdward Tomasz Napierala if (list_empty(&ib_conn->fastreg.pool)) 3974814a0a4SEdward Tomasz Napierala return; 3984814a0a4SEdward Tomasz Napierala 3994814a0a4SEdward Tomasz Napierala ISER_DBG("freeing conn %p fr pool", ib_conn); 4004814a0a4SEdward Tomasz Napierala 4014814a0a4SEdward Tomasz Napierala list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.pool, list) { 4024814a0a4SEdward Tomasz Napierala list_del(&desc->list); 4034814a0a4SEdward Tomasz Napierala iser_free_reg_res(&desc->rsc); 4044814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 4054814a0a4SEdward Tomasz Napierala ++i; 4064814a0a4SEdward Tomasz Napierala } 4074814a0a4SEdward Tomasz Napierala 4084814a0a4SEdward Tomasz Napierala if (i < ib_conn->fastreg.pool_size) 4094814a0a4SEdward Tomasz Napierala ISER_WARN("pool still has %d regions registered", 4104814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size - i); 4114814a0a4SEdward Tomasz Napierala } 4124814a0a4SEdward Tomasz Napierala 4134814a0a4SEdward Tomasz Napierala /** 4144814a0a4SEdward Tomasz Napierala * iser_create_ib_conn_res - Queue-Pair (QP) 4154814a0a4SEdward Tomasz Napierala * 4164814a0a4SEdward Tomasz Napierala * returns 0 on success, 1 on failure 4174814a0a4SEdward Tomasz Napierala */ 4184814a0a4SEdward Tomasz Napierala static int 4194814a0a4SEdward Tomasz Napierala iser_create_ib_conn_res(struct ib_conn *ib_conn) 4204814a0a4SEdward Tomasz Napierala { 4214814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 4224814a0a4SEdward Tomasz Napierala struct iser_device *device; 4234814a0a4SEdward Tomasz Napierala struct ib_device_attr *dev_attr; 4244814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 4254814a0a4SEdward Tomasz Napierala int index, min_index = 0; 4264814a0a4SEdward Tomasz Napierala int ret = -ENOMEM; 4274814a0a4SEdward Tomasz Napierala 4284814a0a4SEdward Tomasz Napierala iser_conn = container_of(ib_conn, struct iser_conn, ib_conn); 4294814a0a4SEdward Tomasz Napierala device = ib_conn->device; 4304814a0a4SEdward Tomasz Napierala dev_attr = &device->dev_attr; 4314814a0a4SEdward Tomasz Napierala 4324814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 4334814a0a4SEdward Tomasz Napierala /* select the CQ with the minimal number of usages */ 4344814a0a4SEdward Tomasz Napierala for (index = 0; index < device->comps_used; index++) { 4354814a0a4SEdward Tomasz Napierala if (device->comps[index].active_qps < 4364814a0a4SEdward Tomasz Napierala device->comps[min_index].active_qps) 4374814a0a4SEdward Tomasz Napierala min_index = index; 4384814a0a4SEdward Tomasz Napierala } 4394814a0a4SEdward Tomasz Napierala ib_conn->comp = &device->comps[min_index]; 4404814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps++; 4414814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 4424814a0a4SEdward Tomasz Napierala ISER_INFO("cq index %d used for ib_conn %p", min_index, ib_conn); 4434814a0a4SEdward Tomasz Napierala 4444814a0a4SEdward Tomasz Napierala memset(&init_attr, 0, sizeof init_attr); 4454814a0a4SEdward Tomasz Napierala init_attr.event_handler = iser_qp_event_callback; 4464814a0a4SEdward Tomasz Napierala init_attr.qp_context = (void *)ib_conn; 4474814a0a4SEdward Tomasz Napierala init_attr.send_cq = ib_conn->comp->cq; 4484814a0a4SEdward Tomasz Napierala init_attr.recv_cq = ib_conn->comp->cq; 4494814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 4504814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_sge = 2; 4514814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_sge = 1; 4524814a0a4SEdward Tomasz Napierala init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 4534814a0a4SEdward Tomasz Napierala init_attr.qp_type = IB_QPT_RC; 4544814a0a4SEdward Tomasz Napierala 4554814a0a4SEdward Tomasz Napierala if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) { 4564814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 4574814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 4584814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS); 4594814a0a4SEdward Tomasz Napierala } else { 4604814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = dev_attr->max_qp_wr; 4614814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 4624814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr); 4634814a0a4SEdward Tomasz Napierala } 4644814a0a4SEdward Tomasz Napierala ISER_DBG("device %s supports max_send_wr %d", 4654814a0a4SEdward Tomasz Napierala device->ib_device->name, dev_attr->max_qp_wr); 4664814a0a4SEdward Tomasz Napierala 4674814a0a4SEdward Tomasz Napierala ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); 4684814a0a4SEdward Tomasz Napierala if (ret) 4694814a0a4SEdward Tomasz Napierala goto out_err; 4704814a0a4SEdward Tomasz Napierala 4714814a0a4SEdward Tomasz Napierala ib_conn->qp = ib_conn->cma_id->qp; 4724814a0a4SEdward Tomasz Napierala ISER_DBG("setting conn %p cma_id %p qp %p", 4734814a0a4SEdward Tomasz Napierala ib_conn, ib_conn->cma_id, 4744814a0a4SEdward Tomasz Napierala ib_conn->cma_id->qp); 4754814a0a4SEdward Tomasz Napierala 4764814a0a4SEdward Tomasz Napierala return (ret); 4774814a0a4SEdward Tomasz Napierala 4784814a0a4SEdward Tomasz Napierala out_err: 4794814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 4804814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 4814814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 4824814a0a4SEdward Tomasz Napierala ISER_ERR("unable to alloc mem or create resource, err %d", ret); 4834814a0a4SEdward Tomasz Napierala 4844814a0a4SEdward Tomasz Napierala return (ret); 4854814a0a4SEdward Tomasz Napierala } 4864814a0a4SEdward Tomasz Napierala 4874814a0a4SEdward Tomasz Napierala /** 4884814a0a4SEdward Tomasz Napierala * based on the resolved device node GUID see if there already allocated 4894814a0a4SEdward Tomasz Napierala * device for this device. If there's no such, create one. 4904814a0a4SEdward Tomasz Napierala */ 4914814a0a4SEdward Tomasz Napierala static struct iser_device * 4924814a0a4SEdward Tomasz Napierala iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) 4934814a0a4SEdward Tomasz Napierala { 4944814a0a4SEdward Tomasz Napierala struct iser_device *device; 4954814a0a4SEdward Tomasz Napierala 4964814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 4974814a0a4SEdward Tomasz Napierala 4984814a0a4SEdward Tomasz Napierala list_for_each_entry(device, &ig.device_list, ig_list) 4994814a0a4SEdward Tomasz Napierala /* find if there's a match using the node GUID */ 5004814a0a4SEdward Tomasz Napierala if (device->ib_device->node_guid == cma_id->device->node_guid) 5014814a0a4SEdward Tomasz Napierala goto inc_refcnt; 5024814a0a4SEdward Tomasz Napierala 5034814a0a4SEdward Tomasz Napierala device = malloc(sizeof *device, M_ISER_VERBS, M_WAITOK | M_ZERO); 5044814a0a4SEdward Tomasz Napierala /* assign this device to the device */ 5054814a0a4SEdward Tomasz Napierala device->ib_device = cma_id->device; 5064814a0a4SEdward Tomasz Napierala /* init the device and link it into ig device list */ 5074814a0a4SEdward Tomasz Napierala if (iser_create_device_ib_res(device)) { 5084814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 5094814a0a4SEdward Tomasz Napierala device = NULL; 5104814a0a4SEdward Tomasz Napierala goto out; 5114814a0a4SEdward Tomasz Napierala } 5124814a0a4SEdward Tomasz Napierala list_add(&device->ig_list, &ig.device_list); 5134814a0a4SEdward Tomasz Napierala 5144814a0a4SEdward Tomasz Napierala inc_refcnt: 5154814a0a4SEdward Tomasz Napierala device->refcount++; 5164814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 5174814a0a4SEdward Tomasz Napierala out: 5184814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 5194814a0a4SEdward Tomasz Napierala return (device); 5204814a0a4SEdward Tomasz Napierala } 5214814a0a4SEdward Tomasz Napierala 5224814a0a4SEdward Tomasz Napierala /* if there's no demand for this device, release it */ 5234814a0a4SEdward Tomasz Napierala static void 5244814a0a4SEdward Tomasz Napierala iser_device_try_release(struct iser_device *device) 5254814a0a4SEdward Tomasz Napierala { 5264814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 5274814a0a4SEdward Tomasz Napierala device->refcount--; 5284814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 5294814a0a4SEdward Tomasz Napierala if (!device->refcount) { 5304814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(device); 5314814a0a4SEdward Tomasz Napierala list_del(&device->ig_list); 5324814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 5334814a0a4SEdward Tomasz Napierala device = NULL; 5344814a0a4SEdward Tomasz Napierala } 5354814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 5364814a0a4SEdward Tomasz Napierala } 5374814a0a4SEdward Tomasz Napierala 5384814a0a4SEdward Tomasz Napierala /** 5394814a0a4SEdward Tomasz Napierala * Called with state mutex held 5404814a0a4SEdward Tomasz Napierala **/ 5414814a0a4SEdward Tomasz Napierala static int iser_conn_state_comp_exch(struct iser_conn *iser_conn, 5424814a0a4SEdward Tomasz Napierala enum iser_conn_state comp, 5434814a0a4SEdward Tomasz Napierala enum iser_conn_state exch) 5444814a0a4SEdward Tomasz Napierala { 5454814a0a4SEdward Tomasz Napierala int ret; 5464814a0a4SEdward Tomasz Napierala 5474814a0a4SEdward Tomasz Napierala ret = (iser_conn->state == comp); 5484814a0a4SEdward Tomasz Napierala if (ret) 5494814a0a4SEdward Tomasz Napierala iser_conn->state = exch; 5504814a0a4SEdward Tomasz Napierala 5514814a0a4SEdward Tomasz Napierala return ret; 5524814a0a4SEdward Tomasz Napierala } 5534814a0a4SEdward Tomasz Napierala 5544814a0a4SEdward Tomasz Napierala /** 5554814a0a4SEdward Tomasz Napierala * iser_free_ib_conn_res - release IB related resources 5564814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection struct 5574814a0a4SEdward Tomasz Napierala * @destroy: indicator if we need to try to release the 5584814a0a4SEdward Tomasz Napierala * iser device and memory regoins pool (only iscsi 5594814a0a4SEdward Tomasz Napierala * shutdown and DEVICE_REMOVAL will use this). 5604814a0a4SEdward Tomasz Napierala * 5614814a0a4SEdward Tomasz Napierala * This routine is called with the iser state mutex held 5624814a0a4SEdward Tomasz Napierala * so the cm_id removal is out of here. It is Safe to 5634814a0a4SEdward Tomasz Napierala * be invoked multiple times. 5644814a0a4SEdward Tomasz Napierala */ 5654814a0a4SEdward Tomasz Napierala void 5664814a0a4SEdward Tomasz Napierala iser_free_ib_conn_res(struct iser_conn *iser_conn, 5674814a0a4SEdward Tomasz Napierala bool destroy) 5684814a0a4SEdward Tomasz Napierala { 5694814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 5704814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 5714814a0a4SEdward Tomasz Napierala 5724814a0a4SEdward Tomasz Napierala ISER_INFO("freeing conn %p cma_id %p qp %p", 5734814a0a4SEdward Tomasz Napierala iser_conn, ib_conn->cma_id, ib_conn->qp); 5744814a0a4SEdward Tomasz Napierala 5754814a0a4SEdward Tomasz Napierala if (ib_conn->qp != NULL) { 5764814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 5774814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 5784814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 5794814a0a4SEdward Tomasz Napierala rdma_destroy_qp(ib_conn->cma_id); 5804814a0a4SEdward Tomasz Napierala ib_conn->qp = NULL; 5814814a0a4SEdward Tomasz Napierala } 5824814a0a4SEdward Tomasz Napierala 5834814a0a4SEdward Tomasz Napierala if (destroy) { 5844814a0a4SEdward Tomasz Napierala if (iser_conn->login_buf) 5854814a0a4SEdward Tomasz Napierala iser_free_login_buf(iser_conn); 5864814a0a4SEdward Tomasz Napierala 5874814a0a4SEdward Tomasz Napierala if (iser_conn->rx_descs) 5884814a0a4SEdward Tomasz Napierala iser_free_rx_descriptors(iser_conn); 5894814a0a4SEdward Tomasz Napierala 5904814a0a4SEdward Tomasz Napierala if (device != NULL) { 5914814a0a4SEdward Tomasz Napierala iser_device_try_release(device); 5924814a0a4SEdward Tomasz Napierala ib_conn->device = NULL; 5934814a0a4SEdward Tomasz Napierala } 5944814a0a4SEdward Tomasz Napierala } 5954814a0a4SEdward Tomasz Napierala } 5964814a0a4SEdward Tomasz Napierala 5974814a0a4SEdward Tomasz Napierala /** 5984814a0a4SEdward Tomasz Napierala * triggers start of the disconnect procedures and wait for them to be done 5994814a0a4SEdward Tomasz Napierala * Called with state mutex held 6004814a0a4SEdward Tomasz Napierala */ 6014814a0a4SEdward Tomasz Napierala int 6024814a0a4SEdward Tomasz Napierala iser_conn_terminate(struct iser_conn *iser_conn) 6034814a0a4SEdward Tomasz Napierala { 6044814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 605*c3987b8eSHans Petter Selasky const struct ib_send_wr *bad_send_wr; 606*c3987b8eSHans Petter Selasky const struct ib_recv_wr *bad_recv_wr; 6074814a0a4SEdward Tomasz Napierala int err = 0; 6084814a0a4SEdward Tomasz Napierala 6094814a0a4SEdward Tomasz Napierala /* terminate the iser conn only if the conn state is UP */ 6104814a0a4SEdward Tomasz Napierala if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP, 6114814a0a4SEdward Tomasz Napierala ISER_CONN_TERMINATING)) 6124814a0a4SEdward Tomasz Napierala return (0); 6134814a0a4SEdward Tomasz Napierala 6144814a0a4SEdward Tomasz Napierala ISER_INFO("iser_conn %p state %d\n", iser_conn, iser_conn->state); 6154814a0a4SEdward Tomasz Napierala 6164814a0a4SEdward Tomasz Napierala if (ib_conn->qp == NULL) { 6174814a0a4SEdward Tomasz Napierala /* HOW can this be??? */ 6184814a0a4SEdward Tomasz Napierala ISER_WARN("qp wasn't created"); 6194814a0a4SEdward Tomasz Napierala return (1); 6204814a0a4SEdward Tomasz Napierala } 6214814a0a4SEdward Tomasz Napierala 6224814a0a4SEdward Tomasz Napierala /* 6234814a0a4SEdward Tomasz Napierala * Todo: This is a temporary workaround. 6244814a0a4SEdward Tomasz Napierala * We serialize the connection closure using global lock in order to 6254814a0a4SEdward Tomasz Napierala * receive all posted beacons completions. 6264814a0a4SEdward Tomasz Napierala * Without Serialization, in case we open many connections (QPs) on 6274814a0a4SEdward Tomasz Napierala * the same CQ, we might miss beacons because of missing interrupts. 6284814a0a4SEdward Tomasz Napierala */ 6294814a0a4SEdward Tomasz Napierala sx_xlock(&ig.close_conns_mutex); 6304814a0a4SEdward Tomasz Napierala 6314814a0a4SEdward Tomasz Napierala /* 6324814a0a4SEdward Tomasz Napierala * In case we didn't already clean up the cma_id (peer initiated 6334814a0a4SEdward Tomasz Napierala * a disconnection), we need to Cause the CMA to change the QP 6344814a0a4SEdward Tomasz Napierala * state to ERROR. 6354814a0a4SEdward Tomasz Napierala */ 6364814a0a4SEdward Tomasz Napierala if (ib_conn->cma_id) { 6374814a0a4SEdward Tomasz Napierala err = rdma_disconnect(ib_conn->cma_id); 6384814a0a4SEdward Tomasz Napierala if (err) 6394814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to disconnect, conn: 0x%p err %d", 6404814a0a4SEdward Tomasz Napierala iser_conn, err); 6414814a0a4SEdward Tomasz Napierala 6424814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 6434814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.send, 0, sizeof(struct ib_send_wr)); 6444814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.wr_id = ISER_BEACON_WRID; 6454814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.opcode = IB_WR_SEND; 6464814a0a4SEdward Tomasz Napierala /* post an indication that all send flush errors were consumed */ 6474814a0a4SEdward Tomasz Napierala err = ib_post_send(ib_conn->qp, &ib_conn->beacon.send, &bad_send_wr); 6484814a0a4SEdward Tomasz Napierala if (err) { 6494814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post send_beacon", ib_conn); 6504814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6514814a0a4SEdward Tomasz Napierala goto out; 6524814a0a4SEdward Tomasz Napierala } 6534814a0a4SEdward Tomasz Napierala 6544814a0a4SEdward Tomasz Napierala ISER_DBG("before send cv_wait: %p", iser_conn); 6554814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 6564814a0a4SEdward Tomasz Napierala ISER_DBG("after send cv_wait: %p", iser_conn); 6574814a0a4SEdward Tomasz Napierala 6584814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.recv, 0, sizeof(struct ib_recv_wr)); 6594814a0a4SEdward Tomasz Napierala ib_conn->beacon.recv.wr_id = ISER_BEACON_WRID; 6604814a0a4SEdward Tomasz Napierala /* post an indication that all recv flush errors were consumed */ 6614814a0a4SEdward Tomasz Napierala err = ib_post_recv(ib_conn->qp, &ib_conn->beacon.recv, &bad_recv_wr); 6624814a0a4SEdward Tomasz Napierala if (err) { 6634814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post recv_beacon", ib_conn); 6644814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6654814a0a4SEdward Tomasz Napierala goto out; 6664814a0a4SEdward Tomasz Napierala } 6674814a0a4SEdward Tomasz Napierala 6684814a0a4SEdward Tomasz Napierala ISER_DBG("before recv cv_wait: %p", iser_conn); 6694814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 6704814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6714814a0a4SEdward Tomasz Napierala ISER_DBG("after recv cv_wait: %p", iser_conn); 6724814a0a4SEdward Tomasz Napierala } 6734814a0a4SEdward Tomasz Napierala out: 6744814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.close_conns_mutex); 6754814a0a4SEdward Tomasz Napierala return (1); 6764814a0a4SEdward Tomasz Napierala } 6774814a0a4SEdward Tomasz Napierala 6784814a0a4SEdward Tomasz Napierala /** 6794814a0a4SEdward Tomasz Napierala * Called with state mutex held 6804814a0a4SEdward Tomasz Napierala **/ 6814814a0a4SEdward Tomasz Napierala static void 6824814a0a4SEdward Tomasz Napierala iser_connect_error(struct rdma_cm_id *cma_id) 6834814a0a4SEdward Tomasz Napierala { 6844814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 6854814a0a4SEdward Tomasz Napierala 6864814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 6874814a0a4SEdward Tomasz Napierala 6884814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p", iser_conn); 6894814a0a4SEdward Tomasz Napierala 6904814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_TERMINATING; 6914814a0a4SEdward Tomasz Napierala 6924814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 6934814a0a4SEdward Tomasz Napierala } 6944814a0a4SEdward Tomasz Napierala 6954814a0a4SEdward Tomasz Napierala /** 6964814a0a4SEdward Tomasz Napierala * Called with state mutex held 6974814a0a4SEdward Tomasz Napierala **/ 6984814a0a4SEdward Tomasz Napierala static void 6994814a0a4SEdward Tomasz Napierala iser_addr_handler(struct rdma_cm_id *cma_id) 7004814a0a4SEdward Tomasz Napierala { 7014814a0a4SEdward Tomasz Napierala struct iser_device *device; 7024814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 7034814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 7044814a0a4SEdward Tomasz Napierala int ret; 7054814a0a4SEdward Tomasz Napierala 7064814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 7074814a0a4SEdward Tomasz Napierala 7084814a0a4SEdward Tomasz Napierala ib_conn = &iser_conn->ib_conn; 7094814a0a4SEdward Tomasz Napierala device = iser_device_find_by_ib_device(cma_id); 7104814a0a4SEdward Tomasz Napierala if (!device) { 7114814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p device lookup/creation failed", 7124814a0a4SEdward Tomasz Napierala iser_conn); 7134814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7144814a0a4SEdward Tomasz Napierala return; 7154814a0a4SEdward Tomasz Napierala } 7164814a0a4SEdward Tomasz Napierala 7174814a0a4SEdward Tomasz Napierala ib_conn->device = device; 7184814a0a4SEdward Tomasz Napierala 7194814a0a4SEdward Tomasz Napierala ret = rdma_resolve_route(cma_id, 1000); 7204814a0a4SEdward Tomasz Napierala if (ret) { 7214814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p resolve route failed: %d", iser_conn, ret); 7224814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7234814a0a4SEdward Tomasz Napierala return; 7244814a0a4SEdward Tomasz Napierala } 7254814a0a4SEdward Tomasz Napierala } 7264814a0a4SEdward Tomasz Napierala 7274814a0a4SEdward Tomasz Napierala /** 7284814a0a4SEdward Tomasz Napierala * Called with state mutex held 7294814a0a4SEdward Tomasz Napierala **/ 7304814a0a4SEdward Tomasz Napierala static void 7314814a0a4SEdward Tomasz Napierala iser_route_handler(struct rdma_cm_id *cma_id) 7324814a0a4SEdward Tomasz Napierala { 7334814a0a4SEdward Tomasz Napierala struct rdma_conn_param conn_param; 7344814a0a4SEdward Tomasz Napierala int ret; 7354814a0a4SEdward Tomasz Napierala struct iser_cm_hdr req_hdr; 7364814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 7374814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 7384814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 7394814a0a4SEdward Tomasz Napierala 7404814a0a4SEdward Tomasz Napierala ret = iser_create_ib_conn_res(ib_conn); 7414814a0a4SEdward Tomasz Napierala if (ret) 7424814a0a4SEdward Tomasz Napierala goto failure; 7434814a0a4SEdward Tomasz Napierala 7444814a0a4SEdward Tomasz Napierala memset(&conn_param, 0, sizeof conn_param); 7454814a0a4SEdward Tomasz Napierala conn_param.responder_resources = device->dev_attr.max_qp_rd_atom; 7464814a0a4SEdward Tomasz Napierala conn_param.retry_count = 7; 7474814a0a4SEdward Tomasz Napierala conn_param.rnr_retry_count = 6; 7484814a0a4SEdward Tomasz Napierala /* 7494814a0a4SEdward Tomasz Napierala * Initiaotr depth should not be set, but in order to compat 7504814a0a4SEdward Tomasz Napierala * with old targets, we keep this value set. 7514814a0a4SEdward Tomasz Napierala */ 7524814a0a4SEdward Tomasz Napierala conn_param.initiator_depth = 1; 7534814a0a4SEdward Tomasz Napierala 7544814a0a4SEdward Tomasz Napierala memset(&req_hdr, 0, sizeof(req_hdr)); 7554814a0a4SEdward Tomasz Napierala req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED | 7564814a0a4SEdward Tomasz Napierala ISER_SEND_W_INV_NOT_SUPPORTED); 7574814a0a4SEdward Tomasz Napierala conn_param.private_data = (void *)&req_hdr; 7584814a0a4SEdward Tomasz Napierala conn_param.private_data_len = sizeof(struct iser_cm_hdr); 7594814a0a4SEdward Tomasz Napierala 7604814a0a4SEdward Tomasz Napierala ret = rdma_connect(cma_id, &conn_param); 7614814a0a4SEdward Tomasz Napierala if (ret) { 7624814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failure connecting: %d", iser_conn, ret); 7634814a0a4SEdward Tomasz Napierala goto failure; 7644814a0a4SEdward Tomasz Napierala } 7654814a0a4SEdward Tomasz Napierala 7664814a0a4SEdward Tomasz Napierala return; 7674814a0a4SEdward Tomasz Napierala failure: 7684814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7694814a0a4SEdward Tomasz Napierala } 7704814a0a4SEdward Tomasz Napierala 7714814a0a4SEdward Tomasz Napierala /** 7724814a0a4SEdward Tomasz Napierala * Called with state mutex held 7734814a0a4SEdward Tomasz Napierala **/ 7744814a0a4SEdward Tomasz Napierala static void 7754814a0a4SEdward Tomasz Napierala iser_connected_handler(struct rdma_cm_id *cma_id) 7764814a0a4SEdward Tomasz Napierala { 7774814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 7784814a0a4SEdward Tomasz Napierala struct ib_qp_attr attr; 7794814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 7804814a0a4SEdward Tomasz Napierala 7814814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 7824814a0a4SEdward Tomasz Napierala 7834814a0a4SEdward Tomasz Napierala (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); 7844814a0a4SEdward Tomasz Napierala 7854814a0a4SEdward Tomasz Napierala ISER_INFO("remote qpn:%x my qpn:%x", 7864814a0a4SEdward Tomasz Napierala attr.dest_qp_num, cma_id->qp->qp_num); 7874814a0a4SEdward Tomasz Napierala 7884814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_UP; 7894814a0a4SEdward Tomasz Napierala 7904814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 7914814a0a4SEdward Tomasz Napierala } 7924814a0a4SEdward Tomasz Napierala 7934814a0a4SEdward Tomasz Napierala /** 7944814a0a4SEdward Tomasz Napierala * Called with state mutex held 7954814a0a4SEdward Tomasz Napierala **/ 7964814a0a4SEdward Tomasz Napierala static void 7974814a0a4SEdward Tomasz Napierala iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy) 7984814a0a4SEdward Tomasz Napierala { 7994814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 8004814a0a4SEdward Tomasz Napierala 8014814a0a4SEdward Tomasz Napierala if (iser_conn_terminate(iser_conn)) 8024814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 8034814a0a4SEdward Tomasz Napierala 8044814a0a4SEdward Tomasz Napierala } 8054814a0a4SEdward Tomasz Napierala 8064814a0a4SEdward Tomasz Napierala int 8074814a0a4SEdward Tomasz Napierala iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 8084814a0a4SEdward Tomasz Napierala { 8094814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 8104814a0a4SEdward Tomasz Napierala int ret = 0; 8114814a0a4SEdward Tomasz Napierala 8124814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 8134814a0a4SEdward Tomasz Napierala ISER_INFO("event %d status %d conn %p id %p", 8144814a0a4SEdward Tomasz Napierala event->event, event->status, cma_id->context, cma_id); 8154814a0a4SEdward Tomasz Napierala 8164814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 8174814a0a4SEdward Tomasz Napierala switch (event->event) { 8184814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_RESOLVED: 8194814a0a4SEdward Tomasz Napierala iser_addr_handler(cma_id); 8204814a0a4SEdward Tomasz Napierala break; 8214814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_RESOLVED: 8224814a0a4SEdward Tomasz Napierala iser_route_handler(cma_id); 8234814a0a4SEdward Tomasz Napierala break; 8244814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ESTABLISHED: 8254814a0a4SEdward Tomasz Napierala iser_connected_handler(cma_id); 8264814a0a4SEdward Tomasz Napierala break; 8274814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_ERROR: 8284814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_ERROR: 8294814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_CONNECT_ERROR: 8304814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_UNREACHABLE: 8314814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_REJECTED: 8324814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 8334814a0a4SEdward Tomasz Napierala break; 8344814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_DISCONNECTED: 8354814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_CHANGE: 8364814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_TIMEWAIT_EXIT: 8374814a0a4SEdward Tomasz Napierala iser_cleanup_handler(cma_id, false); 8384814a0a4SEdward Tomasz Napierala break; 8394814a0a4SEdward Tomasz Napierala default: 8404814a0a4SEdward Tomasz Napierala ISER_ERR("Unexpected RDMA CM event (%d)", event->event); 8414814a0a4SEdward Tomasz Napierala break; 8424814a0a4SEdward Tomasz Napierala } 8434814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 8444814a0a4SEdward Tomasz Napierala 8454814a0a4SEdward Tomasz Napierala return (ret); 8464814a0a4SEdward Tomasz Napierala } 8474814a0a4SEdward Tomasz Napierala 8484814a0a4SEdward Tomasz Napierala int 8494814a0a4SEdward Tomasz Napierala iser_post_recvl(struct iser_conn *iser_conn) 8504814a0a4SEdward Tomasz Napierala { 851*c3987b8eSHans Petter Selasky const struct ib_recv_wr *rx_wr_failed; 852*c3987b8eSHans Petter Selasky struct ib_recv_wr rx_wr; 8534814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 8544814a0a4SEdward Tomasz Napierala struct ib_sge sge; 8554814a0a4SEdward Tomasz Napierala int ib_ret; 8564814a0a4SEdward Tomasz Napierala 8574814a0a4SEdward Tomasz Napierala sge.addr = iser_conn->login_resp_dma; 8584814a0a4SEdward Tomasz Napierala sge.length = ISER_RX_LOGIN_SIZE; 8594814a0a4SEdward Tomasz Napierala sge.lkey = ib_conn->device->mr->lkey; 8604814a0a4SEdward Tomasz Napierala 8614814a0a4SEdward Tomasz Napierala rx_wr.wr_id = (uintptr_t)iser_conn->login_resp_buf; 8624814a0a4SEdward Tomasz Napierala rx_wr.sg_list = &sge; 8634814a0a4SEdward Tomasz Napierala rx_wr.num_sge = 1; 8644814a0a4SEdward Tomasz Napierala rx_wr.next = NULL; 8654814a0a4SEdward Tomasz Napierala 8664814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count++; 8674814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed); 8684814a0a4SEdward Tomasz Napierala if (ib_ret) { 8694814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 8704814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 8714814a0a4SEdward Tomasz Napierala } 8724814a0a4SEdward Tomasz Napierala 8734814a0a4SEdward Tomasz Napierala return (ib_ret); 8744814a0a4SEdward Tomasz Napierala } 8754814a0a4SEdward Tomasz Napierala 8764814a0a4SEdward Tomasz Napierala int 8774814a0a4SEdward Tomasz Napierala iser_post_recvm(struct iser_conn *iser_conn, int count) 8784814a0a4SEdward Tomasz Napierala { 879*c3987b8eSHans Petter Selasky const struct ib_recv_wr *rx_wr_failed; 880*c3987b8eSHans Petter Selasky struct ib_recv_wr *rx_wr; 8814814a0a4SEdward Tomasz Napierala int i, ib_ret; 8824814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 8834814a0a4SEdward Tomasz Napierala unsigned int my_rx_head = iser_conn->rx_desc_head; 8844814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 8854814a0a4SEdward Tomasz Napierala 8864814a0a4SEdward Tomasz Napierala for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8874814a0a4SEdward Tomasz Napierala rx_desc = &iser_conn->rx_descs[my_rx_head]; 8884814a0a4SEdward Tomasz Napierala rx_wr->wr_id = (uintptr_t)rx_desc; 8894814a0a4SEdward Tomasz Napierala rx_wr->sg_list = &rx_desc->rx_sg; 8904814a0a4SEdward Tomasz Napierala rx_wr->num_sge = 1; 8914814a0a4SEdward Tomasz Napierala rx_wr->next = rx_wr + 1; 8924814a0a4SEdward Tomasz Napierala my_rx_head = (my_rx_head + 1) % iser_conn->qp_max_recv_dtos; 8934814a0a4SEdward Tomasz Napierala } 8944814a0a4SEdward Tomasz Napierala 8954814a0a4SEdward Tomasz Napierala rx_wr--; 8964814a0a4SEdward Tomasz Napierala rx_wr->next = NULL; /* mark end of work requests list */ 8974814a0a4SEdward Tomasz Napierala 8984814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count += count; 8994814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed); 9004814a0a4SEdward Tomasz Napierala if (ib_ret) { 9014814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 9024814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count -= count; 9034814a0a4SEdward Tomasz Napierala } else 9044814a0a4SEdward Tomasz Napierala iser_conn->rx_desc_head = my_rx_head; 9054814a0a4SEdward Tomasz Napierala 9064814a0a4SEdward Tomasz Napierala return (ib_ret); 9074814a0a4SEdward Tomasz Napierala } 9084814a0a4SEdward Tomasz Napierala 9094814a0a4SEdward Tomasz Napierala /** 9104814a0a4SEdward Tomasz Napierala * iser_start_send - Initiate a Send DTO operation 9114814a0a4SEdward Tomasz Napierala * 9124814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 9134814a0a4SEdward Tomasz Napierala */ 9144814a0a4SEdward Tomasz Napierala int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 9154814a0a4SEdward Tomasz Napierala bool signal) 9164814a0a4SEdward Tomasz Napierala { 9174814a0a4SEdward Tomasz Napierala int ib_ret; 918*c3987b8eSHans Petter Selasky const struct ib_send_wr *send_wr_failed; 919*c3987b8eSHans Petter Selasky struct ib_send_wr send_wr; 9204814a0a4SEdward Tomasz Napierala 9214814a0a4SEdward Tomasz Napierala ib_dma_sync_single_for_device(ib_conn->device->ib_device, 9224814a0a4SEdward Tomasz Napierala tx_desc->dma_addr, ISER_HEADERS_LEN, 9234814a0a4SEdward Tomasz Napierala DMA_TO_DEVICE); 9244814a0a4SEdward Tomasz Napierala 9254814a0a4SEdward Tomasz Napierala send_wr.next = NULL; 9264814a0a4SEdward Tomasz Napierala send_wr.wr_id = (uintptr_t)tx_desc; 9274814a0a4SEdward Tomasz Napierala send_wr.sg_list = tx_desc->tx_sg; 9284814a0a4SEdward Tomasz Napierala send_wr.num_sge = tx_desc->num_sge; 9294814a0a4SEdward Tomasz Napierala send_wr.opcode = IB_WR_SEND; 9304814a0a4SEdward Tomasz Napierala send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0; 9314814a0a4SEdward Tomasz Napierala 9324814a0a4SEdward Tomasz Napierala ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 9334814a0a4SEdward Tomasz Napierala if (ib_ret) 9344814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_send failed, ret:%d", ib_ret); 9354814a0a4SEdward Tomasz Napierala 9364814a0a4SEdward Tomasz Napierala return (ib_ret); 9374814a0a4SEdward Tomasz Napierala } 938