Lines Matching +full:cmd +full:- +full:db
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
42 #include "mlx4-abi.h"
46 return srq->buf.buf + (n << srq->wqe_shift); in get_wqe()
53 pthread_spin_lock(&srq->lock); in mlx4_free_srq_wqe()
55 next = get_wqe(srq, srq->tail); in mlx4_free_srq_wqe()
56 next->next_wqe_index = htobe16(ind); in mlx4_free_srq_wqe()
57 srq->tail = ind; in mlx4_free_srq_wqe()
59 pthread_spin_unlock(&srq->lock); in mlx4_free_srq_wqe()
73 pthread_spin_lock(&srq->lock); in mlx4_post_srq_recv()
75 for (nreq = 0; wr; ++nreq, wr = wr->next) { in mlx4_post_srq_recv()
76 if (wr->num_sge > srq->max_gs) { in mlx4_post_srq_recv()
77 err = -1; in mlx4_post_srq_recv()
82 if (srq->head == srq->tail) { in mlx4_post_srq_recv()
84 err = -1; in mlx4_post_srq_recv()
89 srq->wrid[srq->head] = wr->wr_id; in mlx4_post_srq_recv()
91 next = get_wqe(srq, srq->head); in mlx4_post_srq_recv()
92 srq->head = be16toh(next->next_wqe_index); in mlx4_post_srq_recv()
95 for (i = 0; i < wr->num_sge; ++i) { in mlx4_post_srq_recv()
96 scat[i].byte_count = htobe32(wr->sg_list[i].length); in mlx4_post_srq_recv()
97 scat[i].lkey = htobe32(wr->sg_list[i].lkey); in mlx4_post_srq_recv()
98 scat[i].addr = htobe64(wr->sg_list[i].addr); in mlx4_post_srq_recv()
101 if (i < srq->max_gs) { in mlx4_post_srq_recv()
109 srq->counter += nreq; in mlx4_post_srq_recv()
117 *srq->db = htobe32(srq->counter); in mlx4_post_srq_recv()
120 pthread_spin_unlock(&srq->lock); in mlx4_post_srq_recv()
134 srq->wrid = malloc(srq->max * sizeof (uint64_t)); in mlx4_alloc_srq_buf()
135 if (!srq->wrid) in mlx4_alloc_srq_buf()
136 return -1; in mlx4_alloc_srq_buf()
139 srq->max_gs * sizeof (struct mlx4_wqe_data_seg); in mlx4_alloc_srq_buf()
141 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) in mlx4_alloc_srq_buf()
144 buf_size = srq->max << srq->wqe_shift; in mlx4_alloc_srq_buf()
146 if (mlx4_alloc_buf(&srq->buf, buf_size, in mlx4_alloc_srq_buf()
147 to_mdev(pd->context->device)->page_size)) { in mlx4_alloc_srq_buf()
148 free(srq->wrid); in mlx4_alloc_srq_buf()
149 return -1; in mlx4_alloc_srq_buf()
152 memset(srq->buf.buf, 0, buf_size); in mlx4_alloc_srq_buf()
159 for (i = 0; i < srq->max; ++i) { in mlx4_alloc_srq_buf()
161 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); in mlx4_alloc_srq_buf()
164 (void *) scatter < (void *) next + (1 << srq->wqe_shift); in mlx4_alloc_srq_buf()
166 scatter->lkey = htobe32(MLX4_INVALID_LKEY); in mlx4_alloc_srq_buf()
169 srq->head = 0; in mlx4_alloc_srq_buf()
170 srq->tail = srq->max - 1; in mlx4_alloc_srq_buf()
177 pthread_mutex_destroy(&xsrq_table->mutex); in mlx4_cleanup_xsrq_table()
184 xsrq_table->num_xsrq = size; in mlx4_init_xsrq_table()
185 xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS; in mlx4_init_xsrq_table()
186 xsrq_table->mask = (1 << xsrq_table->shift) - 1; in mlx4_init_xsrq_table()
188 return pthread_mutex_init(&xsrq_table->mutex, NULL); in mlx4_init_xsrq_table()
195 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; in mlx4_find_xsrq()
196 if (xsrq_table->xsrq_table[index].refcnt) in mlx4_find_xsrq()
197 return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask]; in mlx4_find_xsrq()
207 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; in mlx4_store_xsrq()
208 pthread_mutex_lock(&xsrq_table->mutex); in mlx4_store_xsrq()
209 if (!xsrq_table->xsrq_table[index].refcnt) { in mlx4_store_xsrq()
210 xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1, in mlx4_store_xsrq()
212 if (!xsrq_table->xsrq_table[index].table) { in mlx4_store_xsrq()
213 ret = -1; in mlx4_store_xsrq()
218 xsrq_table->xsrq_table[index].refcnt++; in mlx4_store_xsrq()
219 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq; in mlx4_store_xsrq()
222 pthread_mutex_unlock(&xsrq_table->mutex); in mlx4_store_xsrq()
230 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; in mlx4_clear_xsrq()
231 pthread_mutex_lock(&xsrq_table->mutex); in mlx4_clear_xsrq()
233 if (--xsrq_table->xsrq_table[index].refcnt) in mlx4_clear_xsrq()
234 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL; in mlx4_clear_xsrq()
236 free(xsrq_table->xsrq_table[index].table); in mlx4_clear_xsrq()
238 pthread_mutex_unlock(&xsrq_table->mutex); in mlx4_clear_xsrq()
244 struct mlx4_create_xsrq cmd; in mlx4_create_xrc_srq() local
250 if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64) in mlx4_create_xrc_srq()
257 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) in mlx4_create_xrc_srq()
260 srq->max = align_queue_size(attr_ex->attr.max_wr + 1); in mlx4_create_xrc_srq()
261 srq->max_gs = attr_ex->attr.max_sge; in mlx4_create_xrc_srq()
262 srq->counter = 0; in mlx4_create_xrc_srq()
263 srq->ext_srq = 1; in mlx4_create_xrc_srq()
265 if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq)) in mlx4_create_xrc_srq()
268 srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ); in mlx4_create_xrc_srq()
269 if (!srq->db) in mlx4_create_xrc_srq()
272 *srq->db = 0; in mlx4_create_xrc_srq()
274 cmd.buf_addr = (uintptr_t) srq->buf.buf; in mlx4_create_xrc_srq()
275 cmd.db_addr = (uintptr_t) srq->db; in mlx4_create_xrc_srq()
277 ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, in mlx4_create_xrc_srq()
278 sizeof(srq->verbs_srq), in mlx4_create_xrc_srq()
280 &cmd.ibv_cmd, sizeof cmd, in mlx4_create_xrc_srq()
285 ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table, in mlx4_create_xrc_srq()
286 srq->verbs_srq.srq_num, srq); in mlx4_create_xrc_srq()
290 return &srq->verbs_srq.srq; in mlx4_create_xrc_srq()
293 ibv_cmd_destroy_srq(&srq->verbs_srq.srq); in mlx4_create_xrc_srq()
295 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db); in mlx4_create_xrc_srq()
297 free(srq->wrid); in mlx4_create_xrc_srq()
298 mlx4_free_buf(&srq->buf); in mlx4_create_xrc_srq()
300 pthread_spin_destroy(&srq->lock); in mlx4_create_xrc_srq()
308 struct mlx4_context *mctx = to_mctx(srq->context); in mlx4_destroy_xrc_srq()
313 mcq = to_mcq(msrq->verbs_srq.cq); in mlx4_destroy_xrc_srq()
315 pthread_spin_lock(&mcq->lock); in mlx4_destroy_xrc_srq()
316 mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num); in mlx4_destroy_xrc_srq()
317 pthread_spin_unlock(&mcq->lock); in mlx4_destroy_xrc_srq()
321 pthread_spin_lock(&mcq->lock); in mlx4_destroy_xrc_srq()
322 mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq); in mlx4_destroy_xrc_srq()
323 pthread_spin_unlock(&mcq->lock); in mlx4_destroy_xrc_srq()
327 mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db); in mlx4_destroy_xrc_srq()
328 mlx4_free_buf(&msrq->buf); in mlx4_destroy_xrc_srq()
329 free(msrq->wrid); in mlx4_destroy_xrc_srq()
330 pthread_spin_destroy(&msrq->lock); in mlx4_destroy_xrc_srq()