1 /*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <dev/mlx5/driver.h> 31 #include <dev/mlx5/srq.h> 32 #include <rdma/ib_verbs.h> 33 #include "mlx5_core.h" 34 #include "transobj.h" 35 36 void mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type) 37 { 38 struct mlx5_srq_table *table = &dev->priv.srq_table; 39 struct mlx5_core_srq *srq; 40 41 spin_lock(&table->lock); 42 43 srq = radix_tree_lookup(&table->tree, srqn); 44 if (srq) 45 atomic_inc(&srq->refcount); 46 47 spin_unlock(&table->lock); 48 49 if (!srq) { 50 mlx5_core_warn(dev, "Async event for bogus SRQ 0x%08x\n", srqn); 51 return; 52 } 53 54 srq->event(srq, event_type); 55 56 if (atomic_dec_and_test(&srq->refcount)) 57 complete(&srq->free); 58 } 59 60 static void rmpc_srqc_reformat(void *srqc, void *rmpc, bool srqc_to_rmpc) 61 { 62 void *wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 63 64 if (srqc_to_rmpc) { 65 switch (MLX5_GET(srqc, srqc, state)) { 66 case MLX5_SRQC_STATE_GOOD: 67 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY); 68 break; 69 case MLX5_SRQC_STATE_ERROR: 70 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_ERR); 71 break; 72 default: 73 printf("mlx5_core: WARN: ""%s: %d: Unknown srq state = 0x%x\n", __func__, __LINE__, MLX5_GET(srqc, srqc, state)); 74 } 75 76 MLX5_SET(wq, wq, wq_signature, MLX5_GET(srqc, srqc, wq_signature)); 77 MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(srqc, srqc, log_page_size)); 78 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(srqc, srqc, log_rq_stride) + 4); 79 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(srqc, srqc, log_srq_size)); 80 MLX5_SET(wq, wq, page_offset, MLX5_GET(srqc, srqc, page_offset)); 81 MLX5_SET(wq, wq, lwm, MLX5_GET(srqc, srqc, lwm)); 82 MLX5_SET(wq, wq, pd, MLX5_GET(srqc, srqc, pd)); 83 MLX5_SET64(wq, wq, dbr_addr, 84 ((u64)MLX5_GET(srqc, srqc, db_record_addr_h)) << 32 | 85 ((u64)MLX5_GET(srqc, srqc, db_record_addr_l)) << 2); 86 } else { 87 switch (MLX5_GET(rmpc, rmpc, state)) { 88 case MLX5_RMPC_STATE_RDY: 89 MLX5_SET(srqc, srqc, state, MLX5_SRQC_STATE_GOOD); 90 break; 91 case MLX5_RMPC_STATE_ERR: 92 MLX5_SET(srqc, srqc, state, MLX5_SRQC_STATE_ERROR); 93 break; 94 default: 95 printf("mlx5_core: WARN: ""%s: %d: Unknown rmp state = 0x%x\n", __func__, __LINE__, MLX5_GET(rmpc, rmpc, state)); 96 } 97 98 MLX5_SET(srqc, srqc, wq_signature, MLX5_GET(wq, wq, wq_signature)); 99 MLX5_SET(srqc, srqc, log_page_size, MLX5_GET(wq, wq, log_wq_pg_sz)); 100 MLX5_SET(srqc, srqc, log_rq_stride, MLX5_GET(wq, wq, log_wq_stride) - 4); 101 MLX5_SET(srqc, srqc, log_srq_size, MLX5_GET(wq, wq, log_wq_sz)); 102 MLX5_SET(srqc, srqc, page_offset, MLX5_GET(wq, wq, page_offset)); 103 MLX5_SET(srqc, srqc, lwm, MLX5_GET(wq, wq, lwm)); 104 MLX5_SET(srqc, srqc, pd, MLX5_GET(wq, wq, pd)); 105 MLX5_SET(srqc, srqc, db_record_addr_h, MLX5_GET64(wq, wq, dbr_addr) >> 32); 106 MLX5_SET(srqc, srqc, db_record_addr_l, (MLX5_GET64(wq, wq, dbr_addr) >> 2) & 0x3fffffff); 107 } 108 } 109 110 struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn) 111 { 112 struct mlx5_srq_table *table = &dev->priv.srq_table; 113 struct mlx5_core_srq *srq; 114 115 spin_lock(&table->lock); 116 117 srq = radix_tree_lookup(&table->tree, srqn); 118 if (srq) 119 atomic_inc(&srq->refcount); 120 121 spin_unlock(&table->lock); 122 123 return srq; 124 } 125 EXPORT_SYMBOL(mlx5_core_get_srq); 126 127 static int get_pas_size(void *srqc) 128 { 129 u32 log_page_size = MLX5_GET(srqc, srqc, log_page_size) + 12; 130 u32 log_srq_size = MLX5_GET(srqc, srqc, log_srq_size); 131 u32 log_rq_stride = MLX5_GET(srqc, srqc, log_rq_stride); 132 u32 page_offset = MLX5_GET(srqc, srqc, page_offset); 133 u32 po_quanta = 1 << (log_page_size - 6); 134 u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride); 135 u32 page_size = 1 << log_page_size; 136 u32 rq_sz_po = rq_sz + (page_offset * po_quanta); 137 u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size; 138 139 return rq_num_pas * sizeof(u64); 140 141 } 142 143 static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 144 struct mlx5_create_srq_mbox_in *in, int srq_inlen) 145 { 146 void *create_in; 147 void *rmpc; 148 void *srqc; 149 int pas_size; 150 int inlen; 151 int err; 152 153 srqc = MLX5_ADDR_OF(create_srq_in, in, srq_context_entry); 154 pas_size = get_pas_size(srqc); 155 inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size; 156 create_in = mlx5_vzalloc(inlen); 157 if (!create_in) 158 return -ENOMEM; 159 160 rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx); 161 162 memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size); 163 rmpc_srqc_reformat(srqc, rmpc, true); 164 165 err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn); 166 167 kvfree(create_in); 168 return err; 169 } 170 171 static int destroy_rmp_cmd(struct mlx5_core_dev *dev, 172 struct mlx5_core_srq *srq) 173 { 174 return mlx5_core_destroy_rmp(dev, srq->srqn); 175 } 176 177 static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 178 struct mlx5_query_srq_mbox_out *out) 179 { 180 u32 *rmp_out; 181 void *rmpc; 182 void *srqc; 183 int err; 184 185 rmp_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_rmp_out)); 186 if (!rmp_out) 187 return -ENOMEM; 188 189 err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out); 190 if (err) 191 goto out; 192 193 srqc = MLX5_ADDR_OF(query_srq_out, out, srq_context_entry); 194 rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context); 195 rmpc_srqc_reformat(srqc, rmpc, false); 196 197 out: 198 kvfree(rmp_out); 199 return 0; 200 } 201 202 static int arm_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, u16 lwm) 203 { 204 return mlx5_core_arm_rmp(dev, srq->srqn, lwm); 205 } 206 207 static int create_xrc_srq_cmd(struct mlx5_core_dev *dev, 208 struct mlx5_core_srq *srq, 209 struct mlx5_create_srq_mbox_in *in, 210 int srq_inlen) 211 { 212 void *create_in; 213 void *srqc; 214 void *xrc_srqc; 215 void *pas; 216 int pas_size; 217 int inlen; 218 int err; 219 220 srqc = MLX5_ADDR_OF(create_srq_in, in, srq_context_entry); 221 pas_size = get_pas_size(srqc); 222 inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size; 223 create_in = mlx5_vzalloc(inlen); 224 if (!create_in) 225 return -ENOMEM; 226 227 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in, xrc_srq_context_entry); 228 pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas); 229 230 memcpy(xrc_srqc, srqc, MLX5_ST_SZ_BYTES(srqc)); 231 memcpy(pas, in->pas, pas_size); 232 /* 0xffffff means we ask to work with cqe version 0 */ 233 MLX5_SET(xrc_srqc, xrc_srqc, user_index, 0xffffff); 234 235 err = mlx5_core_create_xsrq(dev, create_in, inlen, &srq->srqn); 236 if (err) 237 goto out; 238 239 out: 240 kvfree(create_in); 241 return err; 242 } 243 244 static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev, 245 struct mlx5_core_srq *srq) 246 { 247 return mlx5_core_destroy_xsrq(dev, srq->srqn); 248 } 249 250 static int query_xrc_srq_cmd(struct mlx5_core_dev *dev, 251 struct mlx5_core_srq *srq, 252 struct mlx5_query_srq_mbox_out *out) 253 { 254 u32 *xrcsrq_out; 255 int err; 256 257 xrcsrq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out)); 258 if (!xrcsrq_out) 259 return -ENOMEM; 260 261 err = mlx5_core_query_xsrq(dev, srq->srqn, xrcsrq_out); 262 if (err) 263 goto out; 264 265 out: 266 kvfree(xrcsrq_out); 267 return err; 268 } 269 270 static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev, 271 struct mlx5_core_srq *srq, u16 lwm) 272 { 273 return mlx5_core_arm_xsrq(dev, srq->srqn, lwm); 274 } 275 276 static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 277 struct mlx5_create_srq_mbox_in *in, int inlen) 278 { 279 struct mlx5_create_srq_mbox_out out; 280 int err; 281 282 memset(&out, 0, sizeof(out)); 283 284 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_SRQ); 285 286 err = mlx5_cmd_exec_check_status(dev, (u32 *)in, inlen, (u32 *)(&out), sizeof(out)); 287 288 srq->srqn = be32_to_cpu(out.srqn) & 0xffffff; 289 290 return err; 291 } 292 293 static int destroy_srq_cmd(struct mlx5_core_dev *dev, 294 struct mlx5_core_srq *srq) 295 { 296 struct mlx5_destroy_srq_mbox_in in; 297 struct mlx5_destroy_srq_mbox_out out; 298 299 memset(&in, 0, sizeof(in)); 300 memset(&out, 0, sizeof(out)); 301 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_SRQ); 302 in.srqn = cpu_to_be32(srq->srqn); 303 304 return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)(&out), sizeof(out)); 305 } 306 307 static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 308 struct mlx5_query_srq_mbox_out *out) 309 { 310 struct mlx5_query_srq_mbox_in in; 311 312 memset(&in, 0, sizeof(in)); 313 314 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SRQ); 315 in.srqn = cpu_to_be32(srq->srqn); 316 317 return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)out, sizeof(*out)); 318 } 319 320 static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 321 u16 lwm, int is_srq) 322 { 323 struct mlx5_arm_srq_mbox_in in; 324 struct mlx5_arm_srq_mbox_out out; 325 326 memset(&in, 0, sizeof(in)); 327 memset(&out, 0, sizeof(out)); 328 329 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ARM_RQ); 330 in.hdr.opmod = cpu_to_be16(!!is_srq); 331 in.srqn = cpu_to_be32(srq->srqn); 332 in.lwm = cpu_to_be16(lwm); 333 334 return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)(&out), sizeof(out)); 335 } 336 337 static int create_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 338 struct mlx5_create_srq_mbox_in *in, int inlen, 339 int is_xrc) 340 { 341 if (!dev->issi) 342 return create_srq_cmd(dev, srq, in, inlen); 343 else if (srq->common.res == MLX5_RES_XSRQ) 344 return create_xrc_srq_cmd(dev, srq, in, inlen); 345 else 346 return create_rmp_cmd(dev, srq, in, inlen); 347 } 348 349 static int destroy_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq) 350 { 351 if (!dev->issi) 352 return destroy_srq_cmd(dev, srq); 353 else if (srq->common.res == MLX5_RES_XSRQ) 354 return destroy_xrc_srq_cmd(dev, srq); 355 else 356 return destroy_rmp_cmd(dev, srq); 357 } 358 359 int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 360 struct mlx5_create_srq_mbox_in *in, int inlen, 361 int is_xrc) 362 { 363 int err; 364 struct mlx5_srq_table *table = &dev->priv.srq_table; 365 366 srq->common.res = is_xrc ? MLX5_RES_XSRQ : MLX5_RES_SRQ; 367 368 err = create_srq_split(dev, srq, in, inlen, is_xrc); 369 if (err) 370 return err; 371 372 atomic_set(&srq->refcount, 1); 373 init_completion(&srq->free); 374 375 spin_lock_irq(&table->lock); 376 err = radix_tree_insert(&table->tree, srq->srqn, srq); 377 spin_unlock_irq(&table->lock); 378 if (err) { 379 mlx5_core_warn(dev, "err %d, srqn 0x%x\n", err, srq->srqn); 380 goto err_destroy_srq_split; 381 } 382 383 return 0; 384 385 err_destroy_srq_split: 386 destroy_srq_split(dev, srq); 387 388 return err; 389 } 390 EXPORT_SYMBOL(mlx5_core_create_srq); 391 392 int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq) 393 { 394 struct mlx5_srq_table *table = &dev->priv.srq_table; 395 struct mlx5_core_srq *tmp; 396 int err; 397 398 spin_lock_irq(&table->lock); 399 tmp = radix_tree_delete(&table->tree, srq->srqn); 400 spin_unlock_irq(&table->lock); 401 if (!tmp) { 402 mlx5_core_warn(dev, "srq 0x%x not found in tree\n", srq->srqn); 403 return -EINVAL; 404 } 405 if (tmp != srq) { 406 mlx5_core_warn(dev, "corruption on srqn 0x%x\n", srq->srqn); 407 return -EINVAL; 408 } 409 410 err = destroy_srq_split(dev, srq); 411 if (err) 412 return err; 413 414 if (atomic_dec_and_test(&srq->refcount)) 415 complete(&srq->free); 416 wait_for_completion(&srq->free); 417 418 return 0; 419 } 420 EXPORT_SYMBOL(mlx5_core_destroy_srq); 421 422 int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 423 struct mlx5_query_srq_mbox_out *out) 424 { 425 if (!dev->issi) 426 return query_srq_cmd(dev, srq, out); 427 else if (srq->common.res == MLX5_RES_XSRQ) 428 return query_xrc_srq_cmd(dev, srq, out); 429 else 430 return query_rmp_cmd(dev, srq, out); 431 } 432 EXPORT_SYMBOL(mlx5_core_query_srq); 433 434 int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 435 u16 lwm, int is_srq) 436 { 437 if (!dev->issi) 438 return arm_srq_cmd(dev, srq, lwm, is_srq); 439 else if (srq->common.res == MLX5_RES_XSRQ) 440 return arm_xrc_srq_cmd(dev, srq, lwm); 441 else 442 return arm_rmp_cmd(dev, srq, lwm); 443 } 444 EXPORT_SYMBOL(mlx5_core_arm_srq); 445 446 void mlx5_init_srq_table(struct mlx5_core_dev *dev) 447 { 448 struct mlx5_srq_table *table = &dev->priv.srq_table; 449 450 spin_lock_init(&table->lock); 451 INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 452 } 453 454 void mlx5_cleanup_srq_table(struct mlx5_core_dev *dev) 455 { 456 /* nothing */ 457 } 458