1 /*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #define LINUXKPI_PARAM_PREFIX mlx5_ 29 30 #include <linux/kmod.h> 31 #include <linux/module.h> 32 #include <linux/errno.h> 33 #include <linux/pci.h> 34 #include <linux/dma-mapping.h> 35 #include <linux/slab.h> 36 #include <linux/io-mapping.h> 37 #include <linux/interrupt.h> 38 #include <dev/mlx5/driver.h> 39 #include <dev/mlx5/cq.h> 40 #include <dev/mlx5/qp.h> 41 #include <dev/mlx5/srq.h> 42 #include <linux/delay.h> 43 #include <dev/mlx5/mlx5_ifc.h> 44 #include "mlx5_core.h" 45 46 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 47 MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 48 MODULE_LICENSE("Dual BSD/GPL"); 49 #if (__FreeBSD_version >= 1100000) 50 MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1); 51 #endif 52 MODULE_VERSION(mlx5, 1); 53 54 int mlx5_core_debug_mask; 55 module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 56 MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 57 58 #define MLX5_DEFAULT_PROF 2 59 static int prof_sel = MLX5_DEFAULT_PROF; 60 module_param_named(prof_sel, prof_sel, int, 0444); 61 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 62 63 #define NUMA_NO_NODE -1 64 65 struct workqueue_struct *mlx5_core_wq; 66 static LIST_HEAD(intf_list); 67 static LIST_HEAD(dev_list); 68 static DEFINE_MUTEX(intf_mutex); 69 70 struct mlx5_device_context { 71 struct list_head list; 72 struct mlx5_interface *intf; 73 void *context; 74 }; 75 76 static struct mlx5_profile profiles[] = { 77 [0] = { 78 .mask = 0, 79 }, 80 [1] = { 81 .mask = MLX5_PROF_MASK_QP_SIZE, 82 .log_max_qp = 12, 83 }, 84 [2] = { 85 .mask = MLX5_PROF_MASK_QP_SIZE | 86 MLX5_PROF_MASK_MR_CACHE, 87 .log_max_qp = 17, 88 .mr_cache[0] = { 89 .size = 500, 90 .limit = 250 91 }, 92 .mr_cache[1] = { 93 .size = 500, 94 .limit = 250 95 }, 96 .mr_cache[2] = { 97 .size = 500, 98 .limit = 250 99 }, 100 .mr_cache[3] = { 101 .size = 500, 102 .limit = 250 103 }, 104 .mr_cache[4] = { 105 .size = 500, 106 .limit = 250 107 }, 108 .mr_cache[5] = { 109 .size = 500, 110 .limit = 250 111 }, 112 .mr_cache[6] = { 113 .size = 500, 114 .limit = 250 115 }, 116 .mr_cache[7] = { 117 .size = 500, 118 .limit = 250 119 }, 120 .mr_cache[8] = { 121 .size = 500, 122 .limit = 250 123 }, 124 .mr_cache[9] = { 125 .size = 500, 126 .limit = 250 127 }, 128 .mr_cache[10] = { 129 .size = 500, 130 .limit = 250 131 }, 132 .mr_cache[11] = { 133 .size = 500, 134 .limit = 250 135 }, 136 .mr_cache[12] = { 137 .size = 64, 138 .limit = 32 139 }, 140 .mr_cache[13] = { 141 .size = 32, 142 .limit = 16 143 }, 144 .mr_cache[14] = { 145 .size = 16, 146 .limit = 8 147 }, 148 }, 149 [3] = { 150 .mask = MLX5_PROF_MASK_QP_SIZE, 151 .log_max_qp = 17, 152 }, 153 }; 154 155 static int set_dma_caps(struct pci_dev *pdev) 156 { 157 int err; 158 159 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 160 if (err) { 161 device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit PCI DMA mask\n"); 162 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 163 if (err) { 164 device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set PCI DMA mask, aborting\n"); 165 return err; 166 } 167 } 168 169 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 170 if (err) { 171 device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 172 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 173 if (err) { 174 device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set consistent PCI DMA mask, aborting\n"); 175 return err; 176 } 177 } 178 179 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 180 return err; 181 } 182 183 static int request_bar(struct pci_dev *pdev) 184 { 185 int err = 0; 186 187 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 188 device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n"); 189 return -ENODEV; 190 } 191 192 err = pci_request_regions(pdev, DRIVER_NAME); 193 if (err) 194 device_printf((&pdev->dev)->bsddev, "ERR: ""Couldn't get PCI resources, aborting\n"); 195 196 return err; 197 } 198 199 static void release_bar(struct pci_dev *pdev) 200 { 201 pci_release_regions(pdev); 202 } 203 204 static int mlx5_enable_msix(struct mlx5_core_dev *dev) 205 { 206 struct mlx5_priv *priv = &dev->priv; 207 struct mlx5_eq_table *table = &priv->eq_table; 208 int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 209 int nvec; 210 int i; 211 212 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 213 MLX5_EQ_VEC_COMP_BASE; 214 nvec = min_t(int, nvec, num_eqs); 215 if (nvec <= MLX5_EQ_VEC_COMP_BASE) 216 return -ENOMEM; 217 218 priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL); 219 220 priv->irq_info = kzalloc(nvec * sizeof(*priv->irq_info), GFP_KERNEL); 221 222 for (i = 0; i < nvec; i++) 223 priv->msix_arr[i].entry = i; 224 225 nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 226 MLX5_EQ_VEC_COMP_BASE + 1, nvec); 227 if (nvec < 0) 228 return nvec; 229 230 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 231 232 return 0; 233 234 } 235 236 static void mlx5_disable_msix(struct mlx5_core_dev *dev) 237 { 238 struct mlx5_priv *priv = &dev->priv; 239 240 pci_disable_msix(dev->pdev); 241 kfree(priv->irq_info); 242 kfree(priv->msix_arr); 243 } 244 245 struct mlx5_reg_host_endianess { 246 u8 he; 247 u8 rsvd[15]; 248 }; 249 250 251 #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 252 253 enum { 254 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 255 MLX5_DEV_CAP_FLAG_DCT | 256 MLX5_DEV_CAP_FLAG_DRAIN_SIGERR, 257 }; 258 259 static u16 to_fw_pkey_sz(u32 size) 260 { 261 switch (size) { 262 case 128: 263 return 0; 264 case 256: 265 return 1; 266 case 512: 267 return 2; 268 case 1024: 269 return 3; 270 case 2048: 271 return 4; 272 case 4096: 273 return 5; 274 default: 275 printf("mlx5_core: WARN: ""invalid pkey table size %d\n", size); 276 return 0; 277 } 278 } 279 280 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 281 enum mlx5_cap_mode cap_mode) 282 { 283 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 284 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 285 void *out, *hca_caps; 286 u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 287 int err; 288 289 memset(in, 0, sizeof(in)); 290 out = kzalloc(out_sz, GFP_KERNEL); 291 292 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 293 MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 294 err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 295 if (err) 296 goto query_ex; 297 298 err = mlx5_cmd_status_to_err_v2(out); 299 if (err) { 300 mlx5_core_warn(dev, 301 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 302 cap_type, cap_mode, err); 303 goto query_ex; 304 } 305 306 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 307 308 switch (cap_mode) { 309 case HCA_CAP_OPMOD_GET_MAX: 310 memcpy(dev->hca_caps_max[cap_type], hca_caps, 311 MLX5_UN_SZ_BYTES(hca_cap_union)); 312 break; 313 case HCA_CAP_OPMOD_GET_CUR: 314 memcpy(dev->hca_caps_cur[cap_type], hca_caps, 315 MLX5_UN_SZ_BYTES(hca_cap_union)); 316 break; 317 default: 318 mlx5_core_warn(dev, 319 "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 320 cap_type, cap_mode); 321 err = -EINVAL; 322 break; 323 } 324 query_ex: 325 kfree(out); 326 return err; 327 } 328 329 static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 330 { 331 u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 332 int err; 333 334 memset(out, 0, sizeof(out)); 335 336 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 337 err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 338 if (err) 339 return err; 340 341 err = mlx5_cmd_status_to_err_v2(out); 342 343 return err; 344 } 345 346 static int handle_hca_cap(struct mlx5_core_dev *dev) 347 { 348 void *set_ctx = NULL; 349 struct mlx5_profile *prof = dev->profile; 350 int err = -ENOMEM; 351 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 352 void *set_hca_cap; 353 354 set_ctx = kzalloc(set_sz, GFP_KERNEL); 355 356 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX); 357 if (err) 358 goto query_ex; 359 360 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR); 361 if (err) 362 goto query_ex; 363 364 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 365 capability); 366 memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 367 MLX5_ST_SZ_BYTES(cmd_hca_cap)); 368 369 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 370 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 371 128); 372 /* we limit the size of the pkey table to 128 entries for now */ 373 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 374 to_fw_pkey_sz(128)); 375 376 if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 377 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 378 prof->log_max_qp); 379 380 /* disable cmdif checksum */ 381 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 382 383 /* enable drain sigerr */ 384 MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1); 385 386 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 387 388 err = set_caps(dev, set_ctx, set_sz); 389 390 query_ex: 391 kfree(set_ctx); 392 return err; 393 } 394 395 static int set_hca_ctrl(struct mlx5_core_dev *dev) 396 { 397 struct mlx5_reg_host_endianess he_in; 398 struct mlx5_reg_host_endianess he_out; 399 int err; 400 401 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH && 402 !MLX5_CAP_GEN(dev, roce)) 403 return 0; 404 405 memset(&he_in, 0, sizeof(he_in)); 406 he_in.he = MLX5_SET_HOST_ENDIANNESS; 407 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 408 &he_out, sizeof(he_out), 409 MLX5_REG_HOST_ENDIANNESS, 0, 1); 410 return err; 411 } 412 413 static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 414 { 415 u32 in[MLX5_ST_SZ_DW(enable_hca_in)]; 416 u32 out[MLX5_ST_SZ_DW(enable_hca_out)]; 417 418 memset(in, 0, sizeof(in)); 419 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 420 memset(out, 0, sizeof(out)); 421 return mlx5_cmd_exec_check_status(dev, in, sizeof(in), 422 out, sizeof(out)); 423 } 424 425 static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 426 { 427 u32 in[MLX5_ST_SZ_DW(disable_hca_in)]; 428 u32 out[MLX5_ST_SZ_DW(disable_hca_out)]; 429 430 memset(in, 0, sizeof(in)); 431 432 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 433 memset(out, 0, sizeof(out)); 434 return mlx5_cmd_exec_check_status(dev, in, sizeof(in), 435 out, sizeof(out)); 436 } 437 438 static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 439 { 440 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]; 441 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)]; 442 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]; 443 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)]; 444 int err; 445 u32 sup_issi; 446 447 memset(query_in, 0, sizeof(query_in)); 448 memset(query_out, 0, sizeof(query_out)); 449 450 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 451 452 err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in), 453 query_out, sizeof(query_out)); 454 if (err) { 455 if (((struct mlx5_outbox_hdr *)query_out)->status == 456 MLX5_CMD_STAT_BAD_OP_ERR) { 457 pr_debug("Only ISSI 0 is supported\n"); 458 return 0; 459 } 460 461 printf("mlx5_core: ERR: ""failed to query ISSI\n"); 462 return err; 463 } 464 465 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 466 467 if (sup_issi & (1 << 1)) { 468 memset(set_in, 0, sizeof(set_in)); 469 memset(set_out, 0, sizeof(set_out)); 470 471 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 472 MLX5_SET(set_issi_in, set_in, current_issi, 1); 473 474 err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in), 475 set_out, sizeof(set_out)); 476 if (err) { 477 printf("mlx5_core: ERR: ""failed to set ISSI=1\n"); 478 return err; 479 } 480 481 dev->issi = 1; 482 483 return 0; 484 } else if (sup_issi & (1 << 0)) { 485 return 0; 486 } 487 488 return -ENOTSUPP; 489 } 490 491 492 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn) 493 { 494 struct mlx5_eq_table *table = &dev->priv.eq_table; 495 struct mlx5_eq *eq; 496 int err = -ENOENT; 497 498 spin_lock(&table->lock); 499 list_for_each_entry(eq, &table->comp_eqs_list, list) { 500 if (eq->index == vector) { 501 *eqn = eq->eqn; 502 *irqn = eq->irqn; 503 err = 0; 504 break; 505 } 506 } 507 spin_unlock(&table->lock); 508 509 return err; 510 } 511 EXPORT_SYMBOL(mlx5_vector2eqn); 512 513 int mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name) 514 { 515 struct mlx5_priv *priv = &dev->priv; 516 struct mlx5_eq_table *table = &priv->eq_table; 517 struct mlx5_eq *eq; 518 int err = -ENOENT; 519 520 spin_lock(&table->lock); 521 list_for_each_entry(eq, &table->comp_eqs_list, list) { 522 if (eq->index == eq_ix) { 523 int irq_ix = eq_ix + MLX5_EQ_VEC_COMP_BASE; 524 525 snprintf(priv->irq_info[irq_ix].name, MLX5_MAX_IRQ_NAME, 526 "%s-%d", name, eq_ix); 527 528 err = 0; 529 break; 530 } 531 } 532 spin_unlock(&table->lock); 533 534 return err; 535 } 536 537 static void free_comp_eqs(struct mlx5_core_dev *dev) 538 { 539 struct mlx5_eq_table *table = &dev->priv.eq_table; 540 struct mlx5_eq *eq, *n; 541 542 spin_lock(&table->lock); 543 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 544 list_del(&eq->list); 545 spin_unlock(&table->lock); 546 if (mlx5_destroy_unmap_eq(dev, eq)) 547 mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 548 eq->eqn); 549 kfree(eq); 550 spin_lock(&table->lock); 551 } 552 spin_unlock(&table->lock); 553 } 554 555 static int alloc_comp_eqs(struct mlx5_core_dev *dev) 556 { 557 struct mlx5_eq_table *table = &dev->priv.eq_table; 558 char name[MLX5_MAX_IRQ_NAME]; 559 struct mlx5_eq *eq; 560 int ncomp_vec; 561 int nent; 562 int err; 563 int i; 564 565 INIT_LIST_HEAD(&table->comp_eqs_list); 566 ncomp_vec = table->num_comp_vectors; 567 nent = MLX5_COMP_EQ_SIZE; 568 for (i = 0; i < ncomp_vec; i++) { 569 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 570 571 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 572 err = mlx5_create_map_eq(dev, eq, 573 i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 574 name, &dev->priv.uuari.uars[0]); 575 if (err) { 576 kfree(eq); 577 goto clean; 578 } 579 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 580 eq->index = i; 581 spin_lock(&table->lock); 582 list_add_tail(&eq->list, &table->comp_eqs_list); 583 spin_unlock(&table->lock); 584 } 585 586 return 0; 587 588 clean: 589 free_comp_eqs(dev); 590 return err; 591 } 592 593 static int map_bf_area(struct mlx5_core_dev *dev) 594 { 595 resource_size_t bf_start = pci_resource_start(dev->pdev, 0); 596 resource_size_t bf_len = pci_resource_len(dev->pdev, 0); 597 598 dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); 599 600 return dev->priv.bf_mapping ? 0 : -ENOMEM; 601 } 602 603 static void unmap_bf_area(struct mlx5_core_dev *dev) 604 { 605 if (dev->priv.bf_mapping) 606 io_mapping_free(dev->priv.bf_mapping); 607 } 608 609 static inline int fw_initializing(struct mlx5_core_dev *dev) 610 { 611 return ioread32be(&dev->iseg->initializing) >> 31; 612 } 613 614 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 615 { 616 u64 end = jiffies + msecs_to_jiffies(max_wait_mili); 617 int err = 0; 618 619 while (fw_initializing(dev)) { 620 if (time_after(jiffies, end)) { 621 err = -EBUSY; 622 break; 623 } 624 msleep(FW_INIT_WAIT_MS); 625 } 626 627 return err; 628 } 629 630 static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev) 631 { 632 struct mlx5_priv *priv = &dev->priv; 633 int err; 634 635 dev->pdev = pdev; 636 pci_set_drvdata(dev->pdev, dev); 637 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 638 priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 639 640 mutex_init(&priv->pgdir_mutex); 641 INIT_LIST_HEAD(&priv->pgdir_list); 642 spin_lock_init(&priv->mkey_lock); 643 644 priv->numa_node = NUMA_NO_NODE; 645 646 err = pci_enable_device(pdev); 647 if (err) { 648 device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n"); 649 goto err_dbg; 650 } 651 652 err = request_bar(pdev); 653 if (err) { 654 device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n"); 655 goto err_disable; 656 } 657 658 pci_set_master(pdev); 659 660 err = set_dma_caps(pdev); 661 if (err) { 662 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed setting DMA capabilities mask, aborting\n"); 663 goto err_clr_master; 664 } 665 666 dev->iseg = ioremap(pci_resource_start(dev->pdev, 0), 667 sizeof(*dev->iseg)); 668 if (!dev->iseg) { 669 err = -ENOMEM; 670 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n"); 671 goto err_clr_master; 672 } 673 device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 674 675 /* 676 * On load removing any previous indication of internal error, 677 * device is up 678 */ 679 dev->state = MLX5_DEVICE_STATE_UP; 680 681 err = mlx5_cmd_init(dev); 682 if (err) { 683 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n"); 684 goto err_unmap; 685 } 686 687 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 688 if (err) { 689 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI); 690 goto err_cmd_cleanup; 691 } 692 693 mlx5_pagealloc_init(dev); 694 695 err = mlx5_core_enable_hca(dev); 696 if (err) { 697 device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n"); 698 goto err_pagealloc_cleanup; 699 } 700 701 err = mlx5_core_set_issi(dev); 702 if (err) { 703 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to set issi\n"); 704 goto err_disable_hca; 705 } 706 707 err = mlx5_pagealloc_start(dev); 708 if (err) { 709 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pagealloc_start failed\n"); 710 goto err_disable_hca; 711 } 712 713 err = mlx5_satisfy_startup_pages(dev, 1); 714 if (err) { 715 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate boot pages\n"); 716 goto err_pagealloc_stop; 717 } 718 719 err = handle_hca_cap(dev); 720 if (err) { 721 device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap failed\n"); 722 goto reclaim_boot_pages; 723 } 724 725 err = set_hca_ctrl(dev); 726 if (err) { 727 device_printf((&pdev->dev)->bsddev, "ERR: ""set_hca_ctrl failed\n"); 728 goto reclaim_boot_pages; 729 } 730 731 err = mlx5_satisfy_startup_pages(dev, 0); 732 if (err) { 733 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate init pages\n"); 734 goto reclaim_boot_pages; 735 } 736 737 err = mlx5_cmd_init_hca(dev); 738 if (err) { 739 device_printf((&pdev->dev)->bsddev, "ERR: ""init hca failed\n"); 740 goto reclaim_boot_pages; 741 } 742 743 mlx5_start_health_poll(dev); 744 745 err = mlx5_query_hca_caps(dev); 746 if (err) { 747 device_printf((&pdev->dev)->bsddev, "ERR: ""query hca failed\n"); 748 goto err_stop_poll; 749 } 750 751 err = mlx5_query_board_id(dev); 752 if (err) { 753 device_printf((&pdev->dev)->bsddev, "ERR: ""query board id failed\n"); 754 goto err_stop_poll; 755 } 756 757 err = mlx5_enable_msix(dev); 758 if (err) { 759 device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n"); 760 goto err_stop_poll; 761 } 762 763 err = mlx5_eq_init(dev); 764 if (err) { 765 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to initialize eq\n"); 766 goto disable_msix; 767 } 768 769 err = mlx5_alloc_uuars(dev, &priv->uuari); 770 if (err) { 771 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n"); 772 goto err_eq_cleanup; 773 } 774 775 err = mlx5_start_eqs(dev); 776 if (err) { 777 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to start pages and async EQs\n"); 778 goto err_free_uar; 779 } 780 781 err = alloc_comp_eqs(dev); 782 if (err) { 783 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to alloc completion EQs\n"); 784 goto err_stop_eqs; 785 } 786 787 if (map_bf_area(dev)) 788 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n"); 789 790 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 791 792 mlx5_init_cq_table(dev); 793 mlx5_init_qp_table(dev); 794 mlx5_init_srq_table(dev); 795 mlx5_init_mr_table(dev); 796 797 return 0; 798 799 err_stop_eqs: 800 mlx5_stop_eqs(dev); 801 802 err_free_uar: 803 mlx5_free_uuars(dev, &priv->uuari); 804 805 err_eq_cleanup: 806 mlx5_eq_cleanup(dev); 807 808 disable_msix: 809 mlx5_disable_msix(dev); 810 811 err_stop_poll: 812 mlx5_stop_health_poll(dev); 813 if (mlx5_cmd_teardown_hca(dev)) { 814 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 815 return err; 816 } 817 818 reclaim_boot_pages: 819 mlx5_reclaim_startup_pages(dev); 820 821 err_pagealloc_stop: 822 mlx5_pagealloc_stop(dev); 823 824 err_disable_hca: 825 mlx5_core_disable_hca(dev); 826 827 err_pagealloc_cleanup: 828 mlx5_pagealloc_cleanup(dev); 829 err_cmd_cleanup: 830 mlx5_cmd_cleanup(dev); 831 832 err_unmap: 833 iounmap(dev->iseg); 834 835 err_clr_master: 836 pci_clear_master(dev->pdev); 837 release_bar(dev->pdev); 838 839 err_disable: 840 pci_disable_device(dev->pdev); 841 842 err_dbg: 843 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 844 return err; 845 } 846 847 static void mlx5_dev_cleanup(struct mlx5_core_dev *dev) 848 { 849 struct mlx5_priv *priv = &dev->priv; 850 851 mlx5_cleanup_mr_table(dev); 852 mlx5_cleanup_srq_table(dev); 853 mlx5_cleanup_qp_table(dev); 854 mlx5_cleanup_cq_table(dev); 855 unmap_bf_area(dev); 856 mlx5_wait_for_reclaim_vfs_pages(dev); 857 free_comp_eqs(dev); 858 mlx5_stop_eqs(dev); 859 mlx5_free_uuars(dev, &priv->uuari); 860 mlx5_eq_cleanup(dev); 861 mlx5_disable_msix(dev); 862 mlx5_stop_health_poll(dev); 863 if (mlx5_cmd_teardown_hca(dev)) { 864 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 865 return; 866 } 867 mlx5_pagealloc_stop(dev); 868 mlx5_reclaim_startup_pages(dev); 869 mlx5_core_disable_hca(dev); 870 mlx5_pagealloc_cleanup(dev); 871 mlx5_cmd_cleanup(dev); 872 iounmap(dev->iseg); 873 pci_clear_master(dev->pdev); 874 release_bar(dev->pdev); 875 pci_disable_device(dev->pdev); 876 } 877 878 static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 879 { 880 struct mlx5_device_context *dev_ctx; 881 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 882 883 dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 884 885 dev_ctx->intf = intf; 886 dev_ctx->context = intf->add(dev); 887 888 if (dev_ctx->context) { 889 spin_lock_irq(&priv->ctx_lock); 890 list_add_tail(&dev_ctx->list, &priv->ctx_list); 891 spin_unlock_irq(&priv->ctx_lock); 892 } else { 893 kfree(dev_ctx); 894 } 895 } 896 897 static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 898 { 899 struct mlx5_device_context *dev_ctx; 900 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 901 902 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 903 if (dev_ctx->intf == intf) { 904 spin_lock_irq(&priv->ctx_lock); 905 list_del(&dev_ctx->list); 906 spin_unlock_irq(&priv->ctx_lock); 907 908 intf->remove(dev, dev_ctx->context); 909 kfree(dev_ctx); 910 return; 911 } 912 } 913 static int mlx5_register_device(struct mlx5_core_dev *dev) 914 { 915 struct mlx5_priv *priv = &dev->priv; 916 struct mlx5_interface *intf; 917 918 mutex_lock(&intf_mutex); 919 list_add_tail(&priv->dev_list, &dev_list); 920 list_for_each_entry(intf, &intf_list, list) 921 mlx5_add_device(intf, priv); 922 mutex_unlock(&intf_mutex); 923 924 return 0; 925 } 926 static void mlx5_unregister_device(struct mlx5_core_dev *dev) 927 { 928 struct mlx5_priv *priv = &dev->priv; 929 struct mlx5_interface *intf; 930 931 mutex_lock(&intf_mutex); 932 list_for_each_entry(intf, &intf_list, list) 933 mlx5_remove_device(intf, priv); 934 list_del(&priv->dev_list); 935 mutex_unlock(&intf_mutex); 936 } 937 938 int mlx5_register_interface(struct mlx5_interface *intf) 939 { 940 struct mlx5_priv *priv; 941 942 if (!intf->add || !intf->remove) 943 return -EINVAL; 944 945 mutex_lock(&intf_mutex); 946 list_add_tail(&intf->list, &intf_list); 947 list_for_each_entry(priv, &dev_list, dev_list) 948 mlx5_add_device(intf, priv); 949 mutex_unlock(&intf_mutex); 950 951 return 0; 952 } 953 EXPORT_SYMBOL(mlx5_register_interface); 954 955 void mlx5_unregister_interface(struct mlx5_interface *intf) 956 { 957 struct mlx5_priv *priv; 958 959 mutex_lock(&intf_mutex); 960 list_for_each_entry(priv, &dev_list, dev_list) 961 mlx5_remove_device(intf, priv); 962 list_del(&intf->list); 963 mutex_unlock(&intf_mutex); 964 } 965 EXPORT_SYMBOL(mlx5_unregister_interface); 966 967 void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 968 { 969 struct mlx5_priv *priv = &mdev->priv; 970 struct mlx5_device_context *dev_ctx; 971 unsigned long flags; 972 void *result = NULL; 973 974 spin_lock_irqsave(&priv->ctx_lock, flags); 975 976 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 977 if ((dev_ctx->intf->protocol == protocol) && 978 dev_ctx->intf->get_dev) { 979 result = dev_ctx->intf->get_dev(dev_ctx->context); 980 break; 981 } 982 983 spin_unlock_irqrestore(&priv->ctx_lock, flags); 984 985 return result; 986 } 987 EXPORT_SYMBOL(mlx5_get_protocol_dev); 988 989 static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 990 unsigned long param) 991 { 992 struct mlx5_priv *priv = &dev->priv; 993 struct mlx5_device_context *dev_ctx; 994 unsigned long flags; 995 996 spin_lock_irqsave(&priv->ctx_lock, flags); 997 998 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 999 if (dev_ctx->intf->event) 1000 dev_ctx->intf->event(dev, dev_ctx->context, event, param); 1001 1002 spin_unlock_irqrestore(&priv->ctx_lock, flags); 1003 } 1004 1005 struct mlx5_core_event_handler { 1006 void (*event)(struct mlx5_core_dev *dev, 1007 enum mlx5_dev_event event, 1008 void *data); 1009 }; 1010 1011 1012 static int init_one(struct pci_dev *pdev, 1013 const struct pci_device_id *id) 1014 { 1015 struct mlx5_core_dev *dev; 1016 struct mlx5_priv *priv; 1017 int err; 1018 1019 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1020 priv = &dev->priv; 1021 if (id) 1022 priv->pci_dev_data = id->driver_data; 1023 1024 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) { 1025 printf("mlx5_core: WARN: ""selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF); 1026 prof_sel = MLX5_DEFAULT_PROF; 1027 } 1028 dev->profile = &profiles[prof_sel]; 1029 dev->event = mlx5_core_event; 1030 1031 INIT_LIST_HEAD(&priv->ctx_list); 1032 spin_lock_init(&priv->ctx_lock); 1033 err = mlx5_dev_init(dev, pdev); 1034 if (err) { 1035 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_dev_init failed %d\n", err); 1036 goto out; 1037 } 1038 1039 err = mlx5_register_device(dev); 1040 if (err) { 1041 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err); 1042 goto out_init; 1043 } 1044 1045 1046 return 0; 1047 1048 out_init: 1049 mlx5_dev_cleanup(dev); 1050 out: 1051 kfree(dev); 1052 return err; 1053 } 1054 1055 static void remove_one(struct pci_dev *pdev) 1056 { 1057 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1058 1059 mlx5_unregister_device(dev); 1060 mlx5_dev_cleanup(dev); 1061 kfree(dev); 1062 } 1063 1064 static const struct pci_device_id mlx5_core_pci_table[] = { 1065 { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */ 1066 { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */ 1067 { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */ 1068 { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */ 1069 { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */ 1070 { PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */ 1071 { PCI_VDEVICE(MELLANOX, 4119) }, /* ConnectX-5 */ 1072 { PCI_VDEVICE(MELLANOX, 4120) }, /* ConnectX-5 VF */ 1073 { PCI_VDEVICE(MELLANOX, 4121) }, 1074 { PCI_VDEVICE(MELLANOX, 4122) }, 1075 { PCI_VDEVICE(MELLANOX, 4123) }, 1076 { PCI_VDEVICE(MELLANOX, 4124) }, 1077 { PCI_VDEVICE(MELLANOX, 4125) }, 1078 { PCI_VDEVICE(MELLANOX, 4126) }, 1079 { PCI_VDEVICE(MELLANOX, 4127) }, 1080 { PCI_VDEVICE(MELLANOX, 4128) }, 1081 { PCI_VDEVICE(MELLANOX, 4129) }, 1082 { PCI_VDEVICE(MELLANOX, 4130) }, 1083 { PCI_VDEVICE(MELLANOX, 4131) }, 1084 { PCI_VDEVICE(MELLANOX, 4132) }, 1085 { PCI_VDEVICE(MELLANOX, 4133) }, 1086 { PCI_VDEVICE(MELLANOX, 4134) }, 1087 { PCI_VDEVICE(MELLANOX, 4135) }, 1088 { PCI_VDEVICE(MELLANOX, 4136) }, 1089 { PCI_VDEVICE(MELLANOX, 4137) }, 1090 { PCI_VDEVICE(MELLANOX, 4138) }, 1091 { PCI_VDEVICE(MELLANOX, 4139) }, 1092 { PCI_VDEVICE(MELLANOX, 4140) }, 1093 { PCI_VDEVICE(MELLANOX, 4141) }, 1094 { PCI_VDEVICE(MELLANOX, 4142) }, 1095 { PCI_VDEVICE(MELLANOX, 4143) }, 1096 { PCI_VDEVICE(MELLANOX, 4144) }, 1097 { 0, } 1098 }; 1099 1100 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 1101 1102 static struct pci_driver mlx5_core_driver = { 1103 .name = DRIVER_NAME, 1104 .id_table = mlx5_core_pci_table, 1105 .probe = init_one, 1106 .remove = remove_one 1107 }; 1108 1109 static int __init init(void) 1110 { 1111 int err; 1112 1113 mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq"); 1114 if (!mlx5_core_wq) { 1115 err = -ENOMEM; 1116 goto err_debug; 1117 } 1118 mlx5_health_init(); 1119 1120 err = pci_register_driver(&mlx5_core_driver); 1121 if (err) 1122 goto err_health; 1123 1124 1125 return 0; 1126 1127 err_health: 1128 mlx5_health_cleanup(); 1129 destroy_workqueue(mlx5_core_wq); 1130 err_debug: 1131 return err; 1132 } 1133 1134 static void __exit cleanup(void) 1135 { 1136 pci_unregister_driver(&mlx5_core_driver); 1137 mlx5_health_cleanup(); 1138 destroy_workqueue(mlx5_core_wq); 1139 } 1140 1141 module_init(init); 1142 module_exit(cleanup); 1143 1144 void mlx5_enter_error_state(struct mlx5_core_dev *dev) 1145 { 1146 if (dev->state != MLX5_DEVICE_STATE_UP) 1147 return; 1148 1149 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1150 mlx5_trigger_cmd_completions(dev); 1151 } 1152 EXPORT_SYMBOL(mlx5_enter_error_state); 1153