1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Huawei Technologies Co., Ltd 3 */ 4 5 #include <rte_bus_pci.h> 6 7 #include "hinic_compat.h" 8 #include "hinic_csr.h" 9 #include "hinic_pmd_hwdev.h" 10 #include "hinic_pmd_hwif.h" 11 12 #define HINIC_CFG_REGS_BAR 0 13 #define HINIC_INTR_MSI_BAR 2 14 #define HINIC_DB_MEM_BAR 4 15 16 #define PAGE_SIZE_4K 0x1000 17 #define PAGE_SIZE_64K 0x10000 18 19 #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT 29 20 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK 0x7U 21 22 #define HINIC_MSIX_CNT_SET(val, member) \ 23 (((val) & HINIC_MSIX_CNT_##member##_MASK) << \ 24 HINIC_MSIX_CNT_##member##_SHIFT) 25 26 /** 27 * hwif_ready - test if the HW initialization passed 28 * @hwdev: the pointer to the private hardware device object 29 * Return: 0 - success, negative - failure 30 */ 31 static int hwif_ready(struct hinic_hwdev *hwdev) 32 { 33 u32 addr, attr0, attr1; 34 35 addr = HINIC_CSR_FUNC_ATTR1_ADDR; 36 attr1 = hinic_hwif_read_reg(hwdev->hwif, addr); 37 if (!HINIC_AF1_GET(attr1, MGMT_INIT_STATUS)) 38 return -EBUSY; 39 40 addr = HINIC_CSR_FUNC_ATTR0_ADDR; 41 attr0 = hinic_hwif_read_reg(hwdev->hwif, addr); 42 if ((HINIC_AF0_GET(attr0, FUNC_TYPE) == TYPE_VF) && 43 !HINIC_AF1_GET(attr1, PF_INIT_STATUS)) 44 return -EBUSY; 45 46 return 0; 47 } 48 49 /** 50 * set_hwif_attr - set the attributes as members in hwif 51 * @hwif: the hardware interface of a pci function device 52 * @attr0: the first attribute that was read from the hw 53 * @attr1: the second attribute that was read from the hw 54 * @attr2: the third attribute that was read from the hw 55 */ 56 static void set_hwif_attr(struct hinic_hwif *hwif, u32 attr0, u32 attr1, 57 u32 attr2) 58 { 59 hwif->attr.func_global_idx = HINIC_AF0_GET(attr0, FUNC_GLOBAL_IDX); 60 hwif->attr.port_to_port_idx = HINIC_AF0_GET(attr0, P2P_IDX); 61 hwif->attr.pci_intf_idx = HINIC_AF0_GET(attr0, PCI_INTF_IDX); 62 hwif->attr.vf_in_pf = HINIC_AF0_GET(attr0, VF_IN_PF); 63 hwif->attr.func_type = HINIC_AF0_GET(attr0, FUNC_TYPE); 64 65 hwif->attr.ppf_idx = HINIC_AF1_GET(attr1, PPF_IDX); 66 67 hwif->attr.num_aeqs = BIT(HINIC_AF1_GET(attr1, AEQS_PER_FUNC)); 68 hwif->attr.num_ceqs = BIT(HINIC_AF1_GET(attr1, CEQS_PER_FUNC)); 69 hwif->attr.num_irqs = BIT(HINIC_AF1_GET(attr1, IRQS_PER_FUNC)); 70 hwif->attr.num_dma_attr = BIT(HINIC_AF1_GET(attr1, DMA_ATTR_PER_FUNC)); 71 72 hwif->attr.global_vf_id_of_pf = HINIC_AF2_GET(attr2, 73 GLOBAL_VF_ID_OF_PF); 74 } 75 76 /** 77 * get_hwif_attr - read and set the attributes as members in hwif 78 * @hwif: the hardware interface of a pci function device 79 */ 80 static void get_hwif_attr(struct hinic_hwif *hwif) 81 { 82 u32 addr, attr0, attr1, attr2; 83 84 addr = HINIC_CSR_FUNC_ATTR0_ADDR; 85 attr0 = hinic_hwif_read_reg(hwif, addr); 86 87 addr = HINIC_CSR_FUNC_ATTR1_ADDR; 88 attr1 = hinic_hwif_read_reg(hwif, addr); 89 90 addr = HINIC_CSR_FUNC_ATTR2_ADDR; 91 attr2 = hinic_hwif_read_reg(hwif, addr); 92 93 set_hwif_attr(hwif, attr0, attr1, attr2); 94 } 95 96 void hinic_set_pf_status(struct hinic_hwif *hwif, enum hinic_pf_status status) 97 { 98 u32 attr5 = HINIC_AF5_SET(status, PF_STATUS); 99 u32 addr = HINIC_CSR_FUNC_ATTR5_ADDR; 100 101 if (hwif->attr.func_type == TYPE_VF) { 102 PMD_DRV_LOG(INFO, "VF doesn't support to set attr5"); 103 return; 104 } 105 106 hinic_hwif_write_reg(hwif, addr, attr5); 107 } 108 109 enum hinic_pf_status hinic_get_pf_status(struct hinic_hwif *hwif) 110 { 111 u32 attr5 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR); 112 113 return HINIC_AF5_GET(attr5, PF_STATUS); 114 } 115 116 static enum hinic_doorbell_ctrl 117 hinic_get_doorbell_ctrl_status(struct hinic_hwif *hwif) 118 { 119 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR); 120 121 return HINIC_AF4_GET(attr4, DOORBELL_CTRL); 122 } 123 124 static enum hinic_outbound_ctrl 125 hinic_get_outbound_ctrl_status(struct hinic_hwif *hwif) 126 { 127 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR); 128 129 return HINIC_AF4_GET(attr4, OUTBOUND_CTRL); 130 } 131 132 void hinic_enable_doorbell(struct hinic_hwif *hwif) 133 { 134 u32 addr, attr4; 135 136 addr = HINIC_CSR_FUNC_ATTR4_ADDR; 137 attr4 = hinic_hwif_read_reg(hwif, addr); 138 139 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL); 140 attr4 |= HINIC_AF4_SET(ENABLE_DOORBELL, DOORBELL_CTRL); 141 142 hinic_hwif_write_reg(hwif, addr, attr4); 143 } 144 145 void hinic_disable_doorbell(struct hinic_hwif *hwif) 146 { 147 u32 addr, attr4; 148 149 addr = HINIC_CSR_FUNC_ATTR4_ADDR; 150 attr4 = hinic_hwif_read_reg(hwif, addr); 151 152 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL); 153 attr4 |= HINIC_AF4_SET(DISABLE_DOORBELL, DOORBELL_CTRL); 154 155 hinic_hwif_write_reg(hwif, addr, attr4); 156 } 157 158 /** 159 * set_ppf - try to set hwif as ppf and set the type of hwif in this case 160 * @hwif: the hardware interface of a pci function device 161 */ 162 static void set_ppf(struct hinic_hwif *hwif) 163 { 164 struct hinic_func_attr *attr = &hwif->attr; 165 u32 addr, val, ppf_election; 166 167 /* Read Modify Write */ 168 addr = HINIC_CSR_PPF_ELECTION_ADDR; 169 170 val = hinic_hwif_read_reg(hwif, addr); 171 val = HINIC_PPF_ELECTION_CLEAR(val, IDX); 172 173 ppf_election = HINIC_PPF_ELECTION_SET(attr->func_global_idx, IDX); 174 val |= ppf_election; 175 176 hinic_hwif_write_reg(hwif, addr, val); 177 178 /* Check PPF */ 179 val = hinic_hwif_read_reg(hwif, addr); 180 181 attr->ppf_idx = HINIC_PPF_ELECTION_GET(val, IDX); 182 if (attr->ppf_idx == attr->func_global_idx) 183 attr->func_type = TYPE_PPF; 184 } 185 186 static void init_db_area_idx(struct hinic_hwif *hwif) 187 { 188 struct hinic_free_db_area *free_db_area = &hwif->free_db_area; 189 u32 db_max_areas = hwif->db_max_areas; 190 u32 i; 191 192 for (i = 0; i < db_max_areas; i++) 193 free_db_area->db_idx[i] = i; 194 195 free_db_area->alloc_pos = 0; 196 free_db_area->return_pos = 0; 197 198 free_db_area->num_free = db_max_areas; 199 200 spin_lock_init(&free_db_area->idx_lock); 201 } 202 203 static int get_db_idx(struct hinic_hwif *hwif, u32 *idx) 204 { 205 struct hinic_free_db_area *free_db_area = &hwif->free_db_area; 206 u32 pos; 207 u32 pg_idx; 208 209 spin_lock(&free_db_area->idx_lock); 210 211 if (free_db_area->num_free == 0) { 212 spin_unlock(&free_db_area->idx_lock); 213 return -ENOMEM; 214 } 215 216 free_db_area->num_free--; 217 218 pos = free_db_area->alloc_pos++; 219 pos &= (hwif->db_max_areas - 1); 220 221 pg_idx = free_db_area->db_idx[pos]; 222 223 free_db_area->db_idx[pos] = 0xFFFFFFFF; 224 225 spin_unlock(&free_db_area->idx_lock); 226 227 *idx = pg_idx; 228 229 return 0; 230 } 231 232 static void free_db_idx(struct hinic_hwif *hwif, u32 idx) 233 { 234 struct hinic_free_db_area *free_db_area = &hwif->free_db_area; 235 u32 pos; 236 237 spin_lock(&free_db_area->idx_lock); 238 239 pos = free_db_area->return_pos++; 240 pos &= (hwif->db_max_areas - 1); 241 242 free_db_area->db_idx[pos] = idx; 243 244 free_db_area->num_free++; 245 246 spin_unlock(&free_db_area->idx_lock); 247 } 248 249 void hinic_free_db_addr(void *hwdev, void __iomem *db_base) 250 { 251 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 252 u32 idx = DB_IDX(db_base, hwif->db_base); 253 254 free_db_idx(hwif, idx); 255 } 256 257 int hinic_alloc_db_addr(void *hwdev, void __iomem **db_base) 258 { 259 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 260 u32 idx; 261 int err; 262 263 err = get_db_idx(hwif, &idx); 264 if (err) 265 return -EFAULT; 266 267 *db_base = hwif->db_base + idx * HINIC_DB_PAGE_SIZE; 268 269 return 0; 270 } 271 272 void hinic_set_msix_state(void *hwdev, u16 msix_idx, enum hinic_msix_state flag) 273 { 274 struct hinic_hwdev *hw = hwdev; 275 struct hinic_hwif *hwif = hw->hwif; 276 u32 offset = msix_idx * HINIC_PCI_MSIX_ENTRY_SIZE 277 + HINIC_PCI_MSIX_ENTRY_VECTOR_CTRL; 278 u32 mask_bits; 279 280 /* vfio-pci does not mmap msi-x vector table to user space, 281 * we can not access the space when kernel driver is vfio-pci 282 */ 283 if (hw->pcidev_hdl->kdrv == RTE_PCI_KDRV_VFIO) 284 return; 285 286 mask_bits = readl(hwif->intr_regs_base + offset); 287 mask_bits &= ~HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT; 288 if (flag) 289 mask_bits |= HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT; 290 291 writel(mask_bits, hwif->intr_regs_base + offset); 292 } 293 294 static void disable_all_msix(struct hinic_hwdev *hwdev) 295 { 296 u16 num_irqs = hwdev->hwif->attr.num_irqs; 297 u16 i; 298 299 for (i = 0; i < num_irqs; i++) 300 hinic_set_msix_state(hwdev, i, HINIC_MSIX_DISABLE); 301 } 302 303 /** 304 * Wait for up enable or disable doorbell flush finished. 305 * @hwif: the hardware interface of a pci function device. 306 * @states: Disable or Enable. 307 */ 308 int wait_until_doorbell_flush_states(struct hinic_hwif *hwif, 309 enum hinic_doorbell_ctrl states) 310 { 311 unsigned long end; 312 enum hinic_doorbell_ctrl db_ctrl; 313 314 end = jiffies + 315 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT); 316 do { 317 db_ctrl = hinic_get_doorbell_ctrl_status(hwif); 318 if (db_ctrl == states) 319 return 0; 320 321 rte_delay_ms(1); 322 } while (time_before(jiffies, end)); 323 324 return -ETIMEDOUT; 325 } 326 327 static int wait_until_doorbell_and_outbound_enabled(struct hinic_hwif *hwif) 328 { 329 unsigned long end; 330 enum hinic_doorbell_ctrl db_ctrl; 331 enum hinic_outbound_ctrl outbound_ctrl; 332 333 end = jiffies + 334 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT); 335 do { 336 db_ctrl = hinic_get_doorbell_ctrl_status(hwif); 337 outbound_ctrl = hinic_get_outbound_ctrl_status(hwif); 338 339 if (outbound_ctrl == ENABLE_OUTBOUND && 340 db_ctrl == ENABLE_DOORBELL) 341 return 0; 342 343 rte_delay_ms(1); 344 } while (time_before(jiffies, end)); 345 346 return -ETIMEDOUT; 347 } 348 349 u16 hinic_global_func_id(void *hwdev) 350 { 351 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 352 353 return hwif->attr.func_global_idx; 354 } 355 356 enum func_type hinic_func_type(void *hwdev) 357 { 358 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 359 360 return hwif->attr.func_type; 361 } 362 363 u8 hinic_ppf_idx(void *hwdev) 364 { 365 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 366 367 return hwif->attr.ppf_idx; 368 } 369 370 /** 371 * hinic_dma_attr_entry_num - get number id of DMA attribute table. 372 * @hwdev: the pointer to the private hardware device object. 373 * Return: The number id of DMA attribute table. 374 */ 375 u8 hinic_dma_attr_entry_num(void *hwdev) 376 { 377 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 378 return hwif->attr.num_dma_attr; 379 } 380 381 /** 382 * hinic_init_hwif - initialize the hw interface 383 * @hwdev: the pointer to the private hardware device object 384 * @cfg_reg_base: base physical address of configuration registers 385 * @intr_reg_base: base physical address of msi-x vector table 386 * @db_base_phy: base physical address of doorbell registers 387 * @db_base: base virtual address of doorbell registers 388 * @dwqe_mapping: direct wqe io mapping address 389 * Return: 0 - success, negative - failure 390 */ 391 static int hinic_init_hwif(struct hinic_hwdev *hwdev, void *cfg_reg_base, 392 void *intr_reg_base, u64 db_base_phy, 393 void *db_base, __rte_unused void *dwqe_mapping) 394 { 395 struct hinic_hwif *hwif; 396 struct rte_pci_device *pci_dev; 397 u64 db_bar_len; 398 int err; 399 400 pci_dev = (struct rte_pci_device *)(hwdev->pcidev_hdl); 401 db_bar_len = pci_dev->mem_resource[HINIC_DB_MEM_BAR].len; 402 403 hwif = hwdev->hwif; 404 405 hwif->cfg_regs_base = (u8 __iomem *)cfg_reg_base; 406 hwif->intr_regs_base = (u8 __iomem *)intr_reg_base; 407 408 hwif->db_base_phy = db_base_phy; 409 hwif->db_base = (u8 __iomem *)db_base; 410 hwif->db_max_areas = db_bar_len / HINIC_DB_PAGE_SIZE; 411 if (hwif->db_max_areas > HINIC_DB_MAX_AREAS) 412 hwif->db_max_areas = HINIC_DB_MAX_AREAS; 413 414 init_db_area_idx(hwif); 415 416 get_hwif_attr(hwif); 417 418 err = hwif_ready(hwdev); 419 if (err) { 420 PMD_DRV_LOG(ERR, "Hwif is not ready"); 421 goto hwif_ready_err; 422 } 423 424 err = wait_until_doorbell_and_outbound_enabled(hwif); 425 if (err) { 426 PMD_DRV_LOG(ERR, "Hw doorbell/outbound is disabled"); 427 goto hwif_ready_err; 428 } 429 430 if (!HINIC_IS_VF(hwdev)) 431 set_ppf(hwif); 432 433 /* disable mgmt cpu report any event */ 434 hinic_set_pf_status(hwdev->hwif, HINIC_PF_STATUS_INIT); 435 436 return 0; 437 438 hwif_ready_err: 439 spin_lock_deinit(&hwif->free_db_area.idx_lock); 440 441 return err; 442 } 443 444 #define HINIC_HWIF_ATTR_REG_PRINT_NUM (6) 445 #define HINIC_HWIF_APICMD_REG_PRINT_NUM (2) 446 #define HINIC_HWIF_EQ_REG_PRINT_NUM (2) 447 448 static void hinic_parse_hwif_attr(struct hinic_hwdev *hwdev) 449 { 450 struct hinic_hwif *hwif = hwdev->hwif; 451 452 PMD_DRV_LOG(INFO, "Device %s hwif attribute:", hwdev->pcidev_hdl->name); 453 PMD_DRV_LOG(INFO, "func_idx: %u, p2p_idx: %u, pciintf_idx: %u, " 454 "vf_in_pf: %u, ppf_idx: %u, global_vf_id: %u, func_type: %u", 455 hwif->attr.func_global_idx, 456 hwif->attr.port_to_port_idx, hwif->attr.pci_intf_idx, 457 hwif->attr.vf_in_pf, hwif->attr.ppf_idx, 458 hwif->attr.global_vf_id_of_pf, hwif->attr.func_type); 459 PMD_DRV_LOG(INFO, "num_aeqs:%u, num_ceqs:%u, num_irqs:%u, dma_attr:%u", 460 hwif->attr.num_aeqs, hwif->attr.num_ceqs, 461 hwif->attr.num_irqs, hwif->attr.num_dma_attr); 462 } 463 464 static void hinic_get_mmio(struct hinic_hwdev *hwdev, void **cfg_regs_base, 465 void **intr_base, void **db_base) 466 { 467 struct rte_pci_device *pci_dev = hwdev->pcidev_hdl; 468 uint64_t bar0_size; 469 uint64_t bar2_size; 470 uint64_t bar0_phy_addr; 471 uint64_t pagesize = sysconf(_SC_PAGESIZE); 472 473 *cfg_regs_base = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].addr; 474 *intr_base = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].addr; 475 *db_base = pci_dev->mem_resource[HINIC_DB_MEM_BAR].addr; 476 477 bar0_size = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].len; 478 bar2_size = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].len; 479 480 if (pagesize == PAGE_SIZE_64K && (bar0_size % pagesize != 0)) { 481 bar0_phy_addr = 482 pci_dev->mem_resource[HINIC_CFG_REGS_BAR].phys_addr; 483 if (bar0_phy_addr % pagesize != 0 && 484 (bar0_size + bar2_size <= pagesize) && 485 bar2_size >= bar0_size) { 486 *cfg_regs_base = (void *)((uint8_t *)(*intr_base) 487 + bar2_size); 488 } 489 } 490 } 491 492 void hinic_hwif_res_free(struct hinic_hwdev *hwdev) 493 { 494 rte_free(hwdev->hwif); 495 hwdev->hwif = NULL; 496 } 497 498 int hinic_hwif_res_init(struct hinic_hwdev *hwdev) 499 { 500 int err = HINIC_ERROR; 501 void *cfg_regs_base, *db_base, *intr_base = NULL; 502 503 /* hinic related init */ 504 hwdev->hwif = rte_zmalloc("hinic_hwif", sizeof(*hwdev->hwif), 505 RTE_CACHE_LINE_SIZE); 506 if (!hwdev->hwif) { 507 PMD_DRV_LOG(ERR, "Allocate hwif failed, dev_name: %s", 508 hwdev->pcidev_hdl->name); 509 return -ENOMEM; 510 } 511 512 hinic_get_mmio(hwdev, &cfg_regs_base, &intr_base, &db_base); 513 514 err = hinic_init_hwif(hwdev, cfg_regs_base, 515 intr_base, 0, db_base, NULL); 516 if (err) { 517 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s", 518 hwdev->pcidev_hdl->name); 519 goto init_hwif_err; 520 } 521 522 /* disable msix interrupt in hw device */ 523 disable_all_msix(hwdev); 524 525 /* print hwif attributes */ 526 hinic_parse_hwif_attr(hwdev); 527 528 return HINIC_OK; 529 530 init_hwif_err: 531 rte_free(hwdev->hwif); 532 hwdev->hwif = NULL; 533 534 return err; 535 } 536 537 /** 538 * hinic_misx_intr_clear_resend_bit - clear interrupt resend configuration 539 * @hwdev: the hardware interface of a nic device 540 * @msix_idx: Index of msix interrupt 541 * @clear_resend_en: enable flag of clear resend configuration 542 */ 543 void hinic_misx_intr_clear_resend_bit(void *hwdev, u16 msix_idx, 544 u8 clear_resend_en) 545 { 546 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 547 u32 msix_ctrl = 0, addr; 548 549 msix_ctrl = HINIC_MSIX_CNT_SET(clear_resend_en, RESEND_TIMER); 550 551 addr = HINIC_CSR_MSIX_CNT_ADDR(msix_idx); 552 553 hinic_hwif_write_reg(hwif, addr, msix_ctrl); 554 } 555