1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Huawei Technologies Co., Ltd 3 */ 4 5 #include <rte_bus_pci.h> 6 7 #include "hinic_compat.h" 8 #include "hinic_csr.h" 9 #include "hinic_pmd_hwdev.h" 10 #include "hinic_pmd_hwif.h" 11 12 #define HINIC_CFG_REGS_BAR 0 13 #define HINIC_INTR_MSI_BAR 2 14 #define HINIC_DB_MEM_BAR 4 15 16 #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT 29 17 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK 0x7U 18 19 #define HINIC_MSIX_CNT_SET(val, member) \ 20 (((val) & HINIC_MSIX_CNT_##member##_MASK) << \ 21 HINIC_MSIX_CNT_##member##_SHIFT) 22 23 /** 24 * hwif_ready - test if the HW initialization passed 25 * @hwdev: the pointer to the private hardware device object 26 * Return: 0 - success, negative - failure 27 **/ 28 static int hwif_ready(struct hinic_hwdev *hwdev) 29 { 30 u32 addr, attr1; 31 32 addr = HINIC_CSR_FUNC_ATTR1_ADDR; 33 attr1 = hinic_hwif_read_reg(hwdev->hwif, addr); 34 35 if (!HINIC_AF1_GET(attr1, MGMT_INIT_STATUS)) 36 return -EBUSY; 37 38 return 0; 39 } 40 41 /** 42 * set_hwif_attr - set the attributes as members in hwif 43 * @hwif: the hardware interface of a pci function device 44 * @attr0: the first attribute that was read from the hw 45 * @attr1: the second attribute that was read from the hw 46 * @attr2: the third attribute that was read from the hw 47 **/ 48 static void set_hwif_attr(struct hinic_hwif *hwif, u32 attr0, u32 attr1, 49 u32 attr2) 50 { 51 hwif->attr.func_global_idx = HINIC_AF0_GET(attr0, FUNC_GLOBAL_IDX); 52 hwif->attr.port_to_port_idx = HINIC_AF0_GET(attr0, P2P_IDX); 53 hwif->attr.pci_intf_idx = HINIC_AF0_GET(attr0, PCI_INTF_IDX); 54 hwif->attr.vf_in_pf = HINIC_AF0_GET(attr0, VF_IN_PF); 55 hwif->attr.func_type = HINIC_AF0_GET(attr0, FUNC_TYPE); 56 57 hwif->attr.ppf_idx = HINIC_AF1_GET(attr1, PPF_IDX); 58 59 hwif->attr.num_aeqs = BIT(HINIC_AF1_GET(attr1, AEQS_PER_FUNC)); 60 hwif->attr.num_ceqs = BIT(HINIC_AF1_GET(attr1, CEQS_PER_FUNC)); 61 hwif->attr.num_irqs = BIT(HINIC_AF1_GET(attr1, IRQS_PER_FUNC)); 62 hwif->attr.num_dma_attr = BIT(HINIC_AF1_GET(attr1, DMA_ATTR_PER_FUNC)); 63 64 hwif->attr.global_vf_id_of_pf = HINIC_AF2_GET(attr2, 65 GLOBAL_VF_ID_OF_PF); 66 } 67 68 /** 69 * get_hwif_attr - read and set the attributes as members in hwif 70 * @hwif: the hardware interface of a pci function device 71 **/ 72 static void get_hwif_attr(struct hinic_hwif *hwif) 73 { 74 u32 addr, attr0, attr1, attr2; 75 76 addr = HINIC_CSR_FUNC_ATTR0_ADDR; 77 attr0 = hinic_hwif_read_reg(hwif, addr); 78 79 addr = HINIC_CSR_FUNC_ATTR1_ADDR; 80 attr1 = hinic_hwif_read_reg(hwif, addr); 81 82 addr = HINIC_CSR_FUNC_ATTR2_ADDR; 83 attr2 = hinic_hwif_read_reg(hwif, addr); 84 85 set_hwif_attr(hwif, attr0, attr1, attr2); 86 } 87 88 void hinic_set_pf_status(struct hinic_hwif *hwif, enum hinic_pf_status status) 89 { 90 u32 attr5 = HINIC_AF5_SET(status, PF_STATUS); 91 u32 addr = HINIC_CSR_FUNC_ATTR5_ADDR; 92 93 hinic_hwif_write_reg(hwif, addr, attr5); 94 } 95 96 enum hinic_pf_status hinic_get_pf_status(struct hinic_hwif *hwif) 97 { 98 u32 attr5 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR); 99 100 return HINIC_AF5_GET(attr5, PF_STATUS); 101 } 102 103 static enum hinic_doorbell_ctrl 104 hinic_get_doorbell_ctrl_status(struct hinic_hwif *hwif) 105 { 106 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR); 107 108 return HINIC_AF4_GET(attr4, DOORBELL_CTRL); 109 } 110 111 static enum hinic_outbound_ctrl 112 hinic_get_outbound_ctrl_status(struct hinic_hwif *hwif) 113 { 114 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR); 115 116 return HINIC_AF4_GET(attr4, OUTBOUND_CTRL); 117 } 118 119 void hinic_enable_doorbell(struct hinic_hwif *hwif) 120 { 121 u32 addr, attr4; 122 123 addr = HINIC_CSR_FUNC_ATTR4_ADDR; 124 attr4 = hinic_hwif_read_reg(hwif, addr); 125 126 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL); 127 attr4 |= HINIC_AF4_SET(ENABLE_DOORBELL, DOORBELL_CTRL); 128 129 hinic_hwif_write_reg(hwif, addr, attr4); 130 } 131 132 void hinic_disable_doorbell(struct hinic_hwif *hwif) 133 { 134 u32 addr, attr4; 135 136 addr = HINIC_CSR_FUNC_ATTR4_ADDR; 137 attr4 = hinic_hwif_read_reg(hwif, addr); 138 139 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL); 140 attr4 |= HINIC_AF4_SET(DISABLE_DOORBELL, DOORBELL_CTRL); 141 142 hinic_hwif_write_reg(hwif, addr, attr4); 143 } 144 145 /** 146 * set_ppf - try to set hwif as ppf and set the type of hwif in this case 147 * @hwif: the hardware interface of a pci function device 148 **/ 149 static void set_ppf(struct hinic_hwif *hwif) 150 { 151 struct hinic_func_attr *attr = &hwif->attr; 152 u32 addr, val, ppf_election; 153 154 /* Read Modify Write */ 155 addr = HINIC_CSR_PPF_ELECTION_ADDR; 156 157 val = hinic_hwif_read_reg(hwif, addr); 158 val = HINIC_PPF_ELECTION_CLEAR(val, IDX); 159 160 ppf_election = HINIC_PPF_ELECTION_SET(attr->func_global_idx, IDX); 161 val |= ppf_election; 162 163 hinic_hwif_write_reg(hwif, addr, val); 164 165 /* Check PPF */ 166 val = hinic_hwif_read_reg(hwif, addr); 167 168 attr->ppf_idx = HINIC_PPF_ELECTION_GET(val, IDX); 169 if (attr->ppf_idx == attr->func_global_idx) 170 attr->func_type = TYPE_PPF; 171 } 172 173 static void init_db_area_idx(struct hinic_free_db_area *free_db_area) 174 { 175 u32 i; 176 177 for (i = 0; i < HINIC_DB_MAX_AREAS; i++) 178 free_db_area->db_idx[i] = i; 179 180 free_db_area->alloc_pos = 0; 181 free_db_area->return_pos = 0; 182 183 free_db_area->num_free = HINIC_DB_MAX_AREAS; 184 185 spin_lock_init(&free_db_area->idx_lock); 186 } 187 188 static int get_db_idx(struct hinic_hwif *hwif, u32 *idx) 189 { 190 struct hinic_free_db_area *free_db_area = &hwif->free_db_area; 191 u32 pos; 192 u32 pg_idx; 193 194 spin_lock(&free_db_area->idx_lock); 195 196 if (free_db_area->num_free == 0) { 197 spin_unlock(&free_db_area->idx_lock); 198 return -ENOMEM; 199 } 200 201 free_db_area->num_free--; 202 203 pos = free_db_area->alloc_pos++; 204 pos &= HINIC_DB_MAX_AREAS - 1; 205 206 pg_idx = free_db_area->db_idx[pos]; 207 208 free_db_area->db_idx[pos] = 0xFFFFFFFF; 209 210 spin_unlock(&free_db_area->idx_lock); 211 212 *idx = pg_idx; 213 214 return 0; 215 } 216 217 static void free_db_idx(struct hinic_hwif *hwif, u32 idx) 218 { 219 struct hinic_free_db_area *free_db_area = &hwif->free_db_area; 220 u32 pos; 221 222 spin_lock(&free_db_area->idx_lock); 223 224 pos = free_db_area->return_pos++; 225 pos &= HINIC_DB_MAX_AREAS - 1; 226 227 free_db_area->db_idx[pos] = idx; 228 229 free_db_area->num_free++; 230 231 spin_unlock(&free_db_area->idx_lock); 232 } 233 234 void hinic_free_db_addr(void *hwdev, void __iomem *db_base) 235 { 236 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 237 u32 idx = DB_IDX(db_base, hwif->db_base); 238 239 free_db_idx(hwif, idx); 240 } 241 242 int hinic_alloc_db_addr(void *hwdev, void __iomem **db_base) 243 { 244 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 245 u32 idx; 246 int err; 247 248 err = get_db_idx(hwif, &idx); 249 if (err) 250 return -EFAULT; 251 252 *db_base = hwif->db_base + idx * HINIC_DB_PAGE_SIZE; 253 254 return 0; 255 } 256 257 void hinic_set_msix_state(void *hwdev, u16 msix_idx, enum hinic_msix_state flag) 258 { 259 struct hinic_hwdev *hw = hwdev; 260 struct hinic_hwif *hwif = hw->hwif; 261 u32 offset = msix_idx * HINIC_PCI_MSIX_ENTRY_SIZE 262 + HINIC_PCI_MSIX_ENTRY_VECTOR_CTRL; 263 u32 mask_bits; 264 265 /* vfio-pci does not mmap msi-x vector table to user space, 266 * we can not access the space when kernel driver is vfio-pci 267 */ 268 if (hw->pcidev_hdl->kdrv == RTE_KDRV_VFIO) 269 return; 270 271 mask_bits = readl(hwif->intr_regs_base + offset); 272 mask_bits &= ~HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT; 273 if (flag) 274 mask_bits |= HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT; 275 276 writel(mask_bits, hwif->intr_regs_base + offset); 277 } 278 279 static void disable_all_msix(struct hinic_hwdev *hwdev) 280 { 281 u16 num_irqs = hwdev->hwif->attr.num_irqs; 282 u16 i; 283 284 for (i = 0; i < num_irqs; i++) 285 hinic_set_msix_state(hwdev, i, HINIC_MSIX_DISABLE); 286 } 287 288 static int wait_until_doorbell_and_outbound_enabled(struct hinic_hwif *hwif) 289 { 290 unsigned long end; 291 enum hinic_doorbell_ctrl db_ctrl; 292 enum hinic_outbound_ctrl outbound_ctrl; 293 294 end = jiffies + 295 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT); 296 do { 297 db_ctrl = hinic_get_doorbell_ctrl_status(hwif); 298 outbound_ctrl = hinic_get_outbound_ctrl_status(hwif); 299 300 if (outbound_ctrl == ENABLE_OUTBOUND && 301 db_ctrl == ENABLE_DOORBELL) 302 return 0; 303 304 rte_delay_ms(1); 305 } while (time_before(jiffies, end)); 306 307 return -EFAULT; 308 } 309 310 u16 hinic_global_func_id(void *hwdev) 311 { 312 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 313 314 return hwif->attr.func_global_idx; 315 } 316 317 enum func_type hinic_func_type(void *hwdev) 318 { 319 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 320 321 return hwif->attr.func_type; 322 } 323 324 u8 hinic_ppf_idx(void *hwdev) 325 { 326 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 327 328 return hwif->attr.ppf_idx; 329 } 330 331 /** 332 * hinic_init_hwif - initialize the hw interface 333 * @hwdev: the pointer to the private hardware device object 334 * @cfg_reg_base: base physical address of configuration registers 335 * @intr_reg_base: base physical address of msi-x vector table 336 * @db_base_phy: base physical address of doorbell registers 337 * @db_base: base virtual address of doorbell registers 338 * @dwqe_mapping: direct wqe io mapping address 339 * Return: 0 - success, negative - failure 340 **/ 341 static int hinic_init_hwif(struct hinic_hwdev *hwdev, void *cfg_reg_base, 342 void *intr_reg_base, u64 db_base_phy, 343 void *db_base, __rte_unused void *dwqe_mapping) 344 { 345 struct hinic_hwif *hwif; 346 int err; 347 348 hwif = hwdev->hwif; 349 350 hwif->cfg_regs_base = (u8 __iomem *)cfg_reg_base; 351 hwif->intr_regs_base = (u8 __iomem *)intr_reg_base; 352 353 hwif->db_base_phy = db_base_phy; 354 hwif->db_base = (u8 __iomem *)db_base; 355 init_db_area_idx(&hwif->free_db_area); 356 357 get_hwif_attr(hwif); 358 359 err = hwif_ready(hwdev); 360 if (err) { 361 PMD_DRV_LOG(ERR, "Hwif is not ready"); 362 goto hwif_ready_err; 363 } 364 365 err = wait_until_doorbell_and_outbound_enabled(hwif); 366 if (err) { 367 PMD_DRV_LOG(ERR, "Hw doorbell/outbound is disabled"); 368 goto hwif_ready_err; 369 } 370 371 if (!HINIC_IS_VF(hwdev)) 372 set_ppf(hwif); 373 374 return 0; 375 376 hwif_ready_err: 377 spin_lock_deinit(&hwif->free_db_area.idx_lock); 378 379 return err; 380 } 381 382 #define HINIC_HWIF_ATTR_REG_PRINT_NUM (6) 383 #define HINIC_HWIF_APICMD_REG_PRINT_NUM (2) 384 #define HINIC_HWIF_EQ_REG_PRINT_NUM (2) 385 386 static void hinic_parse_hwif_attr(struct hinic_hwdev *hwdev) 387 { 388 struct hinic_hwif *hwif = hwdev->hwif; 389 390 PMD_DRV_LOG(INFO, "Device %s hwif attribute:", hwdev->pcidev_hdl->name); 391 PMD_DRV_LOG(INFO, "func_idx:%u, p2p_idx:%u, pciintf_idx:%u, " 392 "vf_in_pf:%u, ppf_idx:%u, global_vf_id:%u, func_type:%u", 393 hwif->attr.func_global_idx, 394 hwif->attr.port_to_port_idx, hwif->attr.pci_intf_idx, 395 hwif->attr.vf_in_pf, hwif->attr.ppf_idx, 396 hwif->attr.global_vf_id_of_pf, hwif->attr.func_type); 397 PMD_DRV_LOG(INFO, "num_aeqs:%u, num_ceqs:%u, num_irqs:%u, dma_attr:%u", 398 hwif->attr.num_aeqs, hwif->attr.num_ceqs, 399 hwif->attr.num_irqs, hwif->attr.num_dma_attr); 400 } 401 402 static void hinic_get_mmio(struct hinic_hwdev *hwdev, void **cfg_regs_base, 403 void **intr_base, void **db_base) 404 { 405 struct rte_pci_device *pci_dev = hwdev->pcidev_hdl; 406 407 *cfg_regs_base = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].addr; 408 *intr_base = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].addr; 409 *db_base = pci_dev->mem_resource[HINIC_DB_MEM_BAR].addr; 410 } 411 412 void hinic_hwif_res_free(struct hinic_hwdev *hwdev) 413 { 414 rte_free(hwdev->hwif); 415 hwdev->hwif = NULL; 416 } 417 418 int hinic_hwif_res_init(struct hinic_hwdev *hwdev) 419 { 420 int err = HINIC_ERROR; 421 void *cfg_regs_base, *db_base, *intr_base = NULL; 422 423 /* hinic related init */ 424 hwdev->hwif = rte_zmalloc("hinic_hwif", sizeof(*hwdev->hwif), 425 RTE_CACHE_LINE_SIZE); 426 if (!hwdev->hwif) { 427 PMD_DRV_LOG(ERR, "Allocate hwif failed, dev_name: %s", 428 hwdev->pcidev_hdl->name); 429 return -ENOMEM; 430 } 431 432 hinic_get_mmio(hwdev, &cfg_regs_base, &intr_base, &db_base); 433 434 err = hinic_init_hwif(hwdev, cfg_regs_base, 435 intr_base, 0, db_base, NULL); 436 if (err) { 437 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s", 438 hwdev->pcidev_hdl->name); 439 goto init_hwif_err; 440 } 441 442 /* disable msix interrupt in hw device */ 443 disable_all_msix(hwdev); 444 445 /* print hwif attributes */ 446 hinic_parse_hwif_attr(hwdev); 447 448 return HINIC_OK; 449 450 init_hwif_err: 451 rte_free(hwdev->hwif); 452 hwdev->hwif = NULL; 453 454 return err; 455 } 456 457 /** 458 * hinic_misx_intr_clear_resend_bit - clear interrupt resend configuration 459 * @hwdev: the hardware interface of a nic device 460 * @msix_idx: Index of msix interrupt 461 * @clear_resend_en: enable flag of clear resend configuration 462 **/ 463 void hinic_misx_intr_clear_resend_bit(void *hwdev, u16 msix_idx, 464 u8 clear_resend_en) 465 { 466 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif; 467 u32 msix_ctrl = 0, addr; 468 469 msix_ctrl = HINIC_MSIX_CNT_SET(clear_resend_en, RESEND_TIMER); 470 471 addr = HINIC_CSR_MSIX_CNT_ADDR(msix_idx); 472 473 hinic_hwif_write_reg(hwif, addr, msix_ctrl); 474 } 475