1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 5 #include "opae_hw_api.h" 6 #include "ifpga_api.h" 7 8 #include "ifpga_hw.h" 9 #include "ifpga_enumerate.h" 10 #include "ifpga_feature_dev.h" 11 12 struct build_feature_devs_info { 13 struct opae_adapter_data_pci *pci_data; 14 15 struct ifpga_afu_info *acc_info; 16 17 void *fiu; 18 enum fpga_id_type current_type; 19 int current_port_id; 20 21 void *ioaddr; 22 void *ioend; 23 uint64_t phys_addr; 24 int current_bar; 25 26 void *pfme_hdr; 27 28 struct ifpga_hw *hw; 29 }; 30 31 static int feature_revision(void __iomem *start) 32 { 33 struct feature_header header; 34 35 header.csr = readq(start); 36 37 return header.revision; 38 } 39 40 static u32 feature_size(void __iomem *start) 41 { 42 struct feature_header header; 43 44 header.csr = readq(start); 45 46 /*the size of private feature is 4KB aligned*/ 47 return header.next_header_offset ? header.next_header_offset:4096; 48 } 49 50 static u64 feature_id(void __iomem *start) 51 { 52 struct feature_header header; 53 54 header.csr = readq(start); 55 56 switch (header.type) { 57 case FEATURE_TYPE_FIU: 58 return FEATURE_ID_FIU_HEADER; 59 case FEATURE_TYPE_PRIVATE: 60 return header.id; 61 case FEATURE_TYPE_AFU: 62 return FEATURE_ID_AFU; 63 } 64 65 WARN_ON(1); 66 return 0; 67 } 68 69 static int 70 build_info_add_sub_feature(struct build_feature_devs_info *binfo, 71 void __iomem *start, u64 fid, unsigned int size, 72 unsigned int vec_start, 73 unsigned int vec_cnt) 74 { 75 struct ifpga_hw *hw = binfo->hw; 76 struct ifpga_feature *feature = NULL; 77 struct feature_irq_ctx *ctx = NULL; 78 int port_id, ret = 0; 79 unsigned int i; 80 81 fid = fid?fid:feature_id(start); 82 size = size?size:feature_size(start); 83 84 feature = opae_malloc(sizeof(struct ifpga_feature)); 85 if (!feature) 86 return -ENOMEM; 87 88 feature->state = IFPGA_FEATURE_ATTACHED; 89 feature->addr = start; 90 feature->id = fid; 91 feature->size = size; 92 feature->revision = feature_revision(start); 93 feature->phys_addr = binfo->phys_addr + 94 ((u8 *)start - (u8 *)binfo->ioaddr); 95 feature->vec_start = vec_start; 96 feature->vec_cnt = vec_cnt; 97 98 dev_debug(binfo, "%s: id=0x%llx, phys_addr=0x%llx, size=%u\n", 99 __func__, (unsigned long long)feature->id, 100 (unsigned long long)feature->phys_addr, size); 101 102 if (vec_cnt) { 103 if (vec_start + vec_cnt <= vec_start) 104 return -EINVAL; 105 106 ctx = zmalloc(sizeof(*ctx) * vec_cnt); 107 if (!ctx) 108 return -ENOMEM; 109 110 for (i = 0; i < vec_cnt; i++) { 111 ctx[i].eventfd = -1; 112 ctx[i].idx = vec_start + i; 113 } 114 } 115 116 feature->ctx = ctx; 117 feature->ctx_num = vec_cnt; 118 feature->vfio_dev_fd = binfo->pci_data->vfio_dev_fd; 119 120 if (binfo->current_type == FME_ID) { 121 feature->parent = &hw->fme; 122 feature->type = FEATURE_FME_TYPE; 123 feature->name = get_fme_feature_name(fid); 124 TAILQ_INSERT_TAIL(&hw->fme.feature_list, feature, next); 125 } else if (binfo->current_type == PORT_ID) { 126 port_id = binfo->current_port_id; 127 feature->parent = &hw->port[port_id]; 128 feature->type = FEATURE_PORT_TYPE; 129 feature->name = get_port_feature_name(fid); 130 TAILQ_INSERT_TAIL(&hw->port[port_id].feature_list, 131 feature, next); 132 } else { 133 return -EFAULT; 134 } 135 return ret; 136 } 137 138 static int 139 create_feature_instance(struct build_feature_devs_info *binfo, 140 void __iomem *start, u64 fid, 141 unsigned int size, unsigned int vec_start, 142 unsigned int vec_cnt) 143 { 144 return build_info_add_sub_feature(binfo, start, fid, size, vec_start, 145 vec_cnt); 146 } 147 148 /* 149 * UAFU GUID is dynamic as it can be changed after FME downloads different 150 * Green Bitstream to the port, so we treat the unknown GUIDs which are 151 * attached on port's feature list as UAFU. 152 */ 153 static bool feature_is_UAFU(struct build_feature_devs_info *binfo) 154 { 155 if (binfo->current_type != PORT_ID) 156 return false; 157 158 return true; 159 } 160 161 static int parse_feature_port_uafu(struct build_feature_devs_info *binfo, 162 struct feature_header *hdr) 163 { 164 u64 id = PORT_FEATURE_ID_UAFU; 165 struct ifpga_afu_info *info; 166 void *start = (void *)hdr; 167 struct feature_port_header *port_hdr = binfo->ioaddr; 168 struct feature_port_capability capability; 169 int ret; 170 int size; 171 172 capability.csr = readq(&port_hdr->capability); 173 174 size = capability.mmio_size << 10; 175 176 ret = create_feature_instance(binfo, hdr, id, size, 0, 0); 177 if (ret) 178 return ret; 179 180 info = opae_malloc(sizeof(*info)); 181 if (!info) 182 return -ENOMEM; 183 184 info->region[0].addr = start; 185 info->region[0].phys_addr = binfo->phys_addr + 186 (uint8_t *)start - (uint8_t *)binfo->ioaddr; 187 info->region[0].len = size; 188 info->num_regions = 1; 189 190 binfo->acc_info = info; 191 192 return ret; 193 } 194 195 static int parse_feature_afus(struct build_feature_devs_info *binfo, 196 struct feature_header *hdr) 197 { 198 int ret; 199 struct feature_afu_header *afu_hdr, header; 200 u8 __iomem *start; 201 u8 __iomem *end = binfo->ioend; 202 203 start = (u8 __iomem *)hdr; 204 for (; start < end; start += header.next_afu) { 205 if ((unsigned int)(end - start) < 206 (unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr))) 207 return -EINVAL; 208 209 hdr = (struct feature_header *)start; 210 afu_hdr = (struct feature_afu_header *)(hdr + 1); 211 header.csr = readq(&afu_hdr->csr); 212 213 if (feature_is_UAFU(binfo)) { 214 ret = parse_feature_port_uafu(binfo, hdr); 215 if (ret) 216 return ret; 217 } 218 219 if (!header.next_afu) 220 break; 221 } 222 223 return 0; 224 } 225 226 /* create and register proper private data */ 227 static int build_info_commit_dev(struct build_feature_devs_info *binfo) 228 { 229 struct ifpga_afu_info *info = binfo->acc_info; 230 struct ifpga_hw *hw = binfo->hw; 231 struct opae_manager *mgr; 232 struct opae_bridge *br; 233 struct opae_accelerator *acc; 234 struct ifpga_port_hw *port; 235 struct ifpga_fme_hw *fme; 236 struct ifpga_feature *feature; 237 238 if (!binfo->fiu) 239 return 0; 240 241 if (binfo->current_type == PORT_ID) { 242 /* return error if no valid acc info data structure */ 243 if (!info) 244 return -EFAULT; 245 246 br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops, 247 binfo->fiu); 248 if (!br) 249 return -ENOMEM; 250 251 br->id = binfo->current_port_id; 252 253 /* update irq info */ 254 port = &hw->port[binfo->current_port_id]; 255 feature = get_feature_by_id(&port->feature_list, 256 PORT_FEATURE_ID_UINT); 257 if (feature) 258 info->num_irqs = feature->vec_cnt; 259 260 acc = opae_accelerator_alloc(hw->adapter->name, 261 &ifpga_acc_ops, info); 262 if (!acc) { 263 opae_bridge_free(br); 264 return -ENOMEM; 265 } 266 267 acc->br = br; 268 if (hw->adapter->mgr) 269 acc->mgr = hw->adapter->mgr; 270 acc->index = br->id; 271 272 fme = &hw->fme; 273 fme->nums_acc_region = info->num_regions; 274 275 opae_adapter_add_acc(hw->adapter, acc); 276 277 } else if (binfo->current_type == FME_ID) { 278 mgr = opae_manager_alloc(hw->adapter->name, &ifpga_mgr_ops, 279 &ifpga_mgr_network_ops, binfo->fiu); 280 if (!mgr) 281 return -ENOMEM; 282 283 mgr->adapter = hw->adapter; 284 hw->adapter->mgr = mgr; 285 } 286 287 binfo->fiu = NULL; 288 289 return 0; 290 } 291 292 static int 293 build_info_create_dev(struct build_feature_devs_info *binfo, 294 enum fpga_id_type type, unsigned int index) 295 { 296 int ret; 297 298 ret = build_info_commit_dev(binfo); 299 if (ret) 300 return ret; 301 302 binfo->current_type = type; 303 304 if (type == FME_ID) { 305 binfo->fiu = &binfo->hw->fme; 306 } else if (type == PORT_ID) { 307 binfo->fiu = &binfo->hw->port[index]; 308 binfo->current_port_id = index; 309 } 310 311 return 0; 312 } 313 314 static int parse_feature_fme(struct build_feature_devs_info *binfo, 315 struct feature_header *start) 316 { 317 struct ifpga_hw *hw = binfo->hw; 318 struct ifpga_fme_hw *fme = &hw->fme; 319 int ret; 320 321 ret = build_info_create_dev(binfo, FME_ID, 0); 322 if (ret) 323 return ret; 324 325 /* Update FME states */ 326 fme->state = IFPGA_FME_IMPLEMENTED; 327 fme->parent = hw; 328 TAILQ_INIT(&fme->feature_list); 329 spinlock_init(&fme->lock); 330 331 return create_feature_instance(binfo, start, 0, 0, 0, 0); 332 } 333 334 static int parse_feature_port(struct build_feature_devs_info *binfo, 335 void __iomem *start) 336 { 337 struct feature_port_header *port_hdr; 338 struct feature_port_capability capability; 339 struct ifpga_hw *hw = binfo->hw; 340 struct ifpga_port_hw *port; 341 unsigned int port_id; 342 int ret; 343 344 /* Get current port's id */ 345 port_hdr = (struct feature_port_header *)start; 346 capability.csr = readq(&port_hdr->capability); 347 port_id = capability.port_number; 348 349 ret = build_info_create_dev(binfo, PORT_ID, port_id); 350 if (ret) 351 return ret; 352 353 /*found a Port device*/ 354 port = &hw->port[port_id]; 355 port->port_id = binfo->current_port_id; 356 port->parent = hw; 357 port->state = IFPGA_PORT_ATTACHED; 358 spinlock_init(&port->lock); 359 TAILQ_INIT(&port->feature_list); 360 361 return create_feature_instance(binfo, start, 0, 0, 0, 0); 362 } 363 364 static void enable_port_uafu(struct build_feature_devs_info *binfo, 365 void __iomem *start) 366 { 367 struct ifpga_port_hw *port = &binfo->hw->port[binfo->current_port_id]; 368 369 UNUSED(start); 370 371 fpga_port_reset(port); 372 } 373 374 static int parse_feature_fiu(struct build_feature_devs_info *binfo, 375 struct feature_header *hdr) 376 { 377 struct feature_header header; 378 struct feature_fiu_header *fiu_hdr, fiu_header; 379 u8 __iomem *start = (u8 __iomem *)hdr; 380 int ret; 381 382 header.csr = readq(hdr); 383 384 switch (header.id) { 385 case FEATURE_FIU_ID_FME: 386 ret = parse_feature_fme(binfo, hdr); 387 binfo->pfme_hdr = hdr; 388 if (ret) 389 return ret; 390 break; 391 case FEATURE_FIU_ID_PORT: 392 ret = parse_feature_port(binfo, hdr); 393 enable_port_uafu(binfo, hdr); 394 if (ret) 395 return ret; 396 397 /* Check Port FIU's next_afu pointer to User AFU DFH */ 398 fiu_hdr = (struct feature_fiu_header *)(hdr + 1); 399 fiu_header.csr = readq(&fiu_hdr->csr); 400 401 if (fiu_header.next_afu) { 402 start += fiu_header.next_afu; 403 ret = parse_feature_afus(binfo, 404 (struct feature_header *)start); 405 if (ret) 406 return ret; 407 } else { 408 dev_info(binfo, "No AFUs detected on Port\n"); 409 } 410 411 break; 412 default: 413 dev_info(binfo, "FIU TYPE %d is not supported yet.\n", 414 header.id); 415 } 416 417 return 0; 418 } 419 420 static void parse_feature_irqs(struct build_feature_devs_info *binfo, 421 void __iomem *start, unsigned int *vec_start, 422 unsigned int *vec_cnt) 423 { 424 UNUSED(binfo); 425 u64 id; 426 427 id = feature_id(start); 428 429 if (id == PORT_FEATURE_ID_UINT) { 430 struct feature_port_uint *port_uint = start; 431 struct feature_port_uint_cap uint_cap; 432 433 uint_cap.csr = readq(&port_uint->capability); 434 if (uint_cap.intr_num) { 435 *vec_start = uint_cap.first_vec_num; 436 *vec_cnt = uint_cap.intr_num; 437 } else { 438 dev_debug(binfo, "UAFU doesn't support interrupt\n"); 439 } 440 } else if (id == PORT_FEATURE_ID_ERROR) { 441 struct feature_port_error *port_err = start; 442 struct feature_port_err_capability port_err_cap; 443 444 port_err_cap.csr = readq(&port_err->error_capability); 445 if (port_err_cap.support_intr) { 446 *vec_start = port_err_cap.intr_vector_num; 447 *vec_cnt = 1; 448 } else { 449 dev_debug(&binfo, "Port error doesn't support interrupt\n"); 450 } 451 452 } else if (id == FME_FEATURE_ID_GLOBAL_ERR) { 453 struct feature_fme_err *fme_err = start; 454 struct feature_fme_error_capability fme_err_cap; 455 456 fme_err_cap.csr = readq(&fme_err->fme_err_capability); 457 if (fme_err_cap.support_intr) { 458 *vec_start = fme_err_cap.intr_vector_num; 459 *vec_cnt = 1; 460 } else { 461 dev_debug(&binfo, "FME error doesn't support interrupt\n"); 462 } 463 } 464 } 465 466 static int parse_feature_fme_private(struct build_feature_devs_info *binfo, 467 struct feature_header *hdr) 468 { 469 unsigned int vec_start = 0; 470 unsigned int vec_cnt = 0; 471 472 parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt); 473 474 return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt); 475 } 476 477 static int parse_feature_port_private(struct build_feature_devs_info *binfo, 478 struct feature_header *hdr) 479 { 480 unsigned int vec_start = 0; 481 unsigned int vec_cnt = 0; 482 483 parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt); 484 485 return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt); 486 } 487 488 static int parse_feature_private(struct build_feature_devs_info *binfo, 489 struct feature_header *hdr) 490 { 491 struct feature_header header; 492 493 header.csr = readq(hdr); 494 495 switch (binfo->current_type) { 496 case FME_ID: 497 return parse_feature_fme_private(binfo, hdr); 498 case PORT_ID: 499 return parse_feature_port_private(binfo, hdr); 500 default: 501 dev_err(binfo, "private feature %x belonging to AFU %d (unknown_type) is not supported yet.\n", 502 header.id, binfo->current_type); 503 } 504 return 0; 505 } 506 507 static int parse_feature(struct build_feature_devs_info *binfo, 508 struct feature_header *hdr) 509 { 510 struct feature_header header; 511 int ret = 0; 512 513 header.csr = readq(hdr); 514 515 switch (header.type) { 516 case FEATURE_TYPE_AFU: 517 ret = parse_feature_afus(binfo, hdr); 518 break; 519 case FEATURE_TYPE_PRIVATE: 520 ret = parse_feature_private(binfo, hdr); 521 break; 522 case FEATURE_TYPE_FIU: 523 ret = parse_feature_fiu(binfo, hdr); 524 break; 525 default: 526 dev_err(binfo, "Feature Type %x is not supported.\n", 527 hdr->type); 528 }; 529 530 return ret; 531 } 532 533 static int 534 parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem *start) 535 { 536 struct feature_header *hdr, header; 537 u8 __iomem *end = (u8 __iomem *)binfo->ioend; 538 int ret = 0; 539 540 for (; start < end; start += header.next_header_offset) { 541 if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) { 542 dev_err(binfo, "The region is too small to contain a feature.\n"); 543 ret = -EINVAL; 544 break; 545 } 546 547 hdr = (struct feature_header *)start; 548 header.csr = readq(hdr); 549 550 dev_debug(binfo, "%s: address=0x%p, val=0x%llx, header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x, header.type=0x%x\n", 551 __func__, hdr, (unsigned long long)header.csr, 552 header.id, header.next_header_offset, 553 header.end_of_list, header.type); 554 555 ret = parse_feature(binfo, hdr); 556 if (ret) 557 return ret; 558 559 if (header.end_of_list || !header.next_header_offset) 560 break; 561 } 562 563 return build_info_commit_dev(binfo); 564 } 565 566 /* switch the memory mapping to BAR# @bar */ 567 static int parse_switch_to(struct build_feature_devs_info *binfo, int bar) 568 { 569 struct opae_adapter_data_pci *pci_data = binfo->pci_data; 570 571 if (!pci_data->region[bar].addr) 572 return -ENOMEM; 573 574 binfo->ioaddr = pci_data->region[bar].addr; 575 binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data->region[bar].len; 576 binfo->phys_addr = pci_data->region[bar].phys_addr; 577 binfo->current_bar = bar; 578 579 return 0; 580 } 581 582 static int parse_ports_from_fme(struct build_feature_devs_info *binfo) 583 { 584 struct feature_fme_header *fme_hdr; 585 struct feature_fme_port port; 586 int i = 0, ret = 0; 587 588 if (!binfo->pfme_hdr) { 589 dev_info(binfo, "VF is detected.\n"); 590 return ret; 591 } 592 593 fme_hdr = binfo->pfme_hdr; 594 595 do { 596 port.csr = readq(&fme_hdr->port[i]); 597 if (!port.port_implemented) 598 break; 599 600 /* skip port which only could be accessed via VF */ 601 if (port.afu_access_control == FME_AFU_ACCESS_VF) 602 continue; 603 604 ret = parse_switch_to(binfo, port.port_bar); 605 if (ret) 606 break; 607 608 ret = parse_feature_list(binfo, 609 (u8 __iomem *)binfo->ioaddr + 610 port.port_offset); 611 if (ret) 612 break; 613 } while (++i < MAX_FPGA_PORT_NUM); 614 615 return ret; 616 } 617 618 static struct build_feature_devs_info * 619 build_info_alloc_and_init(struct ifpga_hw *hw) 620 { 621 struct build_feature_devs_info *binfo; 622 623 binfo = zmalloc(sizeof(*binfo)); 624 if (!binfo) 625 return binfo; 626 627 binfo->hw = hw; 628 binfo->pci_data = hw->pci_data; 629 630 /* fpga feature list starts from BAR 0 */ 631 if (parse_switch_to(binfo, 0)) { 632 free(binfo); 633 return NULL; 634 } 635 636 return binfo; 637 } 638 639 static void build_info_free(struct build_feature_devs_info *binfo) 640 { 641 free(binfo); 642 } 643 644 static void ifpga_print_device_feature_list(struct ifpga_hw *hw) 645 { 646 struct ifpga_fme_hw *fme = &hw->fme; 647 struct ifpga_port_hw *port; 648 struct ifpga_feature *feature; 649 int i; 650 651 dev_info(hw, "found fme_device, is in PF: %s\n", 652 is_ifpga_hw_pf(hw) ? "yes" : "no"); 653 654 ifpga_for_each_fme_feature(fme, feature) { 655 if (feature->state != IFPGA_FEATURE_ATTACHED) 656 continue; 657 658 dev_info(hw, "%12s: %p - %p - paddr: 0x%lx\n", 659 feature->name, feature->addr, 660 feature->addr + feature->size - 1, 661 (unsigned long)feature->phys_addr); 662 663 } 664 665 for (i = 0; i < MAX_FPGA_PORT_NUM; i++) { 666 port = &hw->port[i]; 667 668 if (port->state != IFPGA_PORT_ATTACHED) 669 continue; 670 671 dev_info(hw, "port device: %d\n", port->port_id); 672 673 ifpga_for_each_port_feature(port, feature) { 674 if (feature->state != IFPGA_FEATURE_ATTACHED) 675 continue; 676 677 dev_info(hw, "%12s: %p - %p - paddr:0x%lx\n", 678 feature->name, 679 feature->addr, 680 feature->addr + 681 feature->size - 1, 682 (unsigned long)feature->phys_addr); 683 } 684 685 } 686 } 687 688 int ifpga_bus_enumerate(struct ifpga_hw *hw) 689 { 690 struct build_feature_devs_info *binfo; 691 int ret; 692 693 binfo = build_info_alloc_and_init(hw); 694 if (!binfo) 695 return -ENOMEM; 696 697 ret = parse_feature_list(binfo, binfo->ioaddr); 698 if (ret) 699 goto exit; 700 701 ret = parse_ports_from_fme(binfo); 702 if (ret) 703 goto exit; 704 705 ifpga_print_device_feature_list(hw); 706 707 exit: 708 build_info_free(binfo); 709 return ret; 710 } 711 712 int ifpga_bus_init(struct ifpga_hw *hw) 713 { 714 int i; 715 struct ifpga_port_hw *port; 716 717 fme_hw_init(&hw->fme); 718 for (i = 0; i < MAX_FPGA_PORT_NUM; i++) { 719 port = &hw->port[i]; 720 port_hw_init(port); 721 } 722 723 return 0; 724 } 725