1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <unistd.h> 7 #include <fcntl.h> 8 #include <dirent.h> 9 #include <inttypes.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 #include <sys/sysmacros.h> 13 14 #if defined(RTE_ARCH_X86) 15 #include <sys/io.h> 16 #endif 17 18 #include <rte_string_fns.h> 19 #include <rte_log.h> 20 #include <rte_pci.h> 21 #include <rte_bus_pci.h> 22 #include <rte_common.h> 23 #include <rte_malloc.h> 24 25 #include "eal_filesystem.h" 26 #include "pci_init.h" 27 #include "private.h" 28 29 void *pci_map_addr = NULL; 30 31 #define OFF_MAX ((uint64_t)(off_t)-1) 32 33 int 34 pci_uio_read_config(const struct rte_intr_handle *intr_handle, 35 void *buf, size_t len, off_t offset) 36 { 37 int uio_cfg_fd = rte_intr_dev_fd_get(intr_handle); 38 39 if (uio_cfg_fd < 0) 40 return -1; 41 42 return pread(uio_cfg_fd, buf, len, offset); 43 } 44 45 int 46 pci_uio_write_config(const struct rte_intr_handle *intr_handle, 47 const void *buf, size_t len, off_t offset) 48 { 49 int uio_cfg_fd = rte_intr_dev_fd_get(intr_handle); 50 51 if (uio_cfg_fd < 0) 52 return -1; 53 54 return pwrite(uio_cfg_fd, buf, len, offset); 55 } 56 57 int 58 pci_uio_mmio_read(const struct rte_pci_device *dev, int bar, 59 void *buf, size_t len, off_t offset) 60 { 61 if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL || 62 (uint64_t)offset + len > dev->mem_resource[bar].len) 63 return -1; 64 memcpy(buf, (uint8_t *)dev->mem_resource[bar].addr + offset, len); 65 return len; 66 } 67 68 int 69 pci_uio_mmio_write(const struct rte_pci_device *dev, int bar, 70 const void *buf, size_t len, off_t offset) 71 { 72 if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL || 73 (uint64_t)offset + len > dev->mem_resource[bar].len) 74 return -1; 75 memcpy((uint8_t *)dev->mem_resource[bar].addr + offset, buf, len); 76 return len; 77 } 78 79 static int 80 pci_mknod_uio_dev(const char *sysfs_uio_path, unsigned uio_num) 81 { 82 FILE *f; 83 char filename[PATH_MAX]; 84 int ret; 85 unsigned major, minor; 86 dev_t dev; 87 88 /* get the name of the sysfs file that contains the major and minor 89 * of the uio device and read its content */ 90 snprintf(filename, sizeof(filename), "%s/dev", sysfs_uio_path); 91 92 f = fopen(filename, "r"); 93 if (f == NULL) { 94 PCI_LOG(ERR, "%s(): cannot open sysfs to get major:minor", __func__); 95 return -1; 96 } 97 98 ret = fscanf(f, "%u:%u", &major, &minor); 99 if (ret != 2) { 100 PCI_LOG(ERR, "%s(): cannot parse sysfs to get major:minor", __func__); 101 fclose(f); 102 return -1; 103 } 104 fclose(f); 105 106 /* create the char device "mknod /dev/uioX c major minor" */ 107 snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); 108 dev = makedev(major, minor); 109 ret = mknod(filename, S_IFCHR | S_IRUSR | S_IWUSR, dev); 110 if (ret != 0) { 111 PCI_LOG(ERR, "%s(): mknod() failed %s", __func__, strerror(errno)); 112 return -1; 113 } 114 115 return ret; 116 } 117 118 /* 119 * Return the uioX char device used for a pci device. On success, return 120 * the UIO number and fill dstbuf string with the path of the device in 121 * sysfs. On error, return a negative value. In this case dstbuf is 122 * invalid. 123 */ 124 static int 125 pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, 126 unsigned int buflen, int create) 127 { 128 struct rte_pci_addr *loc = &dev->addr; 129 int uio_num = -1; 130 struct dirent *e; 131 DIR *dir; 132 char dirname[PATH_MAX]; 133 134 /* depending on kernel version, uio can be located in uio/uioX 135 * or uio:uioX */ 136 137 snprintf(dirname, sizeof(dirname), 138 "%s/" PCI_PRI_FMT "/uio", rte_pci_get_sysfs_path(), 139 loc->domain, loc->bus, loc->devid, loc->function); 140 141 dir = opendir(dirname); 142 if (dir == NULL) { 143 /* retry with the parent directory */ 144 snprintf(dirname, sizeof(dirname), 145 "%s/" PCI_PRI_FMT, rte_pci_get_sysfs_path(), 146 loc->domain, loc->bus, loc->devid, loc->function); 147 dir = opendir(dirname); 148 149 if (dir == NULL) { 150 PCI_LOG(ERR, "Cannot opendir %s", dirname); 151 return -1; 152 } 153 } 154 155 /* take the first file starting with "uio" */ 156 while ((e = readdir(dir)) != NULL) { 157 /* format could be uio%d ...*/ 158 int shortprefix_len = sizeof("uio") - 1; 159 /* ... or uio:uio%d */ 160 int longprefix_len = sizeof("uio:uio") - 1; 161 char *endptr; 162 163 if (strncmp(e->d_name, "uio", 3) != 0) 164 continue; 165 166 /* first try uio%d */ 167 errno = 0; 168 uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); 169 if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { 170 snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); 171 break; 172 } 173 174 /* then try uio:uio%d */ 175 errno = 0; 176 uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); 177 if (errno == 0 && endptr != (e->d_name + longprefix_len)) { 178 snprintf(dstbuf, buflen, "%s/uio:uio%u", dirname, uio_num); 179 break; 180 } 181 } 182 closedir(dir); 183 184 /* No uio resource found */ 185 if (e == NULL) 186 return -1; 187 188 /* create uio device if we've been asked to */ 189 if (rte_eal_create_uio_dev() && create && 190 pci_mknod_uio_dev(dstbuf, uio_num) < 0) 191 PCI_LOG(WARNING, "Cannot create /dev/uio%u", uio_num); 192 193 return uio_num; 194 } 195 196 void 197 pci_uio_free_resource(struct rte_pci_device *dev, 198 struct mapped_pci_resource *uio_res) 199 { 200 int uio_cfg_fd = rte_intr_dev_fd_get(dev->intr_handle); 201 202 rte_free(uio_res); 203 204 if (uio_cfg_fd >= 0) { 205 close(uio_cfg_fd); 206 rte_intr_dev_fd_set(dev->intr_handle, -1); 207 } 208 209 if (rte_intr_fd_get(dev->intr_handle) >= 0) { 210 close(rte_intr_fd_get(dev->intr_handle)); 211 rte_intr_fd_set(dev->intr_handle, -1); 212 rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); 213 } 214 } 215 216 int 217 pci_uio_alloc_resource(struct rte_pci_device *dev, 218 struct mapped_pci_resource **uio_res) 219 { 220 char dirname[PATH_MAX]; 221 char cfgname[PATH_MAX]; 222 char devname[PATH_MAX]; /* contains the /dev/uioX */ 223 int uio_num, fd, uio_cfg_fd; 224 struct rte_pci_addr *loc; 225 226 loc = &dev->addr; 227 228 /* find uio resource */ 229 uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 1); 230 if (uio_num < 0) { 231 PCI_LOG(WARNING, " "PCI_PRI_FMT" not managed by UIO driver, skipping", 232 loc->domain, loc->bus, loc->devid, loc->function); 233 return 1; 234 } 235 snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); 236 237 /* save fd if in primary process */ 238 fd = open(devname, O_RDWR); 239 if (fd < 0) { 240 PCI_LOG(ERR, "Cannot open %s: %s", devname, strerror(errno)); 241 goto error; 242 } 243 244 if (rte_intr_fd_set(dev->intr_handle, fd)) 245 goto error; 246 247 snprintf(cfgname, sizeof(cfgname), 248 "/sys/class/uio/uio%u/device/config", uio_num); 249 250 uio_cfg_fd = open(cfgname, O_RDWR); 251 if (uio_cfg_fd < 0) { 252 PCI_LOG(ERR, "Cannot open %s: %s", cfgname, strerror(errno)); 253 goto error; 254 } 255 256 if (rte_intr_dev_fd_set(dev->intr_handle, uio_cfg_fd)) 257 goto error; 258 259 if (dev->kdrv == RTE_PCI_KDRV_IGB_UIO) { 260 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO)) 261 goto error; 262 } else { 263 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) 264 goto error; 265 266 /* set bus master that is not done by uio_pci_generic */ 267 if (rte_pci_set_bus_master(dev, true)) { 268 PCI_LOG(ERR, "Cannot set up bus mastering!"); 269 goto error; 270 } 271 } 272 273 /* allocate the mapping details for secondary processes*/ 274 *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); 275 if (*uio_res == NULL) { 276 PCI_LOG(ERR, "%s(): cannot store uio mmap details", __func__); 277 goto error; 278 } 279 280 strlcpy((*uio_res)->path, devname, sizeof((*uio_res)->path)); 281 memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); 282 283 return 0; 284 285 error: 286 pci_uio_free_resource(dev, *uio_res); 287 return -1; 288 } 289 290 int 291 pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, 292 struct mapped_pci_resource *uio_res, int map_idx) 293 { 294 int fd = -1; 295 char devname[PATH_MAX]; 296 void *mapaddr; 297 struct rte_pci_addr *loc; 298 struct pci_map *maps; 299 int wc_activate = 0; 300 301 if (dev->driver != NULL) 302 wc_activate = dev->driver->drv_flags & RTE_PCI_DRV_WC_ACTIVATE; 303 304 loc = &dev->addr; 305 maps = uio_res->maps; 306 307 /* allocate memory to keep path */ 308 maps[map_idx].path = rte_malloc(NULL, sizeof(devname), 0); 309 if (maps[map_idx].path == NULL) { 310 PCI_LOG(ERR, "Cannot allocate memory for path: %s", strerror(errno)); 311 return -1; 312 } 313 314 /* 315 * open resource file, to mmap it 316 */ 317 if (wc_activate) { 318 /* update devname for mmap */ 319 snprintf(devname, sizeof(devname), 320 "%s/" PCI_PRI_FMT "/resource%d_wc", 321 rte_pci_get_sysfs_path(), 322 loc->domain, loc->bus, loc->devid, 323 loc->function, res_idx); 324 325 fd = open(devname, O_RDWR); 326 if (fd < 0 && errno != ENOENT) { 327 PCI_LOG(INFO, "%s cannot be mapped. Fall-back to non prefetchable mode.", 328 devname); 329 } 330 } 331 332 if (!wc_activate || fd < 0) { 333 snprintf(devname, sizeof(devname), 334 "%s/" PCI_PRI_FMT "/resource%d", 335 rte_pci_get_sysfs_path(), 336 loc->domain, loc->bus, loc->devid, 337 loc->function, res_idx); 338 339 /* then try to map resource file */ 340 fd = open(devname, O_RDWR); 341 if (fd < 0) { 342 PCI_LOG(ERR, "Cannot open %s: %s", devname, strerror(errno)); 343 goto error; 344 } 345 } 346 347 /* try mapping somewhere close to the end of hugepages */ 348 if (pci_map_addr == NULL) 349 pci_map_addr = pci_find_max_end_va(); 350 351 mapaddr = pci_map_resource(pci_map_addr, fd, 0, 352 (size_t)dev->mem_resource[res_idx].len, 0); 353 close(fd); 354 if (mapaddr == NULL) 355 goto error; 356 357 pci_map_addr = RTE_PTR_ADD(mapaddr, 358 (size_t)dev->mem_resource[res_idx].len); 359 360 pci_map_addr = RTE_PTR_ALIGN(pci_map_addr, sysconf(_SC_PAGE_SIZE)); 361 362 maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; 363 maps[map_idx].size = dev->mem_resource[res_idx].len; 364 maps[map_idx].addr = mapaddr; 365 maps[map_idx].offset = 0; 366 strcpy(maps[map_idx].path, devname); 367 dev->mem_resource[res_idx].addr = mapaddr; 368 369 return 0; 370 371 error: 372 rte_free(maps[map_idx].path); 373 return -1; 374 } 375 376 #define PIO_MAX 0x10000 377 378 #if defined(RTE_ARCH_X86) 379 int 380 pci_uio_ioport_map(struct rte_pci_device *dev, int bar, 381 struct rte_pci_ioport *p) 382 { 383 FILE *f = NULL; 384 char dirname[PATH_MAX]; 385 char filename[PATH_MAX]; 386 char buf[BUFSIZ]; 387 uint64_t phys_addr, end_addr, flags; 388 unsigned long base; 389 int i, fd; 390 391 /* open and read addresses of the corresponding resource in sysfs */ 392 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", 393 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 394 dev->addr.devid, dev->addr.function); 395 f = fopen(filename, "r"); 396 if (f == NULL) { 397 PCI_LOG(ERR, "%s(): Cannot open sysfs resource: %s", __func__, strerror(errno)); 398 return -1; 399 } 400 401 for (i = 0; i < bar + 1; i++) { 402 if (fgets(buf, sizeof(buf), f) == NULL) { 403 PCI_LOG(ERR, "%s(): Cannot read sysfs resource", __func__); 404 goto error; 405 } 406 } 407 if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, 408 &end_addr, &flags) < 0) 409 goto error; 410 411 if (flags & IORESOURCE_IO) { 412 if (rte_eal_iopl_init()) { 413 PCI_LOG(ERR, "%s(): insufficient ioport permissions for PCI device %s", 414 __func__, dev->name); 415 goto error; 416 } 417 418 base = (unsigned long)phys_addr; 419 if (base > PIO_MAX) { 420 PCI_LOG(ERR, "%s(): %08lx too large PIO resource", __func__, base); 421 goto error; 422 } 423 424 PCI_LOG(DEBUG, "%s(): PIO BAR %08lx detected", __func__, base); 425 } else if (flags & IORESOURCE_MEM) { 426 base = (unsigned long)dev->mem_resource[bar].addr; 427 PCI_LOG(DEBUG, "%s(): MMIO BAR %08lx detected", __func__, base); 428 } else { 429 PCI_LOG(ERR, "%s(): unknown BAR type", __func__); 430 goto error; 431 } 432 433 /* FIXME only for primary process ? */ 434 if (rte_intr_type_get(dev->intr_handle) == 435 RTE_INTR_HANDLE_UNKNOWN) { 436 int uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); 437 if (uio_num < 0) { 438 PCI_LOG(ERR, "cannot open %s: %s", dirname, strerror(errno)); 439 goto error; 440 } 441 442 snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); 443 fd = open(filename, O_RDWR); 444 if (fd < 0) { 445 PCI_LOG(ERR, "Cannot open %s: %s", filename, strerror(errno)); 446 goto error; 447 } 448 if (rte_intr_fd_set(dev->intr_handle, fd)) 449 goto error; 450 451 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO)) 452 goto error; 453 } 454 455 PCI_LOG(DEBUG, "PCI Port IO found start=0x%lx", base); 456 457 p->base = base; 458 p->len = 0; 459 fclose(f); 460 return 0; 461 error: 462 if (f) 463 fclose(f); 464 return -1; 465 } 466 #else 467 int 468 pci_uio_ioport_map(struct rte_pci_device *dev, int bar, 469 struct rte_pci_ioport *p) 470 { 471 FILE *f; 472 char buf[BUFSIZ]; 473 char filename[PATH_MAX]; 474 uint64_t phys_addr, end_addr, flags; 475 int fd, i; 476 void *addr; 477 478 /* open and read addresses of the corresponding resource in sysfs */ 479 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", 480 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 481 dev->addr.devid, dev->addr.function); 482 f = fopen(filename, "r"); 483 if (f == NULL) { 484 PCI_LOG(ERR, "Cannot open sysfs resource: %s", strerror(errno)); 485 return -1; 486 } 487 for (i = 0; i < bar + 1; i++) { 488 if (fgets(buf, sizeof(buf), f) == NULL) { 489 PCI_LOG(ERR, "Cannot read sysfs resource"); 490 goto error; 491 } 492 } 493 if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, 494 &end_addr, &flags) < 0) 495 goto error; 496 if ((flags & IORESOURCE_IO) == 0) { 497 PCI_LOG(ERR, "BAR %d is not an IO resource", bar); 498 goto error; 499 } 500 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource%d", 501 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 502 dev->addr.devid, dev->addr.function, bar); 503 504 /* mmap the pci resource */ 505 fd = open(filename, O_RDWR); 506 if (fd < 0) { 507 PCI_LOG(ERR, "Cannot open %s: %s", filename, strerror(errno)); 508 goto error; 509 } 510 addr = mmap(NULL, end_addr + 1, PROT_READ | PROT_WRITE, 511 MAP_SHARED, fd, 0); 512 close(fd); 513 if (addr == MAP_FAILED) { 514 PCI_LOG(ERR, "Cannot mmap IO port resource: %s", strerror(errno)); 515 goto error; 516 } 517 518 /* strangely, the base address is mmap addr + phys_addr */ 519 p->base = (uintptr_t)addr + phys_addr; 520 p->len = end_addr + 1; 521 PCI_LOG(DEBUG, "PCI Port IO found start=0x%"PRIx64, p->base); 522 fclose(f); 523 524 return 0; 525 526 error: 527 fclose(f); 528 return -1; 529 } 530 #endif 531 532 #if defined(RTE_ARCH_X86) 533 534 static inline uint8_t ioread8(void *addr) 535 { 536 uint8_t val; 537 538 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 539 *(volatile uint8_t *)addr : 540 #ifdef __GLIBC__ 541 inb_p((unsigned long)addr); 542 #else 543 inb((unsigned long)addr); 544 #endif 545 546 return val; 547 } 548 549 static inline uint16_t ioread16(void *addr) 550 { 551 uint16_t val; 552 553 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 554 *(volatile uint16_t *)addr : 555 #ifdef __GLIBC__ 556 inw_p((unsigned long)addr); 557 #else 558 inw((unsigned long)addr); 559 #endif 560 561 return val; 562 } 563 564 static inline uint32_t ioread32(void *addr) 565 { 566 uint32_t val; 567 568 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 569 *(volatile uint32_t *)addr : 570 #ifdef __GLIBC__ 571 inl_p((unsigned long)addr); 572 #else 573 inl((unsigned long)addr); 574 #endif 575 576 return val; 577 } 578 579 static inline void iowrite8(uint8_t val, void *addr) 580 { 581 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 582 *(volatile uint8_t *)addr = val : 583 #ifdef __GLIBC__ 584 outb_p(val, (unsigned long)addr); 585 #else 586 outb(val, (unsigned long)addr); 587 #endif 588 } 589 590 static inline void iowrite16(uint16_t val, void *addr) 591 { 592 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 593 *(volatile uint16_t *)addr = val : 594 #ifdef __GLIBC__ 595 outw_p(val, (unsigned long)addr); 596 #else 597 outw(val, (unsigned long)addr); 598 #endif 599 } 600 601 static inline void iowrite32(uint32_t val, void *addr) 602 { 603 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 604 *(volatile uint32_t *)addr = val : 605 #ifdef __GLIBC__ 606 outl_p(val, (unsigned long)addr); 607 #else 608 outl(val, (unsigned long)addr); 609 #endif 610 } 611 612 #else /* !RTE_ARCH_X86 */ 613 614 static inline uint8_t ioread8(void *addr) 615 { 616 return *(volatile uint8_t *)addr; 617 } 618 619 static inline uint16_t ioread16(void *addr) 620 { 621 return *(volatile uint16_t *)addr; 622 } 623 624 static inline uint32_t ioread32(void *addr) 625 { 626 return *(volatile uint32_t *)addr; 627 } 628 629 static inline void iowrite8(uint8_t val, void *addr) 630 { 631 *(volatile uint8_t *)addr = val; 632 } 633 634 static inline void iowrite16(uint16_t val, void *addr) 635 { 636 *(volatile uint16_t *)addr = val; 637 } 638 639 static inline void iowrite32(uint32_t val, void *addr) 640 { 641 *(volatile uint32_t *)addr = val; 642 } 643 644 #endif /* !RTE_ARCH_X86 */ 645 646 void 647 pci_uio_ioport_read(struct rte_pci_ioport *p, 648 void *data, size_t len, off_t offset) 649 { 650 uint8_t *d; 651 int size; 652 uintptr_t reg = p->base + offset; 653 654 for (d = data; len > 0; d += size, reg += size, len -= size) { 655 if (len >= 4) { 656 size = 4; 657 *(uint32_t *)d = ioread32((void *)reg); 658 } else if (len >= 2) { 659 size = 2; 660 *(uint16_t *)d = ioread16((void *)reg); 661 } else { 662 size = 1; 663 *d = ioread8((void *)reg); 664 } 665 } 666 } 667 668 void 669 pci_uio_ioport_write(struct rte_pci_ioport *p, 670 const void *data, size_t len, off_t offset) 671 { 672 const uint8_t *s; 673 int size; 674 uintptr_t reg = p->base + offset; 675 676 for (s = data; len > 0; s += size, reg += size, len -= size) { 677 if (len >= 4) { 678 size = 4; 679 iowrite32(*(const uint32_t *)s, (void *)reg); 680 } else if (len >= 2) { 681 size = 2; 682 iowrite16(*(const uint16_t *)s, (void *)reg); 683 } else { 684 size = 1; 685 iowrite8(*s, (void *)reg); 686 } 687 } 688 } 689 690 int 691 pci_uio_ioport_unmap(struct rte_pci_ioport *p) 692 { 693 #if defined(RTE_ARCH_X86) 694 RTE_SET_USED(p); 695 /* FIXME close intr fd ? */ 696 return 0; 697 #else 698 return munmap((void *)(uintptr_t)p->base, p->len); 699 #endif 700 } 701