1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <unistd.h> 7 #include <fcntl.h> 8 #include <dirent.h> 9 #include <inttypes.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 #include <sys/sysmacros.h> 13 14 #if defined(RTE_ARCH_X86) 15 #include <sys/io.h> 16 #endif 17 18 #include <rte_string_fns.h> 19 #include <rte_log.h> 20 #include <rte_pci.h> 21 #include <rte_bus_pci.h> 22 #include <rte_common.h> 23 #include <rte_malloc.h> 24 25 #include "eal_filesystem.h" 26 #include "pci_init.h" 27 #include "private.h" 28 29 void *pci_map_addr = NULL; 30 31 #define OFF_MAX ((uint64_t)(off_t)-1) 32 33 int 34 pci_uio_read_config(const struct rte_intr_handle *intr_handle, 35 void *buf, size_t len, off_t offset) 36 { 37 int uio_cfg_fd = rte_intr_dev_fd_get(intr_handle); 38 39 if (uio_cfg_fd < 0) 40 return -1; 41 42 return pread(uio_cfg_fd, buf, len, offset); 43 } 44 45 int 46 pci_uio_write_config(const struct rte_intr_handle *intr_handle, 47 const void *buf, size_t len, off_t offset) 48 { 49 int uio_cfg_fd = rte_intr_dev_fd_get(intr_handle); 50 51 if (uio_cfg_fd < 0) 52 return -1; 53 54 return pwrite(uio_cfg_fd, buf, len, offset); 55 } 56 57 int 58 pci_uio_mmio_read(const struct rte_pci_device *dev, int bar, 59 void *buf, size_t len, off_t offset) 60 { 61 if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL || 62 (uint64_t)offset + len > dev->mem_resource[bar].len) 63 return -1; 64 memcpy(buf, (uint8_t *)dev->mem_resource[bar].addr + offset, len); 65 return len; 66 } 67 68 int 69 pci_uio_mmio_write(const struct rte_pci_device *dev, int bar, 70 const void *buf, size_t len, off_t offset) 71 { 72 if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL || 73 (uint64_t)offset + len > dev->mem_resource[bar].len) 74 return -1; 75 memcpy((uint8_t *)dev->mem_resource[bar].addr + offset, buf, len); 76 return len; 77 } 78 79 static int 80 pci_mknod_uio_dev(const char *sysfs_uio_path, unsigned uio_num) 81 { 82 FILE *f; 83 char filename[PATH_MAX]; 84 int ret; 85 unsigned major, minor; 86 dev_t dev; 87 88 /* get the name of the sysfs file that contains the major and minor 89 * of the uio device and read its content */ 90 snprintf(filename, sizeof(filename), "%s/dev", sysfs_uio_path); 91 92 f = fopen(filename, "r"); 93 if (f == NULL) { 94 PCI_LOG(ERR, "%s(): cannot open sysfs to get major:minor", __func__); 95 return -1; 96 } 97 98 ret = fscanf(f, "%u:%u", &major, &minor); 99 if (ret != 2) { 100 PCI_LOG(ERR, "%s(): cannot parse sysfs to get major:minor", __func__); 101 fclose(f); 102 return -1; 103 } 104 fclose(f); 105 106 /* create the char device "mknod /dev/uioX c major minor" */ 107 snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); 108 dev = makedev(major, minor); 109 ret = mknod(filename, S_IFCHR | S_IRUSR | S_IWUSR, dev); 110 if (ret != 0) { 111 PCI_LOG(ERR, "%s(): mknod() failed %s", __func__, strerror(errno)); 112 return -1; 113 } 114 115 return ret; 116 } 117 118 /* 119 * Return the uioX char device used for a pci device. On success, return 120 * the UIO number and fill dstbuf string with the path of the device in 121 * sysfs. On error, return a negative value. In this case dstbuf is 122 * invalid. 123 */ 124 static int 125 pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, 126 unsigned int buflen, int create) 127 { 128 struct rte_pci_addr *loc = &dev->addr; 129 int uio_num = -1; 130 struct dirent *e; 131 DIR *dir; 132 char dirname[PATH_MAX]; 133 134 /* depending on kernel version, uio can be located in uio/uioX 135 * or uio:uioX */ 136 137 snprintf(dirname, sizeof(dirname), 138 "%s/" PCI_PRI_FMT "/uio", rte_pci_get_sysfs_path(), 139 loc->domain, loc->bus, loc->devid, loc->function); 140 141 dir = opendir(dirname); 142 if (dir == NULL) { 143 /* retry with the parent directory */ 144 snprintf(dirname, sizeof(dirname), 145 "%s/" PCI_PRI_FMT, rte_pci_get_sysfs_path(), 146 loc->domain, loc->bus, loc->devid, loc->function); 147 dir = opendir(dirname); 148 149 if (dir == NULL) { 150 PCI_LOG(ERR, "Cannot opendir %s", dirname); 151 return -1; 152 } 153 } 154 155 /* take the first file starting with "uio" */ 156 while ((e = readdir(dir)) != NULL) { 157 /* format could be uio%d ...*/ 158 int shortprefix_len = sizeof("uio") - 1; 159 /* ... or uio:uio%d */ 160 int longprefix_len = sizeof("uio:uio") - 1; 161 char *endptr; 162 163 if (strncmp(e->d_name, "uio", 3) != 0) 164 continue; 165 166 /* first try uio%d */ 167 errno = 0; 168 uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); 169 if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { 170 snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); 171 break; 172 } 173 174 /* then try uio:uio%d */ 175 errno = 0; 176 uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); 177 if (errno == 0 && endptr != (e->d_name + longprefix_len)) { 178 snprintf(dstbuf, buflen, "%s/uio:uio%u", dirname, uio_num); 179 break; 180 } 181 } 182 closedir(dir); 183 184 /* No uio resource found */ 185 if (e == NULL) 186 return -1; 187 188 /* create uio device if we've been asked to */ 189 if (rte_eal_create_uio_dev() && create && 190 pci_mknod_uio_dev(dstbuf, uio_num) < 0) 191 PCI_LOG(WARNING, "Cannot create /dev/uio%u", uio_num); 192 193 return uio_num; 194 } 195 196 void 197 pci_uio_free_resource(struct rte_pci_device *dev, 198 struct mapped_pci_resource *uio_res) 199 { 200 int uio_cfg_fd = rte_intr_dev_fd_get(dev->intr_handle); 201 202 rte_free(uio_res); 203 204 if (uio_cfg_fd >= 0) { 205 close(uio_cfg_fd); 206 rte_intr_dev_fd_set(dev->intr_handle, -1); 207 } 208 209 if (rte_intr_fd_get(dev->intr_handle) >= 0) { 210 close(rte_intr_fd_get(dev->intr_handle)); 211 rte_intr_fd_set(dev->intr_handle, -1); 212 rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); 213 } 214 } 215 216 int 217 pci_uio_alloc_resource(struct rte_pci_device *dev, 218 struct mapped_pci_resource **uio_res) 219 { 220 char dirname[PATH_MAX]; 221 char cfgname[PATH_MAX]; 222 char devname[PATH_MAX]; /* contains the /dev/uioX */ 223 int uio_num, fd, uio_cfg_fd; 224 struct rte_pci_addr *loc; 225 226 loc = &dev->addr; 227 228 /* find uio resource */ 229 uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 1); 230 if (uio_num < 0) { 231 PCI_LOG(WARNING, " "PCI_PRI_FMT" not managed by UIO driver, skipping", 232 loc->domain, loc->bus, loc->devid, loc->function); 233 return 1; 234 } 235 snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); 236 237 /* save fd */ 238 fd = open(devname, O_RDWR); 239 if (fd < 0) { 240 PCI_LOG(ERR, "Cannot open %s: %s", devname, strerror(errno)); 241 goto error; 242 } 243 244 if (rte_intr_fd_set(dev->intr_handle, fd)) 245 goto error; 246 247 snprintf(cfgname, sizeof(cfgname), 248 "/sys/class/uio/uio%u/device/config", uio_num); 249 250 uio_cfg_fd = open(cfgname, O_RDWR); 251 if (uio_cfg_fd < 0) { 252 PCI_LOG(ERR, "Cannot open %s: %s", cfgname, strerror(errno)); 253 goto error; 254 } 255 256 if (rte_intr_dev_fd_set(dev->intr_handle, uio_cfg_fd)) 257 goto error; 258 259 if (dev->kdrv == RTE_PCI_KDRV_IGB_UIO) { 260 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO)) 261 goto error; 262 } else { 263 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) 264 goto error; 265 266 /* set bus master that is not done by uio_pci_generic */ 267 if (rte_pci_set_bus_master(dev, true)) { 268 PCI_LOG(ERR, "Cannot set up bus mastering!"); 269 goto error; 270 } 271 } 272 273 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 274 return 0; 275 276 /* allocate the mapping details for secondary processes*/ 277 *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); 278 if (*uio_res == NULL) { 279 PCI_LOG(ERR, "%s(): cannot store uio mmap details", __func__); 280 goto error; 281 } 282 283 strlcpy((*uio_res)->path, devname, sizeof((*uio_res)->path)); 284 memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); 285 286 return 0; 287 288 error: 289 pci_uio_free_resource(dev, *uio_res); 290 return -1; 291 } 292 293 int 294 pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, 295 struct mapped_pci_resource *uio_res, int map_idx) 296 { 297 int fd = -1; 298 char devname[PATH_MAX]; 299 void *mapaddr; 300 struct rte_pci_addr *loc; 301 struct pci_map *maps; 302 int wc_activate = 0; 303 304 if (dev->driver != NULL) 305 wc_activate = dev->driver->drv_flags & RTE_PCI_DRV_WC_ACTIVATE; 306 307 loc = &dev->addr; 308 maps = uio_res->maps; 309 310 /* allocate memory to keep path */ 311 maps[map_idx].path = rte_malloc(NULL, sizeof(devname), 0); 312 if (maps[map_idx].path == NULL) { 313 PCI_LOG(ERR, "Cannot allocate memory for path: %s", strerror(errno)); 314 return -1; 315 } 316 317 /* 318 * open resource file, to mmap it 319 */ 320 if (wc_activate) { 321 /* update devname for mmap */ 322 snprintf(devname, sizeof(devname), 323 "%s/" PCI_PRI_FMT "/resource%d_wc", 324 rte_pci_get_sysfs_path(), 325 loc->domain, loc->bus, loc->devid, 326 loc->function, res_idx); 327 328 fd = open(devname, O_RDWR); 329 if (fd < 0 && errno != ENOENT) { 330 PCI_LOG(INFO, "%s cannot be mapped. Fall-back to non prefetchable mode.", 331 devname); 332 } 333 } 334 335 if (!wc_activate || fd < 0) { 336 snprintf(devname, sizeof(devname), 337 "%s/" PCI_PRI_FMT "/resource%d", 338 rte_pci_get_sysfs_path(), 339 loc->domain, loc->bus, loc->devid, 340 loc->function, res_idx); 341 342 /* then try to map resource file */ 343 fd = open(devname, O_RDWR); 344 if (fd < 0) { 345 PCI_LOG(ERR, "Cannot open %s: %s", devname, strerror(errno)); 346 goto error; 347 } 348 } 349 350 /* try mapping somewhere close to the end of hugepages */ 351 if (pci_map_addr == NULL) 352 pci_map_addr = pci_find_max_end_va(); 353 354 mapaddr = pci_map_resource(pci_map_addr, fd, 0, 355 (size_t)dev->mem_resource[res_idx].len, 0); 356 close(fd); 357 if (mapaddr == NULL) 358 goto error; 359 360 pci_map_addr = RTE_PTR_ADD(mapaddr, 361 (size_t)dev->mem_resource[res_idx].len); 362 363 pci_map_addr = RTE_PTR_ALIGN(pci_map_addr, sysconf(_SC_PAGE_SIZE)); 364 365 maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; 366 maps[map_idx].size = dev->mem_resource[res_idx].len; 367 maps[map_idx].addr = mapaddr; 368 maps[map_idx].offset = 0; 369 strcpy(maps[map_idx].path, devname); 370 dev->mem_resource[res_idx].addr = mapaddr; 371 372 return 0; 373 374 error: 375 rte_free(maps[map_idx].path); 376 return -1; 377 } 378 379 #define PIO_MAX 0x10000 380 381 #if defined(RTE_ARCH_X86) 382 int 383 pci_uio_ioport_map(struct rte_pci_device *dev, int bar, 384 struct rte_pci_ioport *p) 385 { 386 FILE *f = NULL; 387 char dirname[PATH_MAX]; 388 char filename[PATH_MAX]; 389 char buf[BUFSIZ]; 390 uint64_t phys_addr, end_addr, flags; 391 unsigned long base; 392 int i, fd; 393 394 /* open and read addresses of the corresponding resource in sysfs */ 395 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", 396 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 397 dev->addr.devid, dev->addr.function); 398 f = fopen(filename, "r"); 399 if (f == NULL) { 400 PCI_LOG(ERR, "%s(): Cannot open sysfs resource: %s", __func__, strerror(errno)); 401 return -1; 402 } 403 404 for (i = 0; i < bar + 1; i++) { 405 if (fgets(buf, sizeof(buf), f) == NULL) { 406 PCI_LOG(ERR, "%s(): Cannot read sysfs resource", __func__); 407 goto error; 408 } 409 } 410 if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, 411 &end_addr, &flags) < 0) 412 goto error; 413 414 if (flags & IORESOURCE_IO) { 415 if (rte_eal_iopl_init()) { 416 PCI_LOG(ERR, "%s(): insufficient ioport permissions for PCI device %s", 417 __func__, dev->name); 418 goto error; 419 } 420 421 base = (unsigned long)phys_addr; 422 if (base > PIO_MAX) { 423 PCI_LOG(ERR, "%s(): %08lx too large PIO resource", __func__, base); 424 goto error; 425 } 426 427 PCI_LOG(DEBUG, "%s(): PIO BAR %08lx detected", __func__, base); 428 } else if (flags & IORESOURCE_MEM) { 429 base = (unsigned long)dev->mem_resource[bar].addr; 430 PCI_LOG(DEBUG, "%s(): MMIO BAR %08lx detected", __func__, base); 431 } else { 432 PCI_LOG(ERR, "%s(): unknown BAR type", __func__); 433 goto error; 434 } 435 436 /* FIXME only for primary process ? */ 437 if (rte_intr_type_get(dev->intr_handle) == 438 RTE_INTR_HANDLE_UNKNOWN) { 439 int uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); 440 if (uio_num < 0) { 441 PCI_LOG(ERR, "cannot open %s: %s", dirname, strerror(errno)); 442 goto error; 443 } 444 445 snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); 446 fd = open(filename, O_RDWR); 447 if (fd < 0) { 448 PCI_LOG(ERR, "Cannot open %s: %s", filename, strerror(errno)); 449 goto error; 450 } 451 if (rte_intr_fd_set(dev->intr_handle, fd)) 452 goto error; 453 454 if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO)) 455 goto error; 456 } 457 458 PCI_LOG(DEBUG, "PCI Port IO found start=0x%lx", base); 459 460 p->base = base; 461 p->len = 0; 462 fclose(f); 463 return 0; 464 error: 465 if (f) 466 fclose(f); 467 return -1; 468 } 469 #else 470 int 471 pci_uio_ioport_map(struct rte_pci_device *dev, int bar, 472 struct rte_pci_ioport *p) 473 { 474 FILE *f; 475 char buf[BUFSIZ]; 476 char filename[PATH_MAX]; 477 uint64_t phys_addr, end_addr, flags; 478 int fd, i; 479 void *addr; 480 481 /* open and read addresses of the corresponding resource in sysfs */ 482 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", 483 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 484 dev->addr.devid, dev->addr.function); 485 f = fopen(filename, "r"); 486 if (f == NULL) { 487 PCI_LOG(ERR, "Cannot open sysfs resource: %s", strerror(errno)); 488 return -1; 489 } 490 for (i = 0; i < bar + 1; i++) { 491 if (fgets(buf, sizeof(buf), f) == NULL) { 492 PCI_LOG(ERR, "Cannot read sysfs resource"); 493 goto error; 494 } 495 } 496 if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, 497 &end_addr, &flags) < 0) 498 goto error; 499 if ((flags & IORESOURCE_IO) == 0) { 500 PCI_LOG(ERR, "BAR %d is not an IO resource", bar); 501 goto error; 502 } 503 snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource%d", 504 rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, 505 dev->addr.devid, dev->addr.function, bar); 506 507 /* mmap the pci resource */ 508 fd = open(filename, O_RDWR); 509 if (fd < 0) { 510 PCI_LOG(ERR, "Cannot open %s: %s", filename, strerror(errno)); 511 goto error; 512 } 513 addr = mmap(NULL, end_addr + 1, PROT_READ | PROT_WRITE, 514 MAP_SHARED, fd, 0); 515 close(fd); 516 if (addr == MAP_FAILED) { 517 PCI_LOG(ERR, "Cannot mmap IO port resource: %s", strerror(errno)); 518 goto error; 519 } 520 521 /* strangely, the base address is mmap addr + phys_addr */ 522 p->base = (uintptr_t)addr + phys_addr; 523 p->len = end_addr + 1; 524 PCI_LOG(DEBUG, "PCI Port IO found start=0x%"PRIx64, p->base); 525 fclose(f); 526 527 return 0; 528 529 error: 530 fclose(f); 531 return -1; 532 } 533 #endif 534 535 #if defined(RTE_ARCH_X86) 536 537 static inline uint8_t ioread8(void *addr) 538 { 539 uint8_t val; 540 541 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 542 *(volatile uint8_t *)addr : 543 #ifdef __GLIBC__ 544 inb_p((unsigned long)addr); 545 #else 546 inb((unsigned long)addr); 547 #endif 548 549 return val; 550 } 551 552 static inline uint16_t ioread16(void *addr) 553 { 554 uint16_t val; 555 556 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 557 *(volatile uint16_t *)addr : 558 #ifdef __GLIBC__ 559 inw_p((unsigned long)addr); 560 #else 561 inw((unsigned long)addr); 562 #endif 563 564 return val; 565 } 566 567 static inline uint32_t ioread32(void *addr) 568 { 569 uint32_t val; 570 571 val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? 572 *(volatile uint32_t *)addr : 573 #ifdef __GLIBC__ 574 inl_p((unsigned long)addr); 575 #else 576 inl((unsigned long)addr); 577 #endif 578 579 return val; 580 } 581 582 static inline void iowrite8(uint8_t val, void *addr) 583 { 584 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 585 *(volatile uint8_t *)addr = val : 586 #ifdef __GLIBC__ 587 outb_p(val, (unsigned long)addr); 588 #else 589 outb(val, (unsigned long)addr); 590 #endif 591 } 592 593 static inline void iowrite16(uint16_t val, void *addr) 594 { 595 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 596 *(volatile uint16_t *)addr = val : 597 #ifdef __GLIBC__ 598 outw_p(val, (unsigned long)addr); 599 #else 600 outw(val, (unsigned long)addr); 601 #endif 602 } 603 604 static inline void iowrite32(uint32_t val, void *addr) 605 { 606 (uint64_t)(uintptr_t)addr >= PIO_MAX ? 607 *(volatile uint32_t *)addr = val : 608 #ifdef __GLIBC__ 609 outl_p(val, (unsigned long)addr); 610 #else 611 outl(val, (unsigned long)addr); 612 #endif 613 } 614 615 #else /* !RTE_ARCH_X86 */ 616 617 static inline uint8_t ioread8(void *addr) 618 { 619 return *(volatile uint8_t *)addr; 620 } 621 622 static inline uint16_t ioread16(void *addr) 623 { 624 return *(volatile uint16_t *)addr; 625 } 626 627 static inline uint32_t ioread32(void *addr) 628 { 629 return *(volatile uint32_t *)addr; 630 } 631 632 static inline void iowrite8(uint8_t val, void *addr) 633 { 634 *(volatile uint8_t *)addr = val; 635 } 636 637 static inline void iowrite16(uint16_t val, void *addr) 638 { 639 *(volatile uint16_t *)addr = val; 640 } 641 642 static inline void iowrite32(uint32_t val, void *addr) 643 { 644 *(volatile uint32_t *)addr = val; 645 } 646 647 #endif /* !RTE_ARCH_X86 */ 648 649 void 650 pci_uio_ioport_read(struct rte_pci_ioport *p, 651 void *data, size_t len, off_t offset) 652 { 653 uint8_t *d; 654 int size; 655 uintptr_t reg = p->base + offset; 656 657 for (d = data; len > 0; d += size, reg += size, len -= size) { 658 if (len >= 4) { 659 size = 4; 660 *(uint32_t *)d = ioread32((void *)reg); 661 } else if (len >= 2) { 662 size = 2; 663 *(uint16_t *)d = ioread16((void *)reg); 664 } else { 665 size = 1; 666 *d = ioread8((void *)reg); 667 } 668 } 669 } 670 671 void 672 pci_uio_ioport_write(struct rte_pci_ioport *p, 673 const void *data, size_t len, off_t offset) 674 { 675 const uint8_t *s; 676 int size; 677 uintptr_t reg = p->base + offset; 678 679 for (s = data; len > 0; s += size, reg += size, len -= size) { 680 if (len >= 4) { 681 size = 4; 682 iowrite32(*(const uint32_t *)s, (void *)reg); 683 } else if (len >= 2) { 684 size = 2; 685 iowrite16(*(const uint16_t *)s, (void *)reg); 686 } else { 687 size = 1; 688 iowrite8(*s, (void *)reg); 689 } 690 } 691 } 692 693 int 694 pci_uio_ioport_unmap(struct rte_pci_ioport *p) 695 { 696 #if defined(RTE_ARCH_X86) 697 RTE_SET_USED(p); 698 /* FIXME close intr fd ? */ 699 return 0; 700 #else 701 return munmap((void *)(uintptr_t)p->base, p->len); 702 #endif 703 } 704