1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2024 HiSilicon Limited 3 */ 4 5 #include <dirent.h> 6 #include <errno.h> 7 #include <fcntl.h> 8 #include <unistd.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <sys/ioctl.h> 12 #include <sys/mman.h> 13 #include <sys/stat.h> 14 #include <sys/types.h> 15 16 #include <rte_bitops.h> 17 #include <rte_common.h> 18 #include <rte_devargs.h> 19 #include <rte_eal_paging.h> 20 #include <rte_errno.h> 21 #include <rte_log.h> 22 #include <rte_kvargs.h> 23 #include <bus_driver.h> 24 25 #include "bus_uacce_driver.h" 26 27 #define UACCE_BUS_CLASS_PATH "/sys/class/uacce" 28 29 /* UACCE device flag of SVA. */ 30 #define UACCE_DEV_FLGA_SVA RTE_BIT32(0) 31 32 /* Support -a uacce:device-name when start DPDK application. */ 33 #define UACCE_DEV_PREFIX "uacce:" 34 35 /* 36 * Structure describing the UACCE bus. 37 */ 38 struct rte_uacce_bus { 39 struct rte_bus bus; /* Inherit the generic class. */ 40 TAILQ_HEAD(, rte_uacce_device) device_list; /* List of devices. */ 41 TAILQ_HEAD(, rte_uacce_driver) driver_list; /* List of drivers. */ 42 }; 43 44 /* Forward declaration of UACCE bus. */ 45 static struct rte_uacce_bus uacce_bus; 46 47 enum uacce_params { 48 RTE_UACCE_PARAM_NAME, 49 }; 50 51 static const char *const uacce_params_keys[] = { 52 [RTE_UACCE_PARAM_NAME] = "name", 53 NULL, 54 }; 55 56 #define FOREACH_DEVICE_ON_UACCEBUS(p) \ 57 RTE_TAILQ_FOREACH(p, &uacce_bus.device_list, next) 58 #define FOREACH_DRIVER_ON_UACCEBUS(p) \ 59 RTE_TAILQ_FOREACH(p, &uacce_bus.driver_list, next) 60 61 extern int uacce_bus_logtype; 62 #define RTE_LOGTYPE_UACCE_BUS uacce_bus_logtype 63 #define UACCE_BUS_LOG(level, ...) \ 64 RTE_LOG_LINE(level, UACCE_BUS, __VA_ARGS__) 65 #define UACCE_BUS_ERR(fmt, ...) UACCE_BUS_LOG(ERR, fmt, ##__VA_ARGS__) 66 #define UACCE_BUS_WARN(fmt, ...) UACCE_BUS_LOG(WARNING, fmt, ##__VA_ARGS__) 67 #define UACCE_BUS_INFO(fmt, ...) UACCE_BUS_LOG(INFO, fmt, ##__VA_ARGS__) 68 #define UACCE_BUS_DEBUG(fmt, ...) UACCE_BUS_LOG(DEBUG, fmt, ##__VA_ARGS__) 69 70 71 static struct rte_devargs * 72 uacce_devargs_lookup(const char *dev_name) 73 { 74 char name[RTE_UACCE_DEV_PATH_SIZE] = {0}; 75 struct rte_devargs *devargs; 76 77 snprintf(name, sizeof(name), "%s%s", UACCE_DEV_PREFIX, dev_name); 78 RTE_EAL_DEVARGS_FOREACH("uacce", devargs) { 79 if (strcmp(devargs->name, name) == 0) 80 return devargs; 81 } 82 83 return NULL; 84 } 85 86 static bool 87 uacce_ignore_device(const char *dev_name) 88 { 89 struct rte_devargs *devargs = uacce_devargs_lookup(dev_name); 90 91 switch (uacce_bus.bus.conf.scan_mode) { 92 case RTE_BUS_SCAN_ALLOWLIST: 93 if (devargs && devargs->policy == RTE_DEV_ALLOWED) 94 return false; 95 break; 96 case RTE_BUS_SCAN_UNDEFINED: 97 case RTE_BUS_SCAN_BLOCKLIST: 98 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED) 99 return false; 100 break; 101 } 102 103 return true; 104 } 105 106 /* 107 * Returns the number of bytes read (removed last newline) on success. 108 * Otherwise negative value is returned. 109 */ 110 static int 111 uacce_read_attr(const char *dev_root, const char *attr, char *buf, uint32_t sz) 112 { 113 char filename[PATH_MAX] = {0}; 114 int ret; 115 int fd; 116 117 snprintf(filename, sizeof(filename), "%s/%s", dev_root, attr); 118 fd = open(filename, O_RDONLY, 0); 119 if (fd < 0) { 120 UACCE_BUS_ERR("failed to open %s", filename); 121 return -EIO; 122 } 123 124 ret = read(fd, buf, sz); 125 if (ret > 0) { 126 /* Remove the last new line character. */ 127 if (buf[ret - 1] == '\n') { 128 buf[ret - 1] = '\0'; 129 ret--; 130 } 131 } 132 if (ret <= 0) { 133 UACCE_BUS_ERR("failed to read %s", filename); 134 ret = -EIO; 135 } 136 137 close(fd); 138 139 return ret; 140 } 141 142 /* 0 on success. Otherwise negative value is returned. */ 143 static int 144 uacce_read_attr_int(const char *dev_root, const char *attr, int *val) 145 { 146 char buf[RTE_UACCE_ATTR_MAX_SIZE] = {0}; 147 char *s = NULL; 148 int ret; 149 150 ret = uacce_read_attr(dev_root, attr, buf, sizeof(buf) - 1); 151 if (ret < 0) 152 return ret; 153 154 *val = strtol(buf, &s, 0); 155 if (s[0] != '\0') { 156 UACCE_BUS_ERR("read attr %s/%s expect an integer value", dev_root, attr); 157 return -EINVAL; 158 } 159 160 return 0; 161 } 162 163 /* 0 on success. Otherwise negative value is returned. */ 164 static int 165 uacce_read_attr_u32(const char *dev_root, const char *attr, uint32_t *val) 166 { 167 char buf[RTE_UACCE_ATTR_MAX_SIZE] = {0}; 168 char *s = NULL; 169 int ret; 170 171 ret = uacce_read_attr(dev_root, attr, buf, sizeof(buf) - 1); 172 if (ret < 0) 173 return ret; 174 175 *val = strtoul(buf, &s, 0); 176 if (s[0] != '\0') { 177 UACCE_BUS_ERR("read attr %s/%s expect an uint32 value", dev_root, attr); 178 return -EINVAL; 179 } 180 181 return 0; 182 } 183 184 static int 185 uacce_read_api(struct rte_uacce_device *dev) 186 { 187 int ret = uacce_read_attr(dev->dev_root, "api", dev->api, sizeof(dev->api) - 1); 188 if (ret < 0) 189 return ret; 190 return 0; 191 } 192 193 static int 194 uacce_read_algs(struct rte_uacce_device *dev) 195 { 196 int ret = uacce_read_attr(dev->dev_root, "algorithms", dev->algs, sizeof(dev->algs) - 1); 197 if (ret < 0) 198 return ret; 199 return 0; 200 } 201 202 static int 203 uacce_read_flags(struct rte_uacce_device *dev) 204 { 205 return uacce_read_attr_u32(dev->dev_root, "flags", &dev->flags); 206 } 207 208 static void 209 uacce_read_numa_node(struct rte_uacce_device *dev) 210 { 211 int ret = uacce_read_attr_int(dev->dev_root, "device/numa_node", &dev->numa_node); 212 if (ret != 0) { 213 UACCE_BUS_WARN("read attr numa_node failed! set to default"); 214 dev->numa_node = -1; 215 } 216 } 217 218 static int 219 uacce_read_qfrt_sz(struct rte_uacce_device *dev) 220 { 221 int ret = uacce_read_attr_u32(dev->dev_root, "region_mmio_size", 222 &dev->qfrt_sz[RTE_UACCE_QFRT_MMIO]); 223 if (ret != 0) 224 return ret; 225 return uacce_read_attr_u32(dev->dev_root, "region_dus_size", 226 &dev->qfrt_sz[RTE_UACCE_QFRT_DUS]); 227 } 228 229 static int 230 uacce_verify(struct rte_uacce_device *dev) 231 { 232 if (!(dev->flags & UACCE_DEV_FLGA_SVA)) { 233 UACCE_BUS_WARN("device %s don't support SVA, skip it!", dev->name); 234 return 1; /* >0 will skip this device. */ 235 } 236 237 return 0; 238 } 239 240 /* 241 * Scan one UACCE sysfs entry, and fill the devices list from it. 242 * It reads api/algs/flags/numa_node/region-size (please refer Linux kernel: 243 * Documentation/ABI/testing/sysfs-driver-uacce) and stores them for later 244 * device-driver matching, driver init... 245 */ 246 static int 247 uacce_scan_one(const char *dev_name) 248 { 249 struct rte_uacce_device *dev; 250 int ret; 251 252 dev = calloc(1, sizeof(*dev)); 253 if (!dev) 254 return -ENOMEM; 255 256 dev->device.bus = &uacce_bus.bus; 257 dev->device.name = dev->name; 258 dev->device.devargs = uacce_devargs_lookup(dev_name); 259 snprintf(dev->name, sizeof(dev->name), "%s", dev_name); 260 snprintf(dev->dev_root, sizeof(dev->dev_root), "%s/%s", 261 UACCE_BUS_CLASS_PATH, dev_name); 262 snprintf(dev->cdev_path, sizeof(dev->cdev_path), "/dev/%s", dev_name); 263 264 ret = uacce_read_api(dev); 265 if (ret != 0) 266 goto err; 267 ret = uacce_read_algs(dev); 268 if (ret != 0) 269 goto err; 270 ret = uacce_read_flags(dev); 271 if (ret != 0) 272 goto err; 273 uacce_read_numa_node(dev); 274 ret = uacce_read_qfrt_sz(dev); 275 if (ret != 0) 276 goto err; 277 278 ret = uacce_verify(dev); 279 if (ret != 0) 280 goto err; 281 282 TAILQ_INSERT_TAIL(&uacce_bus.device_list, dev, next); 283 return 0; 284 285 err: 286 free(dev); 287 return ret; 288 } 289 290 static int 291 uacce_scan(void) 292 { 293 struct dirent *e; 294 DIR *dir; 295 296 dir = opendir(UACCE_BUS_CLASS_PATH); 297 if (dir == NULL) { 298 UACCE_BUS_LOG(INFO, "open %s failed!", UACCE_BUS_CLASS_PATH); 299 return 0; 300 } 301 302 while ((e = readdir(dir)) != NULL) { 303 if (e->d_name[0] == '.') 304 continue; 305 306 if (strlen(e->d_name) >= RTE_DEV_NAME_MAX_LEN) { 307 UACCE_BUS_LOG(WARNING, "uacce device name %s too long, skip it!", 308 e->d_name); 309 continue; 310 } 311 312 if (uacce_ignore_device(e->d_name)) 313 continue; 314 315 if (uacce_scan_one(e->d_name) < 0) 316 goto error; 317 } 318 closedir(dir); 319 return 0; 320 321 error: 322 closedir(dir); 323 return -1; 324 } 325 326 static bool 327 uacce_match(const struct rte_uacce_driver *dr, const struct rte_uacce_device *dev) 328 { 329 const struct rte_uacce_id *id_table; 330 uint32_t len; 331 char *map; 332 333 for (id_table = dr->id_table; id_table->dev_api != NULL; id_table++) { 334 if (strcmp(id_table->dev_api, dev->api) != 0) 335 continue; 336 337 if (id_table->dev_alg == NULL) 338 return true; 339 340 /* The dev->algs's algrothims is separated by new line, for 341 * example: dev->algs could be: aaa\nbbbb\ncc, which has three 342 * algorithms: aaa, bbbb and cc. 343 * The id_table->dev_alg should be a single algrithm, e.g. bbbb. 344 */ 345 map = strstr(dev->algs, id_table->dev_alg); 346 if (map == NULL) 347 continue; 348 if (map != dev->algs && map[-1] != '\n') 349 continue; 350 len = strlen(id_table->dev_alg); 351 if (map[len] != '\0' && map[len] != '\n') 352 continue; 353 354 return true; 355 } 356 357 return false; 358 } 359 360 static int 361 uacce_probe_one_driver(struct rte_uacce_driver *dr, struct rte_uacce_device *dev) 362 { 363 const char *dev_name = dev->name; 364 bool already_probed; 365 int ret; 366 367 if (!uacce_match(dr, dev)) 368 /* Match of device and driver failed */ 369 return 1; 370 371 already_probed = rte_dev_is_probed(&dev->device); 372 if (already_probed) { 373 UACCE_BUS_INFO("device %s is already probed", dev_name); 374 return -EEXIST; 375 } 376 377 UACCE_BUS_DEBUG("probe device %s using driver %s", dev_name, dr->driver.name); 378 379 ret = dr->probe(dr, dev); 380 if (ret != 0) { 381 UACCE_BUS_ERR("probe device %s with driver %s failed %d", 382 dev_name, dr->driver.name, ret); 383 } else { 384 dev->device.driver = &dr->driver; 385 dev->driver = dr; 386 UACCE_BUS_DEBUG("probe device %s with driver %s success", 387 dev_name, dr->driver.name); 388 } 389 390 return ret; 391 } 392 393 static int 394 uacce_probe_all_drivers(struct rte_uacce_device *dev) 395 { 396 struct rte_uacce_driver *dr; 397 int rc; 398 399 FOREACH_DRIVER_ON_UACCEBUS(dr) { 400 rc = uacce_probe_one_driver(dr, dev); 401 if (rc < 0) 402 /* negative value is an error */ 403 return rc; 404 if (rc > 0) 405 /* positive value means driver doesn't support it */ 406 continue; 407 return 0; 408 } 409 410 return 1; 411 } 412 413 static int 414 uacce_probe(void) 415 { 416 size_t probed = 0, failed = 0; 417 struct rte_uacce_device *dev; 418 int ret; 419 420 FOREACH_DEVICE_ON_UACCEBUS(dev) { 421 probed++; 422 423 ret = uacce_probe_all_drivers(dev); 424 if (ret < 0) { 425 UACCE_BUS_LOG(ERR, "Requested device %s cannot be used", 426 dev->name); 427 rte_errno = errno; 428 failed++; 429 } 430 } 431 432 return (probed && probed == failed) ? -1 : 0; 433 } 434 435 static int 436 uacce_cleanup(void) 437 { 438 struct rte_uacce_device *dev, *tmp_dev; 439 int error = 0; 440 441 RTE_TAILQ_FOREACH_SAFE(dev, &uacce_bus.device_list, next, tmp_dev) { 442 struct rte_uacce_driver *dr = dev->driver; 443 int ret = 0; 444 445 if (dr == NULL || dr->remove == NULL) 446 goto free; 447 448 ret = dr->remove(dev); 449 if (ret < 0) { 450 rte_errno = errno; 451 error = -1; 452 } 453 dev->driver = NULL; 454 dev->device.driver = NULL; 455 456 free: 457 memset(dev, 0, sizeof(*dev)); 458 free(dev); 459 } 460 461 return error; 462 } 463 464 static int 465 uacce_plug(struct rte_device *dev) 466 { 467 return uacce_probe_all_drivers(RTE_DEV_TO_UACCE_DEV(dev)); 468 } 469 470 static int 471 uacce_detach_dev(struct rte_uacce_device *dev) 472 { 473 struct rte_uacce_driver *dr; 474 int ret = 0; 475 476 dr = dev->driver; 477 478 UACCE_BUS_DEBUG("detach device %s using driver: %s", dev->device.name, dr->driver.name); 479 480 if (dr->remove) { 481 ret = dr->remove(dev); 482 if (ret < 0) 483 return ret; 484 } 485 486 dev->driver = NULL; 487 dev->device.driver = NULL; 488 489 return 0; 490 } 491 492 static int 493 uacce_unplug(struct rte_device *dev) 494 { 495 struct rte_uacce_device *uacce_dev; 496 int ret; 497 498 uacce_dev = RTE_DEV_TO_UACCE_DEV(dev); 499 ret = uacce_detach_dev(uacce_dev); 500 if (ret == 0) { 501 TAILQ_REMOVE(&uacce_bus.device_list, uacce_dev, next); 502 rte_devargs_remove(dev->devargs); 503 free(uacce_dev); 504 } 505 506 return ret; 507 } 508 509 static struct rte_device * 510 uacce_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) 511 { 512 const struct rte_uacce_device *uacce_start; 513 struct rte_uacce_device *uacce_dev; 514 515 if (start != NULL) { 516 uacce_start = RTE_DEV_TO_UACCE_DEV_CONST(start); 517 uacce_dev = TAILQ_NEXT(uacce_start, next); 518 } else { 519 uacce_dev = TAILQ_FIRST(&uacce_bus.device_list); 520 } 521 522 while (uacce_dev != NULL) { 523 if (cmp(&uacce_dev->device, data) == 0) 524 return &uacce_dev->device; 525 uacce_dev = TAILQ_NEXT(uacce_dev, next); 526 } 527 528 return NULL; 529 } 530 531 static int 532 uacce_parse(const char *name, void *addr) 533 { 534 const char **out = addr; 535 int ret; 536 537 ret = strncmp(name, UACCE_DEV_PREFIX, strlen(UACCE_DEV_PREFIX)); 538 539 if (ret == 0 && addr) 540 *out = name; 541 542 return ret; 543 } 544 545 static int 546 uacce_dev_match(const struct rte_device *dev, const void *_kvlist) 547 { 548 const char *key = uacce_params_keys[RTE_UACCE_PARAM_NAME]; 549 const struct rte_kvargs *kvlist = _kvlist; 550 const char *name; 551 552 /* no kvlist arg, all devices match. */ 553 if (kvlist == NULL) 554 return 0; 555 556 /* if key is present in kvlist and does not match, filter device. */ 557 name = rte_kvargs_get(kvlist, key); 558 if (name != NULL && strcmp(name, dev->name)) 559 return -1; 560 561 return 0; 562 } 563 564 static void * 565 uacce_dev_iterate(const void *start, const char *str, 566 const struct rte_dev_iterator *it __rte_unused) 567 { 568 rte_bus_find_device_t find_device; 569 struct rte_kvargs *kvargs = NULL; 570 struct rte_device *dev; 571 572 if (str != NULL) { 573 kvargs = rte_kvargs_parse(str, uacce_params_keys); 574 if (kvargs == NULL) { 575 UACCE_BUS_ERR("cannot parse argument list %s", str); 576 return NULL; 577 } 578 } 579 find_device = uacce_bus.bus.find_device; 580 dev = find_device(start, uacce_dev_match, kvargs); 581 rte_kvargs_free(kvargs); 582 return dev; 583 } 584 585 int 586 rte_uacce_avail_queues(struct rte_uacce_device *dev) 587 { 588 int avails = 0; 589 int ret; 590 591 ret = uacce_read_attr_int(dev->dev_root, "available_instances", &avails); 592 if (ret == 0) 593 ret = avails; 594 595 return ret; 596 } 597 598 int 599 rte_uacce_queue_alloc(struct rte_uacce_device *dev, struct rte_uacce_qcontex *qctx) 600 { 601 memset(qctx, 0, sizeof(*qctx)); 602 603 qctx->fd = open(dev->cdev_path, O_RDWR | O_CLOEXEC); 604 if (qctx->fd >= 0) { 605 qctx->dev = dev; 606 return 0; 607 } 608 609 return -EIO; 610 } 611 612 void 613 rte_uacce_queue_free(struct rte_uacce_qcontex *qctx) 614 { 615 if (qctx->fd >= 0) 616 close(qctx->fd); 617 memset(qctx, 0, sizeof(*qctx)); 618 qctx->fd = -1; 619 } 620 621 int 622 rte_uacce_queue_start(struct rte_uacce_qcontex *qctx) 623 { 624 #define UACCE_CMD_START_Q _IO('W', 0) 625 return ioctl(qctx->fd, UACCE_CMD_START_Q); 626 } 627 628 int 629 rte_uacce_queue_ioctl(struct rte_uacce_qcontex *qctx, unsigned long cmd, void *arg) 630 { 631 if (arg == NULL) 632 return ioctl(qctx->fd, cmd); 633 634 return ioctl(qctx->fd, cmd, arg); 635 } 636 637 void * 638 rte_uacce_queue_mmap(struct rte_uacce_qcontex *qctx, enum rte_uacce_qfrt qfrt) 639 { 640 size_t size = qctx->dev->qfrt_sz[qfrt]; 641 off_t off = qfrt * getpagesize(); 642 void *addr; 643 644 if (size == 0 || qctx->qfrt_base[qfrt] != NULL) { 645 UACCE_BUS_ERR("failed to mmap for %s, size is zero or already mmapped!", 646 qctx->dev->name); 647 return NULL; 648 } 649 650 addr = rte_mem_map(NULL, size, RTE_PROT_READ | RTE_PROT_WRITE, RTE_MAP_SHARED, 651 qctx->fd, off); 652 if (addr == NULL) { 653 UACCE_BUS_ERR("failed to mmap for %s, qfrt %d err %s!", 654 qctx->dev->name, qfrt, rte_strerror(rte_errno)); 655 return NULL; 656 } 657 qctx->qfrt_base[qfrt] = addr; 658 659 return addr; 660 } 661 662 void 663 rte_uacce_queue_unmap(struct rte_uacce_qcontex *qctx, enum rte_uacce_qfrt qfrt) 664 { 665 if (qctx->qfrt_base[qfrt] != NULL) { 666 rte_mem_unmap(qctx->qfrt_base[qfrt], qctx->dev->qfrt_sz[qfrt]); 667 qctx->qfrt_base[qfrt] = NULL; 668 } 669 } 670 671 void 672 rte_uacce_register(struct rte_uacce_driver *driver) 673 { 674 TAILQ_INSERT_TAIL(&uacce_bus.driver_list, driver, next); 675 driver->bus = &uacce_bus; 676 } 677 678 void 679 rte_uacce_unregister(struct rte_uacce_driver *driver) 680 { 681 TAILQ_REMOVE(&uacce_bus.driver_list, driver, next); 682 driver->bus = NULL; 683 } 684 685 static struct rte_uacce_bus uacce_bus = { 686 .bus = { 687 .scan = uacce_scan, 688 .probe = uacce_probe, 689 .cleanup = uacce_cleanup, 690 .plug = uacce_plug, 691 .unplug = uacce_unplug, 692 .find_device = uacce_find_device, 693 .parse = uacce_parse, 694 .dev_iterate = uacce_dev_iterate, 695 }, 696 .device_list = TAILQ_HEAD_INITIALIZER(uacce_bus.device_list), 697 .driver_list = TAILQ_HEAD_INITIALIZER(uacce_bus.driver_list), 698 }; 699 700 RTE_REGISTER_BUS(uacce, uacce_bus.bus); 701 RTE_LOG_REGISTER_DEFAULT(uacce_bus_logtype, NOTICE); 702