1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <dev_driver.h> 15 #include <bus_driver.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "bus_vdev_driver.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 /* Forward declare to access virtual bus name */ 31 static struct rte_bus rte_vdev_bus; 32 33 34 static TAILQ_HEAD(, rte_vdev_device) vdev_device_list = 35 TAILQ_HEAD_INITIALIZER(vdev_device_list); 36 /* The lock needs to be recursive because a vdev can manage another vdev. */ 37 static rte_spinlock_recursive_t vdev_device_list_lock = 38 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 39 40 static TAILQ_HEAD(, rte_vdev_driver) vdev_driver_list = 41 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 42 43 struct vdev_custom_scan { 44 TAILQ_ENTRY(vdev_custom_scan) next; 45 rte_vdev_scan_callback callback; 46 void *user_arg; 47 }; 48 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 49 static struct vdev_custom_scans vdev_custom_scans = 50 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 51 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 52 53 /* register a driver */ 54 void 55 rte_vdev_register(struct rte_vdev_driver *driver) 56 { 57 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 58 } 59 60 /* unregister a driver */ 61 void 62 rte_vdev_unregister(struct rte_vdev_driver *driver) 63 { 64 TAILQ_REMOVE(&vdev_driver_list, driver, next); 65 } 66 67 int 68 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 69 { 70 struct vdev_custom_scan *custom_scan; 71 72 rte_spinlock_lock(&vdev_custom_scan_lock); 73 74 /* check if already registered */ 75 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 76 if (custom_scan->callback == callback && 77 custom_scan->user_arg == user_arg) 78 break; 79 } 80 81 if (custom_scan == NULL) { 82 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 83 if (custom_scan != NULL) { 84 custom_scan->callback = callback; 85 custom_scan->user_arg = user_arg; 86 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 87 } 88 } 89 90 rte_spinlock_unlock(&vdev_custom_scan_lock); 91 92 return (custom_scan == NULL) ? -1 : 0; 93 } 94 95 int 96 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 97 { 98 struct vdev_custom_scan *custom_scan, *tmp_scan; 99 100 rte_spinlock_lock(&vdev_custom_scan_lock); 101 RTE_TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, 102 tmp_scan) { 103 if (custom_scan->callback != callback || 104 (custom_scan->user_arg != (void *)-1 && 105 custom_scan->user_arg != user_arg)) 106 continue; 107 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 108 free(custom_scan); 109 } 110 rte_spinlock_unlock(&vdev_custom_scan_lock); 111 112 return 0; 113 } 114 115 static int 116 vdev_parse(const char *name, void *addr) 117 { 118 struct rte_vdev_driver **out = addr; 119 struct rte_vdev_driver *driver = NULL; 120 121 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 122 if (strncmp(driver->driver.name, name, 123 strlen(driver->driver.name)) == 0) 124 break; 125 if (driver->driver.alias && 126 strncmp(driver->driver.alias, name, 127 strlen(driver->driver.alias)) == 0) 128 break; 129 } 130 if (driver != NULL && 131 addr != NULL) 132 *out = driver; 133 return driver == NULL; 134 } 135 136 static int 137 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 138 { 139 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 140 const struct rte_vdev_driver *driver; 141 142 if (!vdev) { 143 rte_errno = EINVAL; 144 return -1; 145 } 146 147 if (!vdev->device.driver) { 148 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 149 return 1; 150 } 151 152 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 153 driver); 154 155 if (driver->dma_map) 156 return driver->dma_map(vdev, addr, iova, len); 157 158 return 0; 159 } 160 161 static int 162 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 163 { 164 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 165 const struct rte_vdev_driver *driver; 166 167 if (!vdev) { 168 rte_errno = EINVAL; 169 return -1; 170 } 171 172 if (!vdev->device.driver) { 173 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 174 return 1; 175 } 176 177 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 178 driver); 179 180 if (driver->dma_unmap) 181 return driver->dma_unmap(vdev, addr, iova, len); 182 183 return 0; 184 } 185 186 static int 187 vdev_probe_all_drivers(struct rte_vdev_device *dev) 188 { 189 const char *name; 190 struct rte_vdev_driver *driver; 191 enum rte_iova_mode iova_mode; 192 int ret; 193 194 if (rte_dev_is_probed(&dev->device)) 195 return -EEXIST; 196 197 name = rte_vdev_device_name(dev); 198 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 199 200 if (vdev_parse(name, &driver)) 201 return -1; 202 203 iova_mode = rte_eal_iova_mode(); 204 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) { 205 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing", 206 name); 207 return -1; 208 } 209 210 ret = driver->probe(dev); 211 if (ret == 0) 212 dev->device.driver = &driver->driver; 213 return ret; 214 } 215 216 /* The caller shall be responsible for thread-safe */ 217 static struct rte_vdev_device * 218 find_vdev(const char *name) 219 { 220 struct rte_vdev_device *dev; 221 222 if (!name) 223 return NULL; 224 225 TAILQ_FOREACH(dev, &vdev_device_list, next) { 226 const char *devname = rte_vdev_device_name(dev); 227 228 if (!strcmp(devname, name)) 229 return dev; 230 } 231 232 return NULL; 233 } 234 235 static struct rte_devargs * 236 alloc_devargs(const char *name, const char *args) 237 { 238 struct rte_devargs *devargs; 239 int ret; 240 241 devargs = calloc(1, sizeof(*devargs)); 242 if (!devargs) 243 return NULL; 244 245 devargs->bus = &rte_vdev_bus; 246 if (args) 247 devargs->data = strdup(args); 248 else 249 devargs->data = strdup(""); 250 if (devargs->data == NULL) { 251 free(devargs); 252 return NULL; 253 } 254 devargs->args = devargs->data; 255 256 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 257 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 258 rte_devargs_reset(devargs); 259 free(devargs); 260 return NULL; 261 } 262 263 return devargs; 264 } 265 266 static int 267 insert_vdev(const char *name, const char *args, 268 struct rte_vdev_device **p_dev, 269 bool init) 270 { 271 struct rte_vdev_device *dev; 272 struct rte_devargs *devargs; 273 int ret; 274 275 if (name == NULL) 276 return -EINVAL; 277 278 devargs = alloc_devargs(name, args); 279 280 if (!devargs) 281 return -ENOMEM; 282 283 dev = calloc(1, sizeof(*dev)); 284 if (!dev) { 285 ret = -ENOMEM; 286 goto fail; 287 } 288 289 dev->device.bus = &rte_vdev_bus; 290 dev->device.numa_node = SOCKET_ID_ANY; 291 292 if (find_vdev(name)) { 293 /* 294 * A vdev is expected to have only one port. 295 * So there is no reason to try probing again, 296 * even with new arguments. 297 */ 298 ret = -EEXIST; 299 goto fail; 300 } 301 302 if (init) 303 rte_devargs_insert(&devargs); 304 dev->device.devargs = devargs; 305 dev->device.name = devargs->name; 306 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 307 308 if (p_dev) 309 *p_dev = dev; 310 311 return 0; 312 fail: 313 rte_devargs_reset(devargs); 314 free(devargs); 315 free(dev); 316 return ret; 317 } 318 319 int 320 rte_vdev_init(const char *name, const char *args) 321 { 322 struct rte_vdev_device *dev; 323 int ret; 324 325 rte_spinlock_recursive_lock(&vdev_device_list_lock); 326 ret = insert_vdev(name, args, &dev, true); 327 if (ret == 0) { 328 ret = vdev_probe_all_drivers(dev); 329 if (ret) { 330 if (ret > 0) 331 VDEV_LOG(ERR, "no driver found for %s", name); 332 /* If fails, remove it from vdev list */ 333 TAILQ_REMOVE(&vdev_device_list, dev, next); 334 rte_devargs_remove(dev->device.devargs); 335 free(dev); 336 } 337 } 338 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 339 return ret; 340 } 341 342 static int 343 vdev_remove_driver(struct rte_vdev_device *dev) 344 { 345 const char *name = rte_vdev_device_name(dev); 346 const struct rte_vdev_driver *driver; 347 348 if (!dev->device.driver) { 349 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 350 return 1; 351 } 352 353 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 354 driver); 355 return driver->remove(dev); 356 } 357 358 int 359 rte_vdev_uninit(const char *name) 360 { 361 struct rte_vdev_device *dev; 362 int ret; 363 364 if (name == NULL) 365 return -EINVAL; 366 367 rte_spinlock_recursive_lock(&vdev_device_list_lock); 368 369 dev = find_vdev(name); 370 if (!dev) { 371 ret = -ENOENT; 372 goto unlock; 373 } 374 375 ret = vdev_remove_driver(dev); 376 if (ret) 377 goto unlock; 378 379 TAILQ_REMOVE(&vdev_device_list, dev, next); 380 rte_devargs_remove(dev->device.devargs); 381 free(dev); 382 383 unlock: 384 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 385 return ret; 386 } 387 388 struct vdev_param { 389 #define VDEV_SCAN_REQ 1 390 #define VDEV_SCAN_ONE 2 391 #define VDEV_SCAN_REP 3 392 int type; 393 int num; 394 char name[RTE_DEV_NAME_MAX_LEN]; 395 }; 396 397 static int vdev_plug(struct rte_device *dev); 398 399 /** 400 * This function works as the action for both primary and secondary process 401 * for static vdev discovery when a secondary process is booting. 402 * 403 * step 1, secondary process sends a sync request to ask for vdev in primary; 404 * step 2, primary process receives the request, and send vdevs one by one; 405 * step 3, primary process sends back reply, which indicates how many vdevs 406 * are sent. 407 */ 408 static int 409 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 410 { 411 struct rte_vdev_device *dev; 412 struct rte_mp_msg mp_resp; 413 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 414 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 415 const char *devname; 416 int num; 417 int ret; 418 419 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 420 mp_resp.len_param = sizeof(*ou); 421 mp_resp.num_fds = 0; 422 423 switch (in->type) { 424 case VDEV_SCAN_REQ: 425 ou->type = VDEV_SCAN_ONE; 426 ou->num = 1; 427 num = 0; 428 429 rte_spinlock_recursive_lock(&vdev_device_list_lock); 430 TAILQ_FOREACH(dev, &vdev_device_list, next) { 431 devname = rte_vdev_device_name(dev); 432 if (strlen(devname) == 0) { 433 VDEV_LOG(INFO, "vdev with no name is not sent"); 434 continue; 435 } 436 VDEV_LOG(INFO, "send vdev, %s", devname); 437 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 438 if (rte_mp_sendmsg(&mp_resp) < 0) 439 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 440 devname, strerror(rte_errno)); 441 num++; 442 } 443 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 444 445 ou->type = VDEV_SCAN_REP; 446 ou->num = num; 447 if (rte_mp_reply(&mp_resp, peer) < 0) 448 VDEV_LOG(ERR, "Failed to reply a scan request"); 449 break; 450 case VDEV_SCAN_ONE: 451 VDEV_LOG(INFO, "receive vdev, %s", in->name); 452 ret = insert_vdev(in->name, NULL, NULL, false); 453 if (ret == -EEXIST) 454 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 455 else if (ret < 0) 456 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 457 break; 458 default: 459 VDEV_LOG(ERR, "vdev cannot recognize this message"); 460 } 461 462 return 0; 463 } 464 465 static int 466 vdev_scan(void) 467 { 468 struct rte_vdev_device *dev; 469 struct rte_devargs *devargs; 470 struct vdev_custom_scan *custom_scan; 471 472 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 473 rte_errno != EEXIST) { 474 /* for primary, unsupported IPC is not an error */ 475 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 476 rte_errno == ENOTSUP) 477 goto scan; 478 VDEV_LOG(ERR, "Failed to add vdev mp action"); 479 return -1; 480 } 481 482 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 483 struct rte_mp_msg mp_req, *mp_rep; 484 struct rte_mp_reply mp_reply; 485 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 486 struct vdev_param *req = (struct vdev_param *)mp_req.param; 487 struct vdev_param *resp; 488 489 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 490 mp_req.len_param = sizeof(*req); 491 mp_req.num_fds = 0; 492 req->type = VDEV_SCAN_REQ; 493 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 494 mp_reply.nb_received == 1) { 495 mp_rep = &mp_reply.msgs[0]; 496 resp = (struct vdev_param *)mp_rep->param; 497 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 498 free(mp_reply.msgs); 499 } else 500 VDEV_LOG(ERR, "Failed to request vdev from primary"); 501 502 /* Fall through to allow private vdevs in secondary process */ 503 } 504 505 scan: 506 /* call custom scan callbacks if any */ 507 rte_spinlock_lock(&vdev_custom_scan_lock); 508 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 509 if (custom_scan->callback != NULL) 510 /* 511 * the callback should update devargs list 512 * by calling rte_devargs_insert() with 513 * devargs.bus = rte_bus_find_by_name("vdev"); 514 * devargs.type = RTE_DEVTYPE_VIRTUAL; 515 * devargs.policy = RTE_DEV_ALLOWED; 516 */ 517 custom_scan->callback(custom_scan->user_arg); 518 } 519 rte_spinlock_unlock(&vdev_custom_scan_lock); 520 521 /* for virtual devices we scan the devargs_list populated via cmdline */ 522 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 523 524 dev = calloc(1, sizeof(*dev)); 525 if (!dev) 526 return -1; 527 528 rte_spinlock_recursive_lock(&vdev_device_list_lock); 529 530 if (find_vdev(devargs->name)) { 531 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 532 free(dev); 533 continue; 534 } 535 536 dev->device.bus = &rte_vdev_bus; 537 dev->device.devargs = devargs; 538 dev->device.numa_node = SOCKET_ID_ANY; 539 dev->device.name = devargs->name; 540 541 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 542 543 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 544 } 545 546 return 0; 547 } 548 549 static int 550 vdev_probe(void) 551 { 552 struct rte_vdev_device *dev; 553 int r, ret = 0; 554 555 /* call the init function for each virtual device */ 556 TAILQ_FOREACH(dev, &vdev_device_list, next) { 557 /* we don't use the vdev lock here, as it's only used in DPDK 558 * initialization; and we don't want to hold such a lock when 559 * we call each driver probe. 560 */ 561 562 r = vdev_probe_all_drivers(dev); 563 if (r != 0) { 564 if (r == -EEXIST) 565 continue; 566 VDEV_LOG(ERR, "failed to initialize %s device", 567 rte_vdev_device_name(dev)); 568 ret = -1; 569 } 570 } 571 572 return ret; 573 } 574 575 static int 576 vdev_cleanup(void) 577 { 578 struct rte_vdev_device *dev, *tmp_dev; 579 int error = 0; 580 581 RTE_TAILQ_FOREACH_SAFE(dev, &vdev_device_list, next, tmp_dev) { 582 const struct rte_vdev_driver *drv; 583 int ret = 0; 584 585 if (dev->device.driver == NULL) 586 goto free; 587 588 drv = container_of(dev->device.driver, const struct rte_vdev_driver, driver); 589 590 if (drv->remove == NULL) 591 goto free; 592 593 ret = drv->remove(dev); 594 if (ret < 0) 595 error = -1; 596 597 dev->device.driver = NULL; 598 free: 599 free(dev); 600 } 601 602 return error; 603 } 604 605 struct rte_device * 606 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 607 const void *data) 608 { 609 const struct rte_vdev_device *vstart; 610 struct rte_vdev_device *dev; 611 612 rte_spinlock_recursive_lock(&vdev_device_list_lock); 613 if (start != NULL) { 614 vstart = RTE_DEV_TO_VDEV_CONST(start); 615 dev = TAILQ_NEXT(vstart, next); 616 } else { 617 dev = TAILQ_FIRST(&vdev_device_list); 618 } 619 while (dev != NULL) { 620 if (cmp(&dev->device, data) == 0) 621 break; 622 dev = TAILQ_NEXT(dev, next); 623 } 624 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 625 626 return dev ? &dev->device : NULL; 627 } 628 629 static int 630 vdev_plug(struct rte_device *dev) 631 { 632 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 633 } 634 635 static int 636 vdev_unplug(struct rte_device *dev) 637 { 638 return rte_vdev_uninit(dev->name); 639 } 640 641 static enum rte_iova_mode 642 vdev_get_iommu_class(void) 643 { 644 const char *name; 645 struct rte_vdev_device *dev; 646 struct rte_vdev_driver *driver; 647 648 TAILQ_FOREACH(dev, &vdev_device_list, next) { 649 name = rte_vdev_device_name(dev); 650 if (vdev_parse(name, &driver)) 651 continue; 652 653 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) 654 return RTE_IOVA_VA; 655 } 656 657 return RTE_IOVA_DC; 658 } 659 660 static struct rte_bus rte_vdev_bus = { 661 .scan = vdev_scan, 662 .probe = vdev_probe, 663 .cleanup = vdev_cleanup, 664 .find_device = rte_vdev_find_device, 665 .plug = vdev_plug, 666 .unplug = vdev_unplug, 667 .parse = vdev_parse, 668 .dma_map = vdev_dma_map, 669 .dma_unmap = vdev_dma_unmap, 670 .get_iommu_class = vdev_get_iommu_class, 671 .dev_iterate = rte_vdev_dev_iterate, 672 }; 673 674 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 675 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE); 676