1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <dev_driver.h> 15 #include <bus_driver.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "bus_vdev_driver.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 /* Forward declare to access virtual bus name */ 31 static struct rte_bus rte_vdev_bus; 32 33 34 static TAILQ_HEAD(, rte_vdev_device) vdev_device_list = 35 TAILQ_HEAD_INITIALIZER(vdev_device_list); 36 /* The lock needs to be recursive because a vdev can manage another vdev. */ 37 static rte_spinlock_recursive_t vdev_device_list_lock = 38 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 39 40 static TAILQ_HEAD(, rte_vdev_driver) vdev_driver_list = 41 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 42 43 struct vdev_custom_scan { 44 TAILQ_ENTRY(vdev_custom_scan) next; 45 rte_vdev_scan_callback callback; 46 void *user_arg; 47 }; 48 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 49 static struct vdev_custom_scans vdev_custom_scans = 50 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 51 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 52 53 /* register a driver */ 54 void 55 rte_vdev_register(struct rte_vdev_driver *driver) 56 { 57 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 58 } 59 60 /* unregister a driver */ 61 void 62 rte_vdev_unregister(struct rte_vdev_driver *driver) 63 { 64 TAILQ_REMOVE(&vdev_driver_list, driver, next); 65 } 66 67 int 68 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 69 { 70 struct vdev_custom_scan *custom_scan; 71 72 rte_spinlock_lock(&vdev_custom_scan_lock); 73 74 /* check if already registered */ 75 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 76 if (custom_scan->callback == callback && 77 custom_scan->user_arg == user_arg) 78 break; 79 } 80 81 if (custom_scan == NULL) { 82 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 83 if (custom_scan != NULL) { 84 custom_scan->callback = callback; 85 custom_scan->user_arg = user_arg; 86 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 87 } 88 } 89 90 rte_spinlock_unlock(&vdev_custom_scan_lock); 91 92 return (custom_scan == NULL) ? -1 : 0; 93 } 94 95 int 96 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 97 { 98 struct vdev_custom_scan *custom_scan, *tmp_scan; 99 100 rte_spinlock_lock(&vdev_custom_scan_lock); 101 RTE_TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, 102 tmp_scan) { 103 if (custom_scan->callback != callback || 104 (custom_scan->user_arg != (void *)-1 && 105 custom_scan->user_arg != user_arg)) 106 continue; 107 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 108 free(custom_scan); 109 } 110 rte_spinlock_unlock(&vdev_custom_scan_lock); 111 112 return 0; 113 } 114 115 static int 116 vdev_parse(const char *name, void *addr) 117 { 118 struct rte_vdev_driver **out = addr; 119 struct rte_vdev_driver *driver = NULL; 120 121 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 122 if (strncmp(driver->driver.name, name, 123 strlen(driver->driver.name)) == 0) 124 break; 125 if (driver->driver.alias && 126 strncmp(driver->driver.alias, name, 127 strlen(driver->driver.alias)) == 0) 128 break; 129 } 130 if (driver != NULL && 131 addr != NULL) 132 *out = driver; 133 return driver == NULL; 134 } 135 136 static int 137 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 138 { 139 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 140 const struct rte_vdev_driver *driver; 141 142 if (!vdev) { 143 rte_errno = EINVAL; 144 return -1; 145 } 146 147 if (!vdev->device.driver) { 148 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 149 return 1; 150 } 151 152 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 153 driver); 154 155 if (driver->dma_map) 156 return driver->dma_map(vdev, addr, iova, len); 157 158 return 0; 159 } 160 161 static int 162 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 163 { 164 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 165 const struct rte_vdev_driver *driver; 166 167 if (!vdev) { 168 rte_errno = EINVAL; 169 return -1; 170 } 171 172 if (!vdev->device.driver) { 173 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 174 return 1; 175 } 176 177 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 178 driver); 179 180 if (driver->dma_unmap) 181 return driver->dma_unmap(vdev, addr, iova, len); 182 183 return 0; 184 } 185 186 static int 187 vdev_probe_all_drivers(struct rte_vdev_device *dev) 188 { 189 const char *name; 190 struct rte_vdev_driver *driver; 191 enum rte_iova_mode iova_mode; 192 int ret; 193 194 if (rte_dev_is_probed(&dev->device)) 195 return -EEXIST; 196 197 name = rte_vdev_device_name(dev); 198 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 199 200 if (vdev_parse(name, &driver)) 201 return -1; 202 203 iova_mode = rte_eal_iova_mode(); 204 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) { 205 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing", 206 name); 207 return -1; 208 } 209 210 ret = driver->probe(dev); 211 if (ret == 0) 212 dev->device.driver = &driver->driver; 213 return ret; 214 } 215 216 /* The caller shall be responsible for thread-safe */ 217 static struct rte_vdev_device * 218 find_vdev(const char *name) 219 { 220 struct rte_vdev_device *dev; 221 222 if (!name) 223 return NULL; 224 225 TAILQ_FOREACH(dev, &vdev_device_list, next) { 226 const char *devname = rte_vdev_device_name(dev); 227 228 if (!strcmp(devname, name)) 229 return dev; 230 } 231 232 return NULL; 233 } 234 235 static struct rte_devargs * 236 alloc_devargs(const char *name, const char *args) 237 { 238 struct rte_devargs *devargs; 239 int ret; 240 241 devargs = calloc(1, sizeof(*devargs)); 242 if (!devargs) 243 return NULL; 244 245 devargs->bus = &rte_vdev_bus; 246 if (args) 247 devargs->data = strdup(args); 248 else 249 devargs->data = strdup(""); 250 if (devargs->data == NULL) { 251 free(devargs); 252 return NULL; 253 } 254 devargs->args = devargs->data; 255 256 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 257 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 258 rte_devargs_reset(devargs); 259 free(devargs); 260 return NULL; 261 } 262 263 return devargs; 264 } 265 266 static int 267 insert_vdev(const char *name, const char *args, 268 struct rte_vdev_device **p_dev, 269 bool init) 270 { 271 struct rte_vdev_device *dev; 272 struct rte_devargs *devargs; 273 int ret; 274 275 if (name == NULL) 276 return -EINVAL; 277 278 devargs = alloc_devargs(name, args); 279 if (!devargs) 280 return -ENOMEM; 281 282 dev = calloc(1, sizeof(*dev)); 283 if (!dev) { 284 ret = -ENOMEM; 285 goto fail; 286 } 287 288 dev->device.bus = &rte_vdev_bus; 289 dev->device.numa_node = SOCKET_ID_ANY; 290 dev->device.name = devargs->name; 291 292 if (find_vdev(name)) { 293 /* 294 * A vdev is expected to have only one port. 295 * So there is no reason to try probing again, 296 * even with new arguments. 297 */ 298 ret = -EEXIST; 299 goto fail; 300 } 301 302 if (init) 303 rte_devargs_insert(&devargs); 304 dev->device.devargs = devargs; 305 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 306 307 if (p_dev) 308 *p_dev = dev; 309 310 return 0; 311 fail: 312 rte_devargs_reset(devargs); 313 free(devargs); 314 free(dev); 315 return ret; 316 } 317 318 int 319 rte_vdev_init(const char *name, const char *args) 320 { 321 struct rte_vdev_device *dev; 322 int ret; 323 324 rte_spinlock_recursive_lock(&vdev_device_list_lock); 325 ret = insert_vdev(name, args, &dev, true); 326 if (ret == 0) { 327 ret = vdev_probe_all_drivers(dev); 328 if (ret) { 329 if (ret > 0) 330 VDEV_LOG(ERR, "no driver found for %s", name); 331 /* If fails, remove it from vdev list */ 332 TAILQ_REMOVE(&vdev_device_list, dev, next); 333 rte_devargs_remove(dev->device.devargs); 334 free(dev); 335 } 336 } 337 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 338 return ret; 339 } 340 341 static int 342 vdev_remove_driver(struct rte_vdev_device *dev) 343 { 344 const char *name = rte_vdev_device_name(dev); 345 const struct rte_vdev_driver *driver; 346 347 if (!dev->device.driver) { 348 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 349 return 1; 350 } 351 352 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 353 driver); 354 return driver->remove(dev); 355 } 356 357 int 358 rte_vdev_uninit(const char *name) 359 { 360 struct rte_vdev_device *dev; 361 int ret; 362 363 if (name == NULL) 364 return -EINVAL; 365 366 rte_spinlock_recursive_lock(&vdev_device_list_lock); 367 368 dev = find_vdev(name); 369 if (!dev) { 370 ret = -ENOENT; 371 goto unlock; 372 } 373 374 ret = vdev_remove_driver(dev); 375 if (ret) 376 goto unlock; 377 378 TAILQ_REMOVE(&vdev_device_list, dev, next); 379 rte_devargs_remove(dev->device.devargs); 380 free(dev); 381 382 unlock: 383 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 384 return ret; 385 } 386 387 struct vdev_param { 388 #define VDEV_SCAN_REQ 1 389 #define VDEV_SCAN_ONE 2 390 #define VDEV_SCAN_REP 3 391 int type; 392 int num; 393 char name[RTE_DEV_NAME_MAX_LEN]; 394 }; 395 396 static int vdev_plug(struct rte_device *dev); 397 398 /** 399 * This function works as the action for both primary and secondary process 400 * for static vdev discovery when a secondary process is booting. 401 * 402 * step 1, secondary process sends a sync request to ask for vdev in primary; 403 * step 2, primary process receives the request, and send vdevs one by one; 404 * step 3, primary process sends back reply, which indicates how many vdevs 405 * are sent. 406 */ 407 static int 408 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 409 { 410 struct rte_vdev_device *dev; 411 struct rte_mp_msg mp_resp; 412 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 413 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 414 const char *devname; 415 int num; 416 int ret; 417 418 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 419 mp_resp.len_param = sizeof(*ou); 420 mp_resp.num_fds = 0; 421 422 switch (in->type) { 423 case VDEV_SCAN_REQ: 424 ou->type = VDEV_SCAN_ONE; 425 ou->num = 1; 426 num = 0; 427 428 rte_spinlock_recursive_lock(&vdev_device_list_lock); 429 TAILQ_FOREACH(dev, &vdev_device_list, next) { 430 devname = rte_vdev_device_name(dev); 431 if (strlen(devname) == 0) { 432 VDEV_LOG(INFO, "vdev with no name is not sent"); 433 continue; 434 } 435 VDEV_LOG(INFO, "send vdev, %s", devname); 436 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 437 if (rte_mp_sendmsg(&mp_resp) < 0) 438 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 439 devname, strerror(rte_errno)); 440 num++; 441 } 442 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 443 444 ou->type = VDEV_SCAN_REP; 445 ou->num = num; 446 if (rte_mp_reply(&mp_resp, peer) < 0) 447 VDEV_LOG(ERR, "Failed to reply a scan request"); 448 break; 449 case VDEV_SCAN_ONE: 450 VDEV_LOG(INFO, "receive vdev, %s", in->name); 451 ret = insert_vdev(in->name, NULL, NULL, false); 452 if (ret == -EEXIST) 453 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 454 else if (ret < 0) 455 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 456 break; 457 default: 458 VDEV_LOG(ERR, "vdev cannot recognize this message"); 459 } 460 461 return 0; 462 } 463 464 static int 465 vdev_scan(void) 466 { 467 struct rte_vdev_device *dev; 468 struct rte_devargs *devargs; 469 struct vdev_custom_scan *custom_scan; 470 471 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 472 rte_errno != EEXIST) { 473 /* for primary, unsupported IPC is not an error */ 474 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 475 rte_errno == ENOTSUP) 476 goto scan; 477 VDEV_LOG(ERR, "Failed to add vdev mp action"); 478 return -1; 479 } 480 481 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 482 struct rte_mp_msg mp_req, *mp_rep; 483 struct rte_mp_reply mp_reply; 484 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 485 struct vdev_param *req = (struct vdev_param *)mp_req.param; 486 struct vdev_param *resp; 487 488 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 489 mp_req.len_param = sizeof(*req); 490 mp_req.num_fds = 0; 491 req->type = VDEV_SCAN_REQ; 492 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 493 mp_reply.nb_received == 1) { 494 mp_rep = &mp_reply.msgs[0]; 495 resp = (struct vdev_param *)mp_rep->param; 496 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 497 free(mp_reply.msgs); 498 } else 499 VDEV_LOG(ERR, "Failed to request vdev from primary"); 500 501 /* Fall through to allow private vdevs in secondary process */ 502 } 503 504 scan: 505 /* call custom scan callbacks if any */ 506 rte_spinlock_lock(&vdev_custom_scan_lock); 507 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 508 if (custom_scan->callback != NULL) 509 /* 510 * the callback should update devargs list 511 * by calling rte_devargs_insert() with 512 * devargs.bus = rte_bus_find_by_name("vdev"); 513 * devargs.type = RTE_DEVTYPE_VIRTUAL; 514 * devargs.policy = RTE_DEV_ALLOWED; 515 */ 516 custom_scan->callback(custom_scan->user_arg); 517 } 518 rte_spinlock_unlock(&vdev_custom_scan_lock); 519 520 /* for virtual devices we scan the devargs_list populated via cmdline */ 521 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 522 523 dev = calloc(1, sizeof(*dev)); 524 if (!dev) 525 return -1; 526 527 rte_spinlock_recursive_lock(&vdev_device_list_lock); 528 529 if (find_vdev(devargs->name)) { 530 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 531 free(dev); 532 continue; 533 } 534 535 dev->device.bus = &rte_vdev_bus; 536 dev->device.devargs = devargs; 537 dev->device.numa_node = SOCKET_ID_ANY; 538 dev->device.name = devargs->name; 539 540 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 541 542 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 543 } 544 545 return 0; 546 } 547 548 static int 549 vdev_probe(void) 550 { 551 struct rte_vdev_device *dev; 552 int r, ret = 0; 553 554 /* call the init function for each virtual device */ 555 TAILQ_FOREACH(dev, &vdev_device_list, next) { 556 /* we don't use the vdev lock here, as it's only used in DPDK 557 * initialization; and we don't want to hold such a lock when 558 * we call each driver probe. 559 */ 560 561 r = vdev_probe_all_drivers(dev); 562 if (r != 0) { 563 if (r == -EEXIST) 564 continue; 565 VDEV_LOG(ERR, "failed to initialize %s device", 566 rte_vdev_device_name(dev)); 567 ret = -1; 568 } 569 } 570 571 return ret; 572 } 573 574 static int 575 vdev_cleanup(void) 576 { 577 struct rte_vdev_device *dev, *tmp_dev; 578 int error = 0; 579 580 RTE_TAILQ_FOREACH_SAFE(dev, &vdev_device_list, next, tmp_dev) { 581 const struct rte_vdev_driver *drv; 582 int ret = 0; 583 584 if (dev->device.driver == NULL) 585 goto free; 586 587 drv = container_of(dev->device.driver, const struct rte_vdev_driver, driver); 588 589 if (drv->remove == NULL) 590 goto free; 591 592 ret = drv->remove(dev); 593 if (ret < 0) 594 error = -1; 595 596 dev->device.driver = NULL; 597 free: 598 free(dev); 599 } 600 601 return error; 602 } 603 604 struct rte_device * 605 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 606 const void *data) 607 { 608 const struct rte_vdev_device *vstart; 609 struct rte_vdev_device *dev; 610 611 rte_spinlock_recursive_lock(&vdev_device_list_lock); 612 if (start != NULL) { 613 vstart = RTE_DEV_TO_VDEV_CONST(start); 614 dev = TAILQ_NEXT(vstart, next); 615 } else { 616 dev = TAILQ_FIRST(&vdev_device_list); 617 } 618 while (dev != NULL) { 619 if (cmp(&dev->device, data) == 0) 620 break; 621 dev = TAILQ_NEXT(dev, next); 622 } 623 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 624 625 return dev ? &dev->device : NULL; 626 } 627 628 static int 629 vdev_plug(struct rte_device *dev) 630 { 631 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 632 } 633 634 static int 635 vdev_unplug(struct rte_device *dev) 636 { 637 return rte_vdev_uninit(dev->name); 638 } 639 640 static enum rte_iova_mode 641 vdev_get_iommu_class(void) 642 { 643 const char *name; 644 struct rte_vdev_device *dev; 645 struct rte_vdev_driver *driver; 646 647 TAILQ_FOREACH(dev, &vdev_device_list, next) { 648 name = rte_vdev_device_name(dev); 649 if (vdev_parse(name, &driver)) 650 continue; 651 652 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) 653 return RTE_IOVA_VA; 654 } 655 656 return RTE_IOVA_DC; 657 } 658 659 static struct rte_bus rte_vdev_bus = { 660 .scan = vdev_scan, 661 .probe = vdev_probe, 662 .cleanup = vdev_cleanup, 663 .find_device = rte_vdev_find_device, 664 .plug = vdev_plug, 665 .unplug = vdev_unplug, 666 .parse = vdev_parse, 667 .dma_map = vdev_dma_map, 668 .dma_unmap = vdev_dma_unmap, 669 .get_iommu_class = vdev_get_iommu_class, 670 .dev_iterate = rte_vdev_dev_iterate, 671 }; 672 673 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 674 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE); 675