1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <rte_dev.h> 15 #include <rte_bus.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "rte_bus_vdev.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 /* Forward declare to access virtual bus name */ 31 static struct rte_bus rte_vdev_bus; 32 33 /** Double linked list of virtual device drivers. */ 34 TAILQ_HEAD(vdev_device_list, rte_vdev_device); 35 36 static struct vdev_device_list vdev_device_list = 37 TAILQ_HEAD_INITIALIZER(vdev_device_list); 38 /* The lock needs to be recursive because a vdev can manage another vdev. */ 39 static rte_spinlock_recursive_t vdev_device_list_lock = 40 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 41 42 static struct vdev_driver_list vdev_driver_list = 43 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 44 45 struct vdev_custom_scan { 46 TAILQ_ENTRY(vdev_custom_scan) next; 47 rte_vdev_scan_callback callback; 48 void *user_arg; 49 }; 50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 51 static struct vdev_custom_scans vdev_custom_scans = 52 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 54 55 /* register a driver */ 56 void 57 rte_vdev_register(struct rte_vdev_driver *driver) 58 { 59 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 60 } 61 62 /* unregister a driver */ 63 void 64 rte_vdev_unregister(struct rte_vdev_driver *driver) 65 { 66 TAILQ_REMOVE(&vdev_driver_list, driver, next); 67 } 68 69 int 70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 71 { 72 struct vdev_custom_scan *custom_scan; 73 74 rte_spinlock_lock(&vdev_custom_scan_lock); 75 76 /* check if already registered */ 77 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 78 if (custom_scan->callback == callback && 79 custom_scan->user_arg == user_arg) 80 break; 81 } 82 83 if (custom_scan == NULL) { 84 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 85 if (custom_scan != NULL) { 86 custom_scan->callback = callback; 87 custom_scan->user_arg = user_arg; 88 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 89 } 90 } 91 92 rte_spinlock_unlock(&vdev_custom_scan_lock); 93 94 return (custom_scan == NULL) ? -1 : 0; 95 } 96 97 int 98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 99 { 100 struct vdev_custom_scan *custom_scan, *tmp_scan; 101 102 rte_spinlock_lock(&vdev_custom_scan_lock); 103 TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, tmp_scan) { 104 if (custom_scan->callback != callback || 105 (custom_scan->user_arg != (void *)-1 && 106 custom_scan->user_arg != user_arg)) 107 continue; 108 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 109 free(custom_scan); 110 } 111 rte_spinlock_unlock(&vdev_custom_scan_lock); 112 113 return 0; 114 } 115 116 static int 117 vdev_parse(const char *name, void *addr) 118 { 119 struct rte_vdev_driver **out = addr; 120 struct rte_vdev_driver *driver = NULL; 121 122 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 123 if (strncmp(driver->driver.name, name, 124 strlen(driver->driver.name)) == 0) 125 break; 126 if (driver->driver.alias && 127 strncmp(driver->driver.alias, name, 128 strlen(driver->driver.alias)) == 0) 129 break; 130 } 131 if (driver != NULL && 132 addr != NULL) 133 *out = driver; 134 return driver == NULL; 135 } 136 137 static int 138 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 139 { 140 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 141 const struct rte_vdev_driver *driver; 142 143 if (!vdev) { 144 rte_errno = EINVAL; 145 return -1; 146 } 147 148 if (!vdev->device.driver) { 149 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 150 return 1; 151 } 152 153 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 154 driver); 155 156 if (driver->dma_map) 157 return driver->dma_map(vdev, addr, iova, len); 158 159 return 0; 160 } 161 162 static int 163 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 164 { 165 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 166 const struct rte_vdev_driver *driver; 167 168 if (!vdev) { 169 rte_errno = EINVAL; 170 return -1; 171 } 172 173 if (!vdev->device.driver) { 174 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 175 return 1; 176 } 177 178 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 179 driver); 180 181 if (driver->dma_unmap) 182 return driver->dma_unmap(vdev, addr, iova, len); 183 184 return 0; 185 } 186 187 static int 188 vdev_probe_all_drivers(struct rte_vdev_device *dev) 189 { 190 const char *name; 191 struct rte_vdev_driver *driver; 192 enum rte_iova_mode iova_mode; 193 int ret; 194 195 if (rte_dev_is_probed(&dev->device)) 196 return -EEXIST; 197 198 name = rte_vdev_device_name(dev); 199 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 200 201 if (vdev_parse(name, &driver)) 202 return -1; 203 204 iova_mode = rte_eal_iova_mode(); 205 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) { 206 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing", 207 name); 208 return -1; 209 } 210 211 ret = driver->probe(dev); 212 if (ret == 0) 213 dev->device.driver = &driver->driver; 214 return ret; 215 } 216 217 /* The caller shall be responsible for thread-safe */ 218 static struct rte_vdev_device * 219 find_vdev(const char *name) 220 { 221 struct rte_vdev_device *dev; 222 223 if (!name) 224 return NULL; 225 226 TAILQ_FOREACH(dev, &vdev_device_list, next) { 227 const char *devname = rte_vdev_device_name(dev); 228 229 if (!strcmp(devname, name)) 230 return dev; 231 } 232 233 return NULL; 234 } 235 236 static struct rte_devargs * 237 alloc_devargs(const char *name, const char *args) 238 { 239 struct rte_devargs *devargs; 240 int ret; 241 242 devargs = calloc(1, sizeof(*devargs)); 243 if (!devargs) 244 return NULL; 245 246 devargs->bus = &rte_vdev_bus; 247 if (args) 248 devargs->data = strdup(args); 249 else 250 devargs->data = strdup(""); 251 devargs->args = devargs->data; 252 253 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 254 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 255 rte_devargs_reset(devargs); 256 free(devargs); 257 return NULL; 258 } 259 260 return devargs; 261 } 262 263 static int 264 insert_vdev(const char *name, const char *args, 265 struct rte_vdev_device **p_dev, 266 bool init) 267 { 268 struct rte_vdev_device *dev; 269 struct rte_devargs *devargs; 270 int ret; 271 272 if (name == NULL) 273 return -EINVAL; 274 275 devargs = alloc_devargs(name, args); 276 if (!devargs) 277 return -ENOMEM; 278 279 dev = calloc(1, sizeof(*dev)); 280 if (!dev) { 281 ret = -ENOMEM; 282 goto fail; 283 } 284 285 dev->device.bus = &rte_vdev_bus; 286 dev->device.numa_node = SOCKET_ID_ANY; 287 dev->device.name = devargs->name; 288 289 if (find_vdev(name)) { 290 /* 291 * A vdev is expected to have only one port. 292 * So there is no reason to try probing again, 293 * even with new arguments. 294 */ 295 ret = -EEXIST; 296 goto fail; 297 } 298 299 if (init) 300 rte_devargs_insert(&devargs); 301 dev->device.devargs = devargs; 302 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 303 304 if (p_dev) 305 *p_dev = dev; 306 307 return 0; 308 fail: 309 rte_devargs_reset(devargs); 310 free(devargs); 311 free(dev); 312 return ret; 313 } 314 315 int 316 rte_vdev_init(const char *name, const char *args) 317 { 318 struct rte_vdev_device *dev; 319 int ret; 320 321 rte_spinlock_recursive_lock(&vdev_device_list_lock); 322 ret = insert_vdev(name, args, &dev, true); 323 if (ret == 0) { 324 ret = vdev_probe_all_drivers(dev); 325 if (ret) { 326 if (ret > 0) 327 VDEV_LOG(ERR, "no driver found for %s", name); 328 /* If fails, remove it from vdev list */ 329 TAILQ_REMOVE(&vdev_device_list, dev, next); 330 rte_devargs_remove(dev->device.devargs); 331 free(dev); 332 } 333 } 334 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 335 return ret; 336 } 337 338 static int 339 vdev_remove_driver(struct rte_vdev_device *dev) 340 { 341 const char *name = rte_vdev_device_name(dev); 342 const struct rte_vdev_driver *driver; 343 344 if (!dev->device.driver) { 345 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 346 return 1; 347 } 348 349 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 350 driver); 351 return driver->remove(dev); 352 } 353 354 int 355 rte_vdev_uninit(const char *name) 356 { 357 struct rte_vdev_device *dev; 358 int ret; 359 360 if (name == NULL) 361 return -EINVAL; 362 363 rte_spinlock_recursive_lock(&vdev_device_list_lock); 364 365 dev = find_vdev(name); 366 if (!dev) { 367 ret = -ENOENT; 368 goto unlock; 369 } 370 371 ret = vdev_remove_driver(dev); 372 if (ret) 373 goto unlock; 374 375 TAILQ_REMOVE(&vdev_device_list, dev, next); 376 rte_devargs_remove(dev->device.devargs); 377 free(dev); 378 379 unlock: 380 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 381 return ret; 382 } 383 384 struct vdev_param { 385 #define VDEV_SCAN_REQ 1 386 #define VDEV_SCAN_ONE 2 387 #define VDEV_SCAN_REP 3 388 int type; 389 int num; 390 char name[RTE_DEV_NAME_MAX_LEN]; 391 }; 392 393 static int vdev_plug(struct rte_device *dev); 394 395 /** 396 * This function works as the action for both primary and secondary process 397 * for static vdev discovery when a secondary process is booting. 398 * 399 * step 1, secondary process sends a sync request to ask for vdev in primary; 400 * step 2, primary process receives the request, and send vdevs one by one; 401 * step 3, primary process sends back reply, which indicates how many vdevs 402 * are sent. 403 */ 404 static int 405 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 406 { 407 struct rte_vdev_device *dev; 408 struct rte_mp_msg mp_resp; 409 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 410 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 411 const char *devname; 412 int num; 413 int ret; 414 415 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 416 mp_resp.len_param = sizeof(*ou); 417 mp_resp.num_fds = 0; 418 419 switch (in->type) { 420 case VDEV_SCAN_REQ: 421 ou->type = VDEV_SCAN_ONE; 422 ou->num = 1; 423 num = 0; 424 425 rte_spinlock_recursive_lock(&vdev_device_list_lock); 426 TAILQ_FOREACH(dev, &vdev_device_list, next) { 427 devname = rte_vdev_device_name(dev); 428 if (strlen(devname) == 0) { 429 VDEV_LOG(INFO, "vdev with no name is not sent"); 430 continue; 431 } 432 VDEV_LOG(INFO, "send vdev, %s", devname); 433 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 434 if (rte_mp_sendmsg(&mp_resp) < 0) 435 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 436 devname, strerror(rte_errno)); 437 num++; 438 } 439 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 440 441 ou->type = VDEV_SCAN_REP; 442 ou->num = num; 443 if (rte_mp_reply(&mp_resp, peer) < 0) 444 VDEV_LOG(ERR, "Failed to reply a scan request"); 445 break; 446 case VDEV_SCAN_ONE: 447 VDEV_LOG(INFO, "receive vdev, %s", in->name); 448 ret = insert_vdev(in->name, NULL, NULL, false); 449 if (ret == -EEXIST) 450 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 451 else if (ret < 0) 452 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 453 break; 454 default: 455 VDEV_LOG(ERR, "vdev cannot recognize this message"); 456 } 457 458 return 0; 459 } 460 461 static int 462 vdev_scan(void) 463 { 464 struct rte_vdev_device *dev; 465 struct rte_devargs *devargs; 466 struct vdev_custom_scan *custom_scan; 467 468 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 469 rte_errno != EEXIST) { 470 /* for primary, unsupported IPC is not an error */ 471 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 472 rte_errno == ENOTSUP) 473 goto scan; 474 VDEV_LOG(ERR, "Failed to add vdev mp action"); 475 return -1; 476 } 477 478 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 479 struct rte_mp_msg mp_req, *mp_rep; 480 struct rte_mp_reply mp_reply; 481 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 482 struct vdev_param *req = (struct vdev_param *)mp_req.param; 483 struct vdev_param *resp; 484 485 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 486 mp_req.len_param = sizeof(*req); 487 mp_req.num_fds = 0; 488 req->type = VDEV_SCAN_REQ; 489 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 490 mp_reply.nb_received == 1) { 491 mp_rep = &mp_reply.msgs[0]; 492 resp = (struct vdev_param *)mp_rep->param; 493 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 494 free(mp_reply.msgs); 495 } else 496 VDEV_LOG(ERR, "Failed to request vdev from primary"); 497 498 /* Fall through to allow private vdevs in secondary process */ 499 } 500 501 scan: 502 /* call custom scan callbacks if any */ 503 rte_spinlock_lock(&vdev_custom_scan_lock); 504 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 505 if (custom_scan->callback != NULL) 506 /* 507 * the callback should update devargs list 508 * by calling rte_devargs_insert() with 509 * devargs.bus = rte_bus_find_by_name("vdev"); 510 * devargs.type = RTE_DEVTYPE_VIRTUAL; 511 * devargs.policy = RTE_DEV_ALLOWED; 512 */ 513 custom_scan->callback(custom_scan->user_arg); 514 } 515 rte_spinlock_unlock(&vdev_custom_scan_lock); 516 517 /* for virtual devices we scan the devargs_list populated via cmdline */ 518 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 519 520 dev = calloc(1, sizeof(*dev)); 521 if (!dev) 522 return -1; 523 524 rte_spinlock_recursive_lock(&vdev_device_list_lock); 525 526 if (find_vdev(devargs->name)) { 527 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 528 free(dev); 529 continue; 530 } 531 532 dev->device.bus = &rte_vdev_bus; 533 dev->device.devargs = devargs; 534 dev->device.numa_node = SOCKET_ID_ANY; 535 dev->device.name = devargs->name; 536 537 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 538 539 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 540 } 541 542 return 0; 543 } 544 545 static int 546 vdev_probe(void) 547 { 548 struct rte_vdev_device *dev; 549 int r, ret = 0; 550 551 /* call the init function for each virtual device */ 552 TAILQ_FOREACH(dev, &vdev_device_list, next) { 553 /* we don't use the vdev lock here, as it's only used in DPDK 554 * initialization; and we don't want to hold such a lock when 555 * we call each driver probe. 556 */ 557 558 r = vdev_probe_all_drivers(dev); 559 if (r != 0) { 560 if (r == -EEXIST) 561 continue; 562 VDEV_LOG(ERR, "failed to initialize %s device", 563 rte_vdev_device_name(dev)); 564 ret = -1; 565 } 566 } 567 568 return ret; 569 } 570 571 struct rte_device * 572 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 573 const void *data) 574 { 575 const struct rte_vdev_device *vstart; 576 struct rte_vdev_device *dev; 577 578 rte_spinlock_recursive_lock(&vdev_device_list_lock); 579 if (start != NULL) { 580 vstart = RTE_DEV_TO_VDEV_CONST(start); 581 dev = TAILQ_NEXT(vstart, next); 582 } else { 583 dev = TAILQ_FIRST(&vdev_device_list); 584 } 585 while (dev != NULL) { 586 if (cmp(&dev->device, data) == 0) 587 break; 588 dev = TAILQ_NEXT(dev, next); 589 } 590 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 591 592 return dev ? &dev->device : NULL; 593 } 594 595 static int 596 vdev_plug(struct rte_device *dev) 597 { 598 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 599 } 600 601 static int 602 vdev_unplug(struct rte_device *dev) 603 { 604 return rte_vdev_uninit(dev->name); 605 } 606 607 static enum rte_iova_mode 608 vdev_get_iommu_class(void) 609 { 610 const char *name; 611 struct rte_vdev_device *dev; 612 struct rte_vdev_driver *driver; 613 614 TAILQ_FOREACH(dev, &vdev_device_list, next) { 615 name = rte_vdev_device_name(dev); 616 if (vdev_parse(name, &driver)) 617 continue; 618 619 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) 620 return RTE_IOVA_VA; 621 } 622 623 return RTE_IOVA_DC; 624 } 625 626 static struct rte_bus rte_vdev_bus = { 627 .scan = vdev_scan, 628 .probe = vdev_probe, 629 .find_device = rte_vdev_find_device, 630 .plug = vdev_plug, 631 .unplug = vdev_unplug, 632 .parse = vdev_parse, 633 .dma_map = vdev_dma_map, 634 .dma_unmap = vdev_dma_unmap, 635 .get_iommu_class = vdev_get_iommu_class, 636 .dev_iterate = rte_vdev_dev_iterate, 637 }; 638 639 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 640 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE); 641