1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <rte_dev.h> 15 #include <rte_bus.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "rte_bus_vdev.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 /* Forward declare to access virtual bus name */ 31 static struct rte_bus rte_vdev_bus; 32 33 /** Double linked list of virtual device drivers. */ 34 TAILQ_HEAD(vdev_device_list, rte_vdev_device); 35 36 static struct vdev_device_list vdev_device_list = 37 TAILQ_HEAD_INITIALIZER(vdev_device_list); 38 /* The lock needs to be recursive because a vdev can manage another vdev. */ 39 static rte_spinlock_recursive_t vdev_device_list_lock = 40 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 41 42 static struct vdev_driver_list vdev_driver_list = 43 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 44 45 struct vdev_custom_scan { 46 TAILQ_ENTRY(vdev_custom_scan) next; 47 rte_vdev_scan_callback callback; 48 void *user_arg; 49 }; 50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 51 static struct vdev_custom_scans vdev_custom_scans = 52 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 54 55 /* register a driver */ 56 void 57 rte_vdev_register(struct rte_vdev_driver *driver) 58 { 59 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 60 } 61 62 /* unregister a driver */ 63 void 64 rte_vdev_unregister(struct rte_vdev_driver *driver) 65 { 66 TAILQ_REMOVE(&vdev_driver_list, driver, next); 67 } 68 69 int 70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 71 { 72 struct vdev_custom_scan *custom_scan; 73 74 rte_spinlock_lock(&vdev_custom_scan_lock); 75 76 /* check if already registered */ 77 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 78 if (custom_scan->callback == callback && 79 custom_scan->user_arg == user_arg) 80 break; 81 } 82 83 if (custom_scan == NULL) { 84 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 85 if (custom_scan != NULL) { 86 custom_scan->callback = callback; 87 custom_scan->user_arg = user_arg; 88 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 89 } 90 } 91 92 rte_spinlock_unlock(&vdev_custom_scan_lock); 93 94 return (custom_scan == NULL) ? -1 : 0; 95 } 96 97 int 98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 99 { 100 struct vdev_custom_scan *custom_scan, *tmp_scan; 101 102 rte_spinlock_lock(&vdev_custom_scan_lock); 103 RTE_TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, 104 tmp_scan) { 105 if (custom_scan->callback != callback || 106 (custom_scan->user_arg != (void *)-1 && 107 custom_scan->user_arg != user_arg)) 108 continue; 109 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 110 free(custom_scan); 111 } 112 rte_spinlock_unlock(&vdev_custom_scan_lock); 113 114 return 0; 115 } 116 117 static int 118 vdev_parse(const char *name, void *addr) 119 { 120 struct rte_vdev_driver **out = addr; 121 struct rte_vdev_driver *driver = NULL; 122 123 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 124 if (strncmp(driver->driver.name, name, 125 strlen(driver->driver.name)) == 0) 126 break; 127 if (driver->driver.alias && 128 strncmp(driver->driver.alias, name, 129 strlen(driver->driver.alias)) == 0) 130 break; 131 } 132 if (driver != NULL && 133 addr != NULL) 134 *out = driver; 135 return driver == NULL; 136 } 137 138 static int 139 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 140 { 141 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 142 const struct rte_vdev_driver *driver; 143 144 if (!vdev) { 145 rte_errno = EINVAL; 146 return -1; 147 } 148 149 if (!vdev->device.driver) { 150 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 151 return 1; 152 } 153 154 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 155 driver); 156 157 if (driver->dma_map) 158 return driver->dma_map(vdev, addr, iova, len); 159 160 return 0; 161 } 162 163 static int 164 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 165 { 166 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev); 167 const struct rte_vdev_driver *driver; 168 169 if (!vdev) { 170 rte_errno = EINVAL; 171 return -1; 172 } 173 174 if (!vdev->device.driver) { 175 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name); 176 return 1; 177 } 178 179 driver = container_of(vdev->device.driver, const struct rte_vdev_driver, 180 driver); 181 182 if (driver->dma_unmap) 183 return driver->dma_unmap(vdev, addr, iova, len); 184 185 return 0; 186 } 187 188 static int 189 vdev_probe_all_drivers(struct rte_vdev_device *dev) 190 { 191 const char *name; 192 struct rte_vdev_driver *driver; 193 enum rte_iova_mode iova_mode; 194 int ret; 195 196 if (rte_dev_is_probed(&dev->device)) 197 return -EEXIST; 198 199 name = rte_vdev_device_name(dev); 200 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 201 202 if (vdev_parse(name, &driver)) 203 return -1; 204 205 iova_mode = rte_eal_iova_mode(); 206 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) { 207 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing", 208 name); 209 return -1; 210 } 211 212 ret = driver->probe(dev); 213 if (ret == 0) 214 dev->device.driver = &driver->driver; 215 return ret; 216 } 217 218 /* The caller shall be responsible for thread-safe */ 219 static struct rte_vdev_device * 220 find_vdev(const char *name) 221 { 222 struct rte_vdev_device *dev; 223 224 if (!name) 225 return NULL; 226 227 TAILQ_FOREACH(dev, &vdev_device_list, next) { 228 const char *devname = rte_vdev_device_name(dev); 229 230 if (!strcmp(devname, name)) 231 return dev; 232 } 233 234 return NULL; 235 } 236 237 static struct rte_devargs * 238 alloc_devargs(const char *name, const char *args) 239 { 240 struct rte_devargs *devargs; 241 int ret; 242 243 devargs = calloc(1, sizeof(*devargs)); 244 if (!devargs) 245 return NULL; 246 247 devargs->bus = &rte_vdev_bus; 248 if (args) 249 devargs->data = strdup(args); 250 else 251 devargs->data = strdup(""); 252 devargs->args = devargs->data; 253 254 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 255 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 256 rte_devargs_reset(devargs); 257 free(devargs); 258 return NULL; 259 } 260 261 return devargs; 262 } 263 264 static int 265 insert_vdev(const char *name, const char *args, 266 struct rte_vdev_device **p_dev, 267 bool init) 268 { 269 struct rte_vdev_device *dev; 270 struct rte_devargs *devargs; 271 int ret; 272 273 if (name == NULL) 274 return -EINVAL; 275 276 devargs = alloc_devargs(name, args); 277 if (!devargs) 278 return -ENOMEM; 279 280 dev = calloc(1, sizeof(*dev)); 281 if (!dev) { 282 ret = -ENOMEM; 283 goto fail; 284 } 285 286 dev->device.bus = &rte_vdev_bus; 287 dev->device.numa_node = SOCKET_ID_ANY; 288 dev->device.name = devargs->name; 289 290 if (find_vdev(name)) { 291 /* 292 * A vdev is expected to have only one port. 293 * So there is no reason to try probing again, 294 * even with new arguments. 295 */ 296 ret = -EEXIST; 297 goto fail; 298 } 299 300 if (init) 301 rte_devargs_insert(&devargs); 302 dev->device.devargs = devargs; 303 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 304 305 if (p_dev) 306 *p_dev = dev; 307 308 return 0; 309 fail: 310 rte_devargs_reset(devargs); 311 free(devargs); 312 free(dev); 313 return ret; 314 } 315 316 int 317 rte_vdev_init(const char *name, const char *args) 318 { 319 struct rte_vdev_device *dev; 320 int ret; 321 322 rte_spinlock_recursive_lock(&vdev_device_list_lock); 323 ret = insert_vdev(name, args, &dev, true); 324 if (ret == 0) { 325 ret = vdev_probe_all_drivers(dev); 326 if (ret) { 327 if (ret > 0) 328 VDEV_LOG(ERR, "no driver found for %s", name); 329 /* If fails, remove it from vdev list */ 330 TAILQ_REMOVE(&vdev_device_list, dev, next); 331 rte_devargs_remove(dev->device.devargs); 332 free(dev); 333 } 334 } 335 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 336 return ret; 337 } 338 339 static int 340 vdev_remove_driver(struct rte_vdev_device *dev) 341 { 342 const char *name = rte_vdev_device_name(dev); 343 const struct rte_vdev_driver *driver; 344 345 if (!dev->device.driver) { 346 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 347 return 1; 348 } 349 350 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 351 driver); 352 return driver->remove(dev); 353 } 354 355 int 356 rte_vdev_uninit(const char *name) 357 { 358 struct rte_vdev_device *dev; 359 int ret; 360 361 if (name == NULL) 362 return -EINVAL; 363 364 rte_spinlock_recursive_lock(&vdev_device_list_lock); 365 366 dev = find_vdev(name); 367 if (!dev) { 368 ret = -ENOENT; 369 goto unlock; 370 } 371 372 ret = vdev_remove_driver(dev); 373 if (ret) 374 goto unlock; 375 376 TAILQ_REMOVE(&vdev_device_list, dev, next); 377 rte_devargs_remove(dev->device.devargs); 378 free(dev); 379 380 unlock: 381 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 382 return ret; 383 } 384 385 struct vdev_param { 386 #define VDEV_SCAN_REQ 1 387 #define VDEV_SCAN_ONE 2 388 #define VDEV_SCAN_REP 3 389 int type; 390 int num; 391 char name[RTE_DEV_NAME_MAX_LEN]; 392 }; 393 394 static int vdev_plug(struct rte_device *dev); 395 396 /** 397 * This function works as the action for both primary and secondary process 398 * for static vdev discovery when a secondary process is booting. 399 * 400 * step 1, secondary process sends a sync request to ask for vdev in primary; 401 * step 2, primary process receives the request, and send vdevs one by one; 402 * step 3, primary process sends back reply, which indicates how many vdevs 403 * are sent. 404 */ 405 static int 406 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 407 { 408 struct rte_vdev_device *dev; 409 struct rte_mp_msg mp_resp; 410 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 411 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 412 const char *devname; 413 int num; 414 int ret; 415 416 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 417 mp_resp.len_param = sizeof(*ou); 418 mp_resp.num_fds = 0; 419 420 switch (in->type) { 421 case VDEV_SCAN_REQ: 422 ou->type = VDEV_SCAN_ONE; 423 ou->num = 1; 424 num = 0; 425 426 rte_spinlock_recursive_lock(&vdev_device_list_lock); 427 TAILQ_FOREACH(dev, &vdev_device_list, next) { 428 devname = rte_vdev_device_name(dev); 429 if (strlen(devname) == 0) { 430 VDEV_LOG(INFO, "vdev with no name is not sent"); 431 continue; 432 } 433 VDEV_LOG(INFO, "send vdev, %s", devname); 434 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 435 if (rte_mp_sendmsg(&mp_resp) < 0) 436 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 437 devname, strerror(rte_errno)); 438 num++; 439 } 440 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 441 442 ou->type = VDEV_SCAN_REP; 443 ou->num = num; 444 if (rte_mp_reply(&mp_resp, peer) < 0) 445 VDEV_LOG(ERR, "Failed to reply a scan request"); 446 break; 447 case VDEV_SCAN_ONE: 448 VDEV_LOG(INFO, "receive vdev, %s", in->name); 449 ret = insert_vdev(in->name, NULL, NULL, false); 450 if (ret == -EEXIST) 451 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 452 else if (ret < 0) 453 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 454 break; 455 default: 456 VDEV_LOG(ERR, "vdev cannot recognize this message"); 457 } 458 459 return 0; 460 } 461 462 static int 463 vdev_scan(void) 464 { 465 struct rte_vdev_device *dev; 466 struct rte_devargs *devargs; 467 struct vdev_custom_scan *custom_scan; 468 469 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 470 rte_errno != EEXIST) { 471 /* for primary, unsupported IPC is not an error */ 472 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 473 rte_errno == ENOTSUP) 474 goto scan; 475 VDEV_LOG(ERR, "Failed to add vdev mp action"); 476 return -1; 477 } 478 479 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 480 struct rte_mp_msg mp_req, *mp_rep; 481 struct rte_mp_reply mp_reply; 482 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 483 struct vdev_param *req = (struct vdev_param *)mp_req.param; 484 struct vdev_param *resp; 485 486 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 487 mp_req.len_param = sizeof(*req); 488 mp_req.num_fds = 0; 489 req->type = VDEV_SCAN_REQ; 490 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 491 mp_reply.nb_received == 1) { 492 mp_rep = &mp_reply.msgs[0]; 493 resp = (struct vdev_param *)mp_rep->param; 494 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 495 free(mp_reply.msgs); 496 } else 497 VDEV_LOG(ERR, "Failed to request vdev from primary"); 498 499 /* Fall through to allow private vdevs in secondary process */ 500 } 501 502 scan: 503 /* call custom scan callbacks if any */ 504 rte_spinlock_lock(&vdev_custom_scan_lock); 505 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 506 if (custom_scan->callback != NULL) 507 /* 508 * the callback should update devargs list 509 * by calling rte_devargs_insert() with 510 * devargs.bus = rte_bus_find_by_name("vdev"); 511 * devargs.type = RTE_DEVTYPE_VIRTUAL; 512 * devargs.policy = RTE_DEV_ALLOWED; 513 */ 514 custom_scan->callback(custom_scan->user_arg); 515 } 516 rte_spinlock_unlock(&vdev_custom_scan_lock); 517 518 /* for virtual devices we scan the devargs_list populated via cmdline */ 519 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 520 521 dev = calloc(1, sizeof(*dev)); 522 if (!dev) 523 return -1; 524 525 rte_spinlock_recursive_lock(&vdev_device_list_lock); 526 527 if (find_vdev(devargs->name)) { 528 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 529 free(dev); 530 continue; 531 } 532 533 dev->device.bus = &rte_vdev_bus; 534 dev->device.devargs = devargs; 535 dev->device.numa_node = SOCKET_ID_ANY; 536 dev->device.name = devargs->name; 537 538 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 539 540 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 541 } 542 543 return 0; 544 } 545 546 static int 547 vdev_probe(void) 548 { 549 struct rte_vdev_device *dev; 550 int r, ret = 0; 551 552 /* call the init function for each virtual device */ 553 TAILQ_FOREACH(dev, &vdev_device_list, next) { 554 /* we don't use the vdev lock here, as it's only used in DPDK 555 * initialization; and we don't want to hold such a lock when 556 * we call each driver probe. 557 */ 558 559 r = vdev_probe_all_drivers(dev); 560 if (r != 0) { 561 if (r == -EEXIST) 562 continue; 563 VDEV_LOG(ERR, "failed to initialize %s device", 564 rte_vdev_device_name(dev)); 565 ret = -1; 566 } 567 } 568 569 return ret; 570 } 571 572 struct rte_device * 573 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 574 const void *data) 575 { 576 const struct rte_vdev_device *vstart; 577 struct rte_vdev_device *dev; 578 579 rte_spinlock_recursive_lock(&vdev_device_list_lock); 580 if (start != NULL) { 581 vstart = RTE_DEV_TO_VDEV_CONST(start); 582 dev = TAILQ_NEXT(vstart, next); 583 } else { 584 dev = TAILQ_FIRST(&vdev_device_list); 585 } 586 while (dev != NULL) { 587 if (cmp(&dev->device, data) == 0) 588 break; 589 dev = TAILQ_NEXT(dev, next); 590 } 591 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 592 593 return dev ? &dev->device : NULL; 594 } 595 596 static int 597 vdev_plug(struct rte_device *dev) 598 { 599 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 600 } 601 602 static int 603 vdev_unplug(struct rte_device *dev) 604 { 605 return rte_vdev_uninit(dev->name); 606 } 607 608 static enum rte_iova_mode 609 vdev_get_iommu_class(void) 610 { 611 const char *name; 612 struct rte_vdev_device *dev; 613 struct rte_vdev_driver *driver; 614 615 TAILQ_FOREACH(dev, &vdev_device_list, next) { 616 name = rte_vdev_device_name(dev); 617 if (vdev_parse(name, &driver)) 618 continue; 619 620 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) 621 return RTE_IOVA_VA; 622 } 623 624 return RTE_IOVA_DC; 625 } 626 627 static struct rte_bus rte_vdev_bus = { 628 .scan = vdev_scan, 629 .probe = vdev_probe, 630 .find_device = rte_vdev_find_device, 631 .plug = vdev_plug, 632 .unplug = vdev_unplug, 633 .parse = vdev_parse, 634 .dma_map = vdev_dma_map, 635 .dma_unmap = vdev_dma_unmap, 636 .get_iommu_class = vdev_get_iommu_class, 637 .dev_iterate = rte_vdev_dev_iterate, 638 }; 639 640 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 641 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE); 642