1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <rte_dev.h> 15 #include <rte_bus.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "rte_bus_vdev.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 /* Forward declare to access virtual bus name */ 31 static struct rte_bus rte_vdev_bus; 32 33 /** Double linked list of virtual device drivers. */ 34 TAILQ_HEAD(vdev_device_list, rte_vdev_device); 35 36 static struct vdev_device_list vdev_device_list = 37 TAILQ_HEAD_INITIALIZER(vdev_device_list); 38 /* The lock needs to be recursive because a vdev can manage another vdev. */ 39 static rte_spinlock_recursive_t vdev_device_list_lock = 40 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 41 42 static struct vdev_driver_list vdev_driver_list = 43 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 44 45 struct vdev_custom_scan { 46 TAILQ_ENTRY(vdev_custom_scan) next; 47 rte_vdev_scan_callback callback; 48 void *user_arg; 49 }; 50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 51 static struct vdev_custom_scans vdev_custom_scans = 52 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 54 55 /* register a driver */ 56 void 57 rte_vdev_register(struct rte_vdev_driver *driver) 58 { 59 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 60 } 61 62 /* unregister a driver */ 63 void 64 rte_vdev_unregister(struct rte_vdev_driver *driver) 65 { 66 TAILQ_REMOVE(&vdev_driver_list, driver, next); 67 } 68 69 int 70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 71 { 72 struct vdev_custom_scan *custom_scan; 73 74 rte_spinlock_lock(&vdev_custom_scan_lock); 75 76 /* check if already registered */ 77 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 78 if (custom_scan->callback == callback && 79 custom_scan->user_arg == user_arg) 80 break; 81 } 82 83 if (custom_scan == NULL) { 84 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 85 if (custom_scan != NULL) { 86 custom_scan->callback = callback; 87 custom_scan->user_arg = user_arg; 88 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 89 } 90 } 91 92 rte_spinlock_unlock(&vdev_custom_scan_lock); 93 94 return (custom_scan == NULL) ? -1 : 0; 95 } 96 97 int 98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 99 { 100 struct vdev_custom_scan *custom_scan, *tmp_scan; 101 102 rte_spinlock_lock(&vdev_custom_scan_lock); 103 TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, tmp_scan) { 104 if (custom_scan->callback != callback || 105 (custom_scan->user_arg != (void *)-1 && 106 custom_scan->user_arg != user_arg)) 107 continue; 108 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 109 free(custom_scan); 110 } 111 rte_spinlock_unlock(&vdev_custom_scan_lock); 112 113 return 0; 114 } 115 116 static int 117 vdev_parse(const char *name, void *addr) 118 { 119 struct rte_vdev_driver **out = addr; 120 struct rte_vdev_driver *driver = NULL; 121 122 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 123 if (strncmp(driver->driver.name, name, 124 strlen(driver->driver.name)) == 0) 125 break; 126 if (driver->driver.alias && 127 strncmp(driver->driver.alias, name, 128 strlen(driver->driver.alias)) == 0) 129 break; 130 } 131 if (driver != NULL && 132 addr != NULL) 133 *out = driver; 134 return driver == NULL; 135 } 136 137 static int 138 vdev_probe_all_drivers(struct rte_vdev_device *dev) 139 { 140 const char *name; 141 struct rte_vdev_driver *driver; 142 int ret; 143 144 if (rte_dev_is_probed(&dev->device)) 145 return -EEXIST; 146 147 name = rte_vdev_device_name(dev); 148 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 149 150 if (vdev_parse(name, &driver)) 151 return -1; 152 ret = driver->probe(dev); 153 if (ret == 0) 154 dev->device.driver = &driver->driver; 155 return ret; 156 } 157 158 /* The caller shall be responsible for thread-safe */ 159 static struct rte_vdev_device * 160 find_vdev(const char *name) 161 { 162 struct rte_vdev_device *dev; 163 164 if (!name) 165 return NULL; 166 167 TAILQ_FOREACH(dev, &vdev_device_list, next) { 168 const char *devname = rte_vdev_device_name(dev); 169 170 if (!strcmp(devname, name)) 171 return dev; 172 } 173 174 return NULL; 175 } 176 177 static struct rte_devargs * 178 alloc_devargs(const char *name, const char *args) 179 { 180 struct rte_devargs *devargs; 181 int ret; 182 183 devargs = calloc(1, sizeof(*devargs)); 184 if (!devargs) 185 return NULL; 186 187 devargs->bus = &rte_vdev_bus; 188 if (args) 189 devargs->args = strdup(args); 190 else 191 devargs->args = strdup(""); 192 193 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 194 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 195 free(devargs->args); 196 free(devargs); 197 return NULL; 198 } 199 200 return devargs; 201 } 202 203 static int 204 insert_vdev(const char *name, const char *args, 205 struct rte_vdev_device **p_dev, 206 bool init) 207 { 208 struct rte_vdev_device *dev; 209 struct rte_devargs *devargs; 210 int ret; 211 212 if (name == NULL) 213 return -EINVAL; 214 215 devargs = alloc_devargs(name, args); 216 if (!devargs) 217 return -ENOMEM; 218 219 dev = calloc(1, sizeof(*dev)); 220 if (!dev) { 221 ret = -ENOMEM; 222 goto fail; 223 } 224 225 dev->device.bus = &rte_vdev_bus; 226 dev->device.numa_node = SOCKET_ID_ANY; 227 dev->device.name = devargs->name; 228 229 if (find_vdev(name)) { 230 /* 231 * A vdev is expected to have only one port. 232 * So there is no reason to try probing again, 233 * even with new arguments. 234 */ 235 ret = -EEXIST; 236 goto fail; 237 } 238 239 if (init) 240 rte_devargs_insert(&devargs); 241 dev->device.devargs = devargs; 242 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 243 244 if (p_dev) 245 *p_dev = dev; 246 247 return 0; 248 fail: 249 free(devargs->args); 250 free(devargs); 251 free(dev); 252 return ret; 253 } 254 255 int 256 rte_vdev_init(const char *name, const char *args) 257 { 258 struct rte_vdev_device *dev; 259 int ret; 260 261 rte_spinlock_recursive_lock(&vdev_device_list_lock); 262 ret = insert_vdev(name, args, &dev, true); 263 if (ret == 0) { 264 ret = vdev_probe_all_drivers(dev); 265 if (ret) { 266 if (ret > 0) 267 VDEV_LOG(ERR, "no driver found for %s", name); 268 /* If fails, remove it from vdev list */ 269 TAILQ_REMOVE(&vdev_device_list, dev, next); 270 rte_devargs_remove(dev->device.devargs); 271 free(dev); 272 } 273 } 274 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 275 return ret; 276 } 277 278 static int 279 vdev_remove_driver(struct rte_vdev_device *dev) 280 { 281 const char *name = rte_vdev_device_name(dev); 282 const struct rte_vdev_driver *driver; 283 284 if (!dev->device.driver) { 285 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 286 return 1; 287 } 288 289 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 290 driver); 291 return driver->remove(dev); 292 } 293 294 int 295 rte_vdev_uninit(const char *name) 296 { 297 struct rte_vdev_device *dev; 298 int ret; 299 300 if (name == NULL) 301 return -EINVAL; 302 303 rte_spinlock_recursive_lock(&vdev_device_list_lock); 304 305 dev = find_vdev(name); 306 if (!dev) { 307 ret = -ENOENT; 308 goto unlock; 309 } 310 311 ret = vdev_remove_driver(dev); 312 if (ret) 313 goto unlock; 314 315 TAILQ_REMOVE(&vdev_device_list, dev, next); 316 rte_devargs_remove(dev->device.devargs); 317 free(dev); 318 319 unlock: 320 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 321 return ret; 322 } 323 324 struct vdev_param { 325 #define VDEV_SCAN_REQ 1 326 #define VDEV_SCAN_ONE 2 327 #define VDEV_SCAN_REP 3 328 int type; 329 int num; 330 char name[RTE_DEV_NAME_MAX_LEN]; 331 }; 332 333 static int vdev_plug(struct rte_device *dev); 334 335 /** 336 * This function works as the action for both primary and secondary process 337 * for static vdev discovery when a secondary process is booting. 338 * 339 * step 1, secondary process sends a sync request to ask for vdev in primary; 340 * step 2, primary process receives the request, and send vdevs one by one; 341 * step 3, primary process sends back reply, which indicates how many vdevs 342 * are sent. 343 */ 344 static int 345 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 346 { 347 struct rte_vdev_device *dev; 348 struct rte_mp_msg mp_resp; 349 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 350 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 351 const char *devname; 352 int num; 353 int ret; 354 355 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 356 mp_resp.len_param = sizeof(*ou); 357 mp_resp.num_fds = 0; 358 359 switch (in->type) { 360 case VDEV_SCAN_REQ: 361 ou->type = VDEV_SCAN_ONE; 362 ou->num = 1; 363 num = 0; 364 365 rte_spinlock_recursive_lock(&vdev_device_list_lock); 366 TAILQ_FOREACH(dev, &vdev_device_list, next) { 367 devname = rte_vdev_device_name(dev); 368 if (strlen(devname) == 0) { 369 VDEV_LOG(INFO, "vdev with no name is not sent"); 370 continue; 371 } 372 VDEV_LOG(INFO, "send vdev, %s", devname); 373 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 374 if (rte_mp_sendmsg(&mp_resp) < 0) 375 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 376 devname, strerror(rte_errno)); 377 num++; 378 } 379 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 380 381 ou->type = VDEV_SCAN_REP; 382 ou->num = num; 383 if (rte_mp_reply(&mp_resp, peer) < 0) 384 VDEV_LOG(ERR, "Failed to reply a scan request"); 385 break; 386 case VDEV_SCAN_ONE: 387 VDEV_LOG(INFO, "receive vdev, %s", in->name); 388 ret = insert_vdev(in->name, NULL, NULL, false); 389 if (ret == -EEXIST) 390 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 391 else if (ret < 0) 392 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 393 break; 394 default: 395 VDEV_LOG(ERR, "vdev cannot recognize this message"); 396 } 397 398 return 0; 399 } 400 401 static int 402 vdev_scan(void) 403 { 404 struct rte_vdev_device *dev; 405 struct rte_devargs *devargs; 406 struct vdev_custom_scan *custom_scan; 407 408 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 409 rte_errno != EEXIST) { 410 /* for primary, unsupported IPC is not an error */ 411 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 412 rte_errno == ENOTSUP) 413 goto scan; 414 VDEV_LOG(ERR, "Failed to add vdev mp action"); 415 return -1; 416 } 417 418 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 419 struct rte_mp_msg mp_req, *mp_rep; 420 struct rte_mp_reply mp_reply; 421 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 422 struct vdev_param *req = (struct vdev_param *)mp_req.param; 423 struct vdev_param *resp; 424 425 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 426 mp_req.len_param = sizeof(*req); 427 mp_req.num_fds = 0; 428 req->type = VDEV_SCAN_REQ; 429 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 430 mp_reply.nb_received == 1) { 431 mp_rep = &mp_reply.msgs[0]; 432 resp = (struct vdev_param *)mp_rep->param; 433 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 434 free(mp_reply.msgs); 435 } else 436 VDEV_LOG(ERR, "Failed to request vdev from primary"); 437 438 /* Fall through to allow private vdevs in secondary process */ 439 } 440 441 scan: 442 /* call custom scan callbacks if any */ 443 rte_spinlock_lock(&vdev_custom_scan_lock); 444 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 445 if (custom_scan->callback != NULL) 446 /* 447 * the callback should update devargs list 448 * by calling rte_devargs_insert() with 449 * devargs.bus = rte_bus_find_by_name("vdev"); 450 * devargs.type = RTE_DEVTYPE_VIRTUAL; 451 * devargs.policy = RTE_DEV_WHITELISTED; 452 */ 453 custom_scan->callback(custom_scan->user_arg); 454 } 455 rte_spinlock_unlock(&vdev_custom_scan_lock); 456 457 /* for virtual devices we scan the devargs_list populated via cmdline */ 458 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 459 460 dev = calloc(1, sizeof(*dev)); 461 if (!dev) 462 return -1; 463 464 rte_spinlock_recursive_lock(&vdev_device_list_lock); 465 466 if (find_vdev(devargs->name)) { 467 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 468 free(dev); 469 continue; 470 } 471 472 dev->device.bus = &rte_vdev_bus; 473 dev->device.devargs = devargs; 474 dev->device.numa_node = SOCKET_ID_ANY; 475 dev->device.name = devargs->name; 476 477 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 478 479 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 480 } 481 482 return 0; 483 } 484 485 static int 486 vdev_probe(void) 487 { 488 struct rte_vdev_device *dev; 489 int r, ret = 0; 490 491 /* call the init function for each virtual device */ 492 TAILQ_FOREACH(dev, &vdev_device_list, next) { 493 /* we don't use the vdev lock here, as it's only used in DPDK 494 * initialization; and we don't want to hold such a lock when 495 * we call each driver probe. 496 */ 497 498 r = vdev_probe_all_drivers(dev); 499 if (r != 0) { 500 if (r == -EEXIST) 501 continue; 502 VDEV_LOG(ERR, "failed to initialize %s device", 503 rte_vdev_device_name(dev)); 504 ret = -1; 505 } 506 } 507 508 return ret; 509 } 510 511 struct rte_device * 512 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 513 const void *data) 514 { 515 const struct rte_vdev_device *vstart; 516 struct rte_vdev_device *dev; 517 518 rte_spinlock_recursive_lock(&vdev_device_list_lock); 519 if (start != NULL) { 520 vstart = RTE_DEV_TO_VDEV_CONST(start); 521 dev = TAILQ_NEXT(vstart, next); 522 } else { 523 dev = TAILQ_FIRST(&vdev_device_list); 524 } 525 while (dev != NULL) { 526 if (cmp(&dev->device, data) == 0) 527 break; 528 dev = TAILQ_NEXT(dev, next); 529 } 530 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 531 532 return dev ? &dev->device : NULL; 533 } 534 535 static int 536 vdev_plug(struct rte_device *dev) 537 { 538 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 539 } 540 541 static int 542 vdev_unplug(struct rte_device *dev) 543 { 544 return rte_vdev_uninit(dev->name); 545 } 546 547 static struct rte_bus rte_vdev_bus = { 548 .scan = vdev_scan, 549 .probe = vdev_probe, 550 .find_device = rte_vdev_find_device, 551 .plug = vdev_plug, 552 .unplug = vdev_unplug, 553 .parse = vdev_parse, 554 .dev_iterate = rte_vdev_dev_iterate, 555 }; 556 557 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 558 RTE_LOG_REGISTER(vdev_logtype_bus, bus.vdev, NOTICE); 559