1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 RehiveTech. All rights reserved. 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <sys/queue.h> 12 13 #include <rte_eal.h> 14 #include <rte_dev.h> 15 #include <rte_bus.h> 16 #include <rte_common.h> 17 #include <rte_devargs.h> 18 #include <rte_memory.h> 19 #include <rte_tailq.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 24 #include "rte_bus_vdev.h" 25 #include "vdev_logs.h" 26 #include "vdev_private.h" 27 28 #define VDEV_MP_KEY "bus_vdev_mp" 29 30 int vdev_logtype_bus; 31 32 /* Forward declare to access virtual bus name */ 33 static struct rte_bus rte_vdev_bus; 34 35 /** Double linked list of virtual device drivers. */ 36 TAILQ_HEAD(vdev_device_list, rte_vdev_device); 37 38 static struct vdev_device_list vdev_device_list = 39 TAILQ_HEAD_INITIALIZER(vdev_device_list); 40 /* The lock needs to be recursive because a vdev can manage another vdev. */ 41 static rte_spinlock_recursive_t vdev_device_list_lock = 42 RTE_SPINLOCK_RECURSIVE_INITIALIZER; 43 44 static struct vdev_driver_list vdev_driver_list = 45 TAILQ_HEAD_INITIALIZER(vdev_driver_list); 46 47 struct vdev_custom_scan { 48 TAILQ_ENTRY(vdev_custom_scan) next; 49 rte_vdev_scan_callback callback; 50 void *user_arg; 51 }; 52 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan); 53 static struct vdev_custom_scans vdev_custom_scans = 54 TAILQ_HEAD_INITIALIZER(vdev_custom_scans); 55 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER; 56 57 /* register a driver */ 58 void 59 rte_vdev_register(struct rte_vdev_driver *driver) 60 { 61 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); 62 } 63 64 /* unregister a driver */ 65 void 66 rte_vdev_unregister(struct rte_vdev_driver *driver) 67 { 68 TAILQ_REMOVE(&vdev_driver_list, driver, next); 69 } 70 71 int 72 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 73 { 74 struct vdev_custom_scan *custom_scan; 75 76 rte_spinlock_lock(&vdev_custom_scan_lock); 77 78 /* check if already registered */ 79 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 80 if (custom_scan->callback == callback && 81 custom_scan->user_arg == user_arg) 82 break; 83 } 84 85 if (custom_scan == NULL) { 86 custom_scan = malloc(sizeof(struct vdev_custom_scan)); 87 if (custom_scan != NULL) { 88 custom_scan->callback = callback; 89 custom_scan->user_arg = user_arg; 90 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next); 91 } 92 } 93 94 rte_spinlock_unlock(&vdev_custom_scan_lock); 95 96 return (custom_scan == NULL) ? -1 : 0; 97 } 98 99 int 100 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg) 101 { 102 struct vdev_custom_scan *custom_scan, *tmp_scan; 103 104 rte_spinlock_lock(&vdev_custom_scan_lock); 105 TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, tmp_scan) { 106 if (custom_scan->callback != callback || 107 (custom_scan->user_arg != (void *)-1 && 108 custom_scan->user_arg != user_arg)) 109 continue; 110 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next); 111 free(custom_scan); 112 } 113 rte_spinlock_unlock(&vdev_custom_scan_lock); 114 115 return 0; 116 } 117 118 static int 119 vdev_parse(const char *name, void *addr) 120 { 121 struct rte_vdev_driver **out = addr; 122 struct rte_vdev_driver *driver = NULL; 123 124 TAILQ_FOREACH(driver, &vdev_driver_list, next) { 125 if (strncmp(driver->driver.name, name, 126 strlen(driver->driver.name)) == 0) 127 break; 128 if (driver->driver.alias && 129 strncmp(driver->driver.alias, name, 130 strlen(driver->driver.alias)) == 0) 131 break; 132 } 133 if (driver != NULL && 134 addr != NULL) 135 *out = driver; 136 return driver == NULL; 137 } 138 139 static int 140 vdev_probe_all_drivers(struct rte_vdev_device *dev) 141 { 142 const char *name; 143 struct rte_vdev_driver *driver; 144 int ret; 145 146 if (rte_dev_is_probed(&dev->device)) 147 return -EEXIST; 148 149 name = rte_vdev_device_name(dev); 150 VDEV_LOG(DEBUG, "Search driver to probe device %s", name); 151 152 if (vdev_parse(name, &driver)) 153 return -1; 154 ret = driver->probe(dev); 155 if (ret == 0) 156 dev->device.driver = &driver->driver; 157 return ret; 158 } 159 160 /* The caller shall be responsible for thread-safe */ 161 static struct rte_vdev_device * 162 find_vdev(const char *name) 163 { 164 struct rte_vdev_device *dev; 165 166 if (!name) 167 return NULL; 168 169 TAILQ_FOREACH(dev, &vdev_device_list, next) { 170 const char *devname = rte_vdev_device_name(dev); 171 172 if (!strcmp(devname, name)) 173 return dev; 174 } 175 176 return NULL; 177 } 178 179 static struct rte_devargs * 180 alloc_devargs(const char *name, const char *args) 181 { 182 struct rte_devargs *devargs; 183 int ret; 184 185 devargs = calloc(1, sizeof(*devargs)); 186 if (!devargs) 187 return NULL; 188 189 devargs->bus = &rte_vdev_bus; 190 if (args) 191 devargs->args = strdup(args); 192 else 193 devargs->args = strdup(""); 194 195 ret = strlcpy(devargs->name, name, sizeof(devargs->name)); 196 if (ret < 0 || ret >= (int)sizeof(devargs->name)) { 197 free(devargs->args); 198 free(devargs); 199 return NULL; 200 } 201 202 return devargs; 203 } 204 205 static int 206 insert_vdev(const char *name, const char *args, 207 struct rte_vdev_device **p_dev, 208 bool init) 209 { 210 struct rte_vdev_device *dev; 211 struct rte_devargs *devargs; 212 int ret; 213 214 if (name == NULL) 215 return -EINVAL; 216 217 devargs = alloc_devargs(name, args); 218 if (!devargs) 219 return -ENOMEM; 220 221 dev = calloc(1, sizeof(*dev)); 222 if (!dev) { 223 ret = -ENOMEM; 224 goto fail; 225 } 226 227 dev->device.bus = &rte_vdev_bus; 228 dev->device.numa_node = SOCKET_ID_ANY; 229 dev->device.name = devargs->name; 230 231 if (find_vdev(name)) { 232 /* 233 * A vdev is expected to have only one port. 234 * So there is no reason to try probing again, 235 * even with new arguments. 236 */ 237 ret = -EEXIST; 238 goto fail; 239 } 240 241 if (init) 242 rte_devargs_insert(&devargs); 243 dev->device.devargs = devargs; 244 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 245 246 if (p_dev) 247 *p_dev = dev; 248 249 return 0; 250 fail: 251 free(devargs->args); 252 free(devargs); 253 free(dev); 254 return ret; 255 } 256 257 int 258 rte_vdev_init(const char *name, const char *args) 259 { 260 struct rte_vdev_device *dev; 261 int ret; 262 263 rte_spinlock_recursive_lock(&vdev_device_list_lock); 264 ret = insert_vdev(name, args, &dev, true); 265 if (ret == 0) { 266 ret = vdev_probe_all_drivers(dev); 267 if (ret) { 268 if (ret > 0) 269 VDEV_LOG(ERR, "no driver found for %s", name); 270 /* If fails, remove it from vdev list */ 271 TAILQ_REMOVE(&vdev_device_list, dev, next); 272 rte_devargs_remove(dev->device.devargs); 273 free(dev); 274 } 275 } 276 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 277 return ret; 278 } 279 280 static int 281 vdev_remove_driver(struct rte_vdev_device *dev) 282 { 283 const char *name = rte_vdev_device_name(dev); 284 const struct rte_vdev_driver *driver; 285 286 if (!dev->device.driver) { 287 VDEV_LOG(DEBUG, "no driver attach to device %s", name); 288 return 1; 289 } 290 291 driver = container_of(dev->device.driver, const struct rte_vdev_driver, 292 driver); 293 return driver->remove(dev); 294 } 295 296 int 297 rte_vdev_uninit(const char *name) 298 { 299 struct rte_vdev_device *dev; 300 int ret; 301 302 if (name == NULL) 303 return -EINVAL; 304 305 rte_spinlock_recursive_lock(&vdev_device_list_lock); 306 307 dev = find_vdev(name); 308 if (!dev) { 309 ret = -ENOENT; 310 goto unlock; 311 } 312 313 ret = vdev_remove_driver(dev); 314 if (ret) 315 goto unlock; 316 317 TAILQ_REMOVE(&vdev_device_list, dev, next); 318 rte_devargs_remove(dev->device.devargs); 319 free(dev); 320 321 unlock: 322 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 323 return ret; 324 } 325 326 struct vdev_param { 327 #define VDEV_SCAN_REQ 1 328 #define VDEV_SCAN_ONE 2 329 #define VDEV_SCAN_REP 3 330 int type; 331 int num; 332 char name[RTE_DEV_NAME_MAX_LEN]; 333 }; 334 335 static int vdev_plug(struct rte_device *dev); 336 337 /** 338 * This function works as the action for both primary and secondary process 339 * for static vdev discovery when a secondary process is booting. 340 * 341 * step 1, secondary process sends a sync request to ask for vdev in primary; 342 * step 2, primary process receives the request, and send vdevs one by one; 343 * step 3, primary process sends back reply, which indicates how many vdevs 344 * are sent. 345 */ 346 static int 347 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) 348 { 349 struct rte_vdev_device *dev; 350 struct rte_mp_msg mp_resp; 351 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; 352 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; 353 const char *devname; 354 int num; 355 int ret; 356 357 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); 358 mp_resp.len_param = sizeof(*ou); 359 mp_resp.num_fds = 0; 360 361 switch (in->type) { 362 case VDEV_SCAN_REQ: 363 ou->type = VDEV_SCAN_ONE; 364 ou->num = 1; 365 num = 0; 366 367 rte_spinlock_recursive_lock(&vdev_device_list_lock); 368 TAILQ_FOREACH(dev, &vdev_device_list, next) { 369 devname = rte_vdev_device_name(dev); 370 if (strlen(devname) == 0) { 371 VDEV_LOG(INFO, "vdev with no name is not sent"); 372 continue; 373 } 374 VDEV_LOG(INFO, "send vdev, %s", devname); 375 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); 376 if (rte_mp_sendmsg(&mp_resp) < 0) 377 VDEV_LOG(ERR, "send vdev, %s, failed, %s", 378 devname, strerror(rte_errno)); 379 num++; 380 } 381 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 382 383 ou->type = VDEV_SCAN_REP; 384 ou->num = num; 385 if (rte_mp_reply(&mp_resp, peer) < 0) 386 VDEV_LOG(ERR, "Failed to reply a scan request"); 387 break; 388 case VDEV_SCAN_ONE: 389 VDEV_LOG(INFO, "receive vdev, %s", in->name); 390 ret = insert_vdev(in->name, NULL, NULL, false); 391 if (ret == -EEXIST) 392 VDEV_LOG(DEBUG, "device already exist, %s", in->name); 393 else if (ret < 0) 394 VDEV_LOG(ERR, "failed to add vdev, %s", in->name); 395 break; 396 default: 397 VDEV_LOG(ERR, "vdev cannot recognize this message"); 398 } 399 400 return 0; 401 } 402 403 static int 404 vdev_scan(void) 405 { 406 struct rte_vdev_device *dev; 407 struct rte_devargs *devargs; 408 struct vdev_custom_scan *custom_scan; 409 410 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && 411 rte_errno != EEXIST) { 412 /* for primary, unsupported IPC is not an error */ 413 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 414 rte_errno == ENOTSUP) 415 goto scan; 416 VDEV_LOG(ERR, "Failed to add vdev mp action"); 417 return -1; 418 } 419 420 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 421 struct rte_mp_msg mp_req, *mp_rep; 422 struct rte_mp_reply mp_reply; 423 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; 424 struct vdev_param *req = (struct vdev_param *)mp_req.param; 425 struct vdev_param *resp; 426 427 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); 428 mp_req.len_param = sizeof(*req); 429 mp_req.num_fds = 0; 430 req->type = VDEV_SCAN_REQ; 431 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && 432 mp_reply.nb_received == 1) { 433 mp_rep = &mp_reply.msgs[0]; 434 resp = (struct vdev_param *)mp_rep->param; 435 VDEV_LOG(INFO, "Received %d vdevs", resp->num); 436 free(mp_reply.msgs); 437 } else 438 VDEV_LOG(ERR, "Failed to request vdev from primary"); 439 440 /* Fall through to allow private vdevs in secondary process */ 441 } 442 443 scan: 444 /* call custom scan callbacks if any */ 445 rte_spinlock_lock(&vdev_custom_scan_lock); 446 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { 447 if (custom_scan->callback != NULL) 448 /* 449 * the callback should update devargs list 450 * by calling rte_devargs_insert() with 451 * devargs.bus = rte_bus_find_by_name("vdev"); 452 * devargs.type = RTE_DEVTYPE_VIRTUAL; 453 * devargs.policy = RTE_DEV_WHITELISTED; 454 */ 455 custom_scan->callback(custom_scan->user_arg); 456 } 457 rte_spinlock_unlock(&vdev_custom_scan_lock); 458 459 /* for virtual devices we scan the devargs_list populated via cmdline */ 460 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { 461 462 dev = calloc(1, sizeof(*dev)); 463 if (!dev) 464 return -1; 465 466 rte_spinlock_recursive_lock(&vdev_device_list_lock); 467 468 if (find_vdev(devargs->name)) { 469 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 470 free(dev); 471 continue; 472 } 473 474 dev->device.bus = &rte_vdev_bus; 475 dev->device.devargs = devargs; 476 dev->device.numa_node = SOCKET_ID_ANY; 477 dev->device.name = devargs->name; 478 479 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); 480 481 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 482 } 483 484 return 0; 485 } 486 487 static int 488 vdev_probe(void) 489 { 490 struct rte_vdev_device *dev; 491 int r, ret = 0; 492 493 /* call the init function for each virtual device */ 494 TAILQ_FOREACH(dev, &vdev_device_list, next) { 495 /* we don't use the vdev lock here, as it's only used in DPDK 496 * initialization; and we don't want to hold such a lock when 497 * we call each driver probe. 498 */ 499 500 r = vdev_probe_all_drivers(dev); 501 if (r != 0) { 502 if (r == -EEXIST) 503 continue; 504 VDEV_LOG(ERR, "failed to initialize %s device", 505 rte_vdev_device_name(dev)); 506 ret = -1; 507 } 508 } 509 510 return ret; 511 } 512 513 struct rte_device * 514 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 515 const void *data) 516 { 517 const struct rte_vdev_device *vstart; 518 struct rte_vdev_device *dev; 519 520 rte_spinlock_recursive_lock(&vdev_device_list_lock); 521 if (start != NULL) { 522 vstart = RTE_DEV_TO_VDEV_CONST(start); 523 dev = TAILQ_NEXT(vstart, next); 524 } else { 525 dev = TAILQ_FIRST(&vdev_device_list); 526 } 527 while (dev != NULL) { 528 if (cmp(&dev->device, data) == 0) 529 break; 530 dev = TAILQ_NEXT(dev, next); 531 } 532 rte_spinlock_recursive_unlock(&vdev_device_list_lock); 533 534 return dev ? &dev->device : NULL; 535 } 536 537 static int 538 vdev_plug(struct rte_device *dev) 539 { 540 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); 541 } 542 543 static int 544 vdev_unplug(struct rte_device *dev) 545 { 546 return rte_vdev_uninit(dev->name); 547 } 548 549 static struct rte_bus rte_vdev_bus = { 550 .scan = vdev_scan, 551 .probe = vdev_probe, 552 .find_device = rte_vdev_find_device, 553 .plug = vdev_plug, 554 .unplug = vdev_unplug, 555 .parse = vdev_parse, 556 .dev_iterate = rte_vdev_dev_iterate, 557 }; 558 559 RTE_REGISTER_BUS(vdev, rte_vdev_bus); 560 561 RTE_INIT(vdev_init_log) 562 { 563 vdev_logtype_bus = rte_log_register("bus.vdev"); 564 if (vdev_logtype_bus >= 0) 565 rte_log_set_level(vdev_logtype_bus, RTE_LOG_NOTICE); 566 } 567