1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "env_internal.h" 35 36 #include <rte_alarm.h> 37 #include "spdk/env.h" 38 39 #define SYSFS_PCI_DRIVERS "/sys/bus/pci/drivers" 40 41 #define PCI_CFG_SIZE 256 42 #define PCI_EXT_CAP_ID_SN 0x03 43 44 /* DPDK 18.11+ hotplug isn't robust. Multiple apps starting at the same time 45 * might cause the internal IPC to misbehave. Just retry in such case. 46 */ 47 #define DPDK_HOTPLUG_RETRY_COUNT 4 48 49 /* DPDK alarm/interrupt thread */ 50 static pthread_t g_dpdk_tid; 51 static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER; 52 static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); 53 /* devices hotplugged on a dpdk thread */ 54 static TAILQ_HEAD(, spdk_pci_device) g_pci_hotplugged_devices = 55 TAILQ_HEAD_INITIALIZER(g_pci_hotplugged_devices); 56 static TAILQ_HEAD(, spdk_pci_driver) g_pci_drivers = TAILQ_HEAD_INITIALIZER(g_pci_drivers); 57 58 static int 59 spdk_map_bar_rte(struct spdk_pci_device *device, uint32_t bar, 60 void **mapped_addr, uint64_t *phys_addr, uint64_t *size) 61 { 62 struct rte_pci_device *dev = device->dev_handle; 63 64 *mapped_addr = dev->mem_resource[bar].addr; 65 *phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr; 66 *size = (uint64_t)dev->mem_resource[bar].len; 67 68 return 0; 69 } 70 71 static int 72 spdk_unmap_bar_rte(struct spdk_pci_device *device, uint32_t bar, void *addr) 73 { 74 return 0; 75 } 76 77 static int 78 spdk_cfg_read_rte(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 79 { 80 int rc; 81 82 rc = rte_pci_read_config(dev->dev_handle, value, len, offset); 83 84 #if defined(__FreeBSD__) && RTE_VERSION < RTE_VERSION_NUM(18, 11, 0, 0) 85 /* Older DPDKs return 0 on success and -1 on failure */ 86 return rc; 87 #endif 88 return (rc > 0 && (uint32_t) rc == len) ? 0 : -1; 89 } 90 91 static int 92 spdk_cfg_write_rte(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 93 { 94 int rc; 95 96 rc = rte_pci_write_config(dev->dev_handle, value, len, offset); 97 98 #ifdef __FreeBSD__ 99 /* DPDK returns 0 on success and -1 on failure */ 100 return rc; 101 #endif 102 return (rc > 0 && (uint32_t) rc == len) ? 0 : -1; 103 } 104 105 static void 106 spdk_detach_rte_cb(void *_dev) 107 { 108 struct rte_pci_device *rte_dev = _dev; 109 110 #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) 111 char bdf[32]; 112 int i = 0, rc; 113 114 snprintf(bdf, sizeof(bdf), "%s", rte_dev->device.name); 115 do { 116 rc = rte_eal_hotplug_remove("pci", bdf); 117 } while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT); 118 #else 119 rte_eal_dev_detach(&rte_dev->device); 120 #endif 121 } 122 123 static void 124 spdk_detach_rte(struct spdk_pci_device *dev) 125 { 126 struct rte_pci_device *rte_dev = dev->dev_handle; 127 int i; 128 bool removed; 129 130 /* The device was already marked as available and could be attached 131 * again while we go asynchronous, so we explicitly forbid that. 132 */ 133 dev->internal.pending_removal = true; 134 if (spdk_process_is_primary() && !pthread_equal(g_dpdk_tid, pthread_self())) { 135 rte_eal_alarm_set(1, spdk_detach_rte_cb, rte_dev); 136 /* wait up to 20ms for the cb to start executing */ 137 for (i = 20; i > 0; i--) { 138 139 spdk_delay_us(1000); 140 pthread_mutex_lock(&g_pci_mutex); 141 removed = dev->internal.removed; 142 pthread_mutex_unlock(&g_pci_mutex); 143 144 if (removed) { 145 break; 146 } 147 } 148 149 /* besides checking the removed flag, we also need to wait 150 * for the dpdk detach function to unwind, as it's doing some 151 * operations even after calling our detach callback. Simply 152 * cancell the alarm - if it started executing already, this 153 * call will block and wait for it to finish. 154 */ 155 rte_eal_alarm_cancel(spdk_detach_rte_cb, rte_dev); 156 157 /* the device could have been finally removed, so just check 158 * it again. 159 */ 160 pthread_mutex_lock(&g_pci_mutex); 161 removed = dev->internal.removed; 162 pthread_mutex_unlock(&g_pci_mutex); 163 if (!removed) { 164 fprintf(stderr, "Timeout waiting for DPDK to remove PCI device %s.\n", 165 rte_dev->name); 166 } 167 } else { 168 spdk_detach_rte_cb(rte_dev); 169 } 170 } 171 172 void 173 spdk_pci_driver_register(struct spdk_pci_driver *driver) 174 { 175 TAILQ_INSERT_TAIL(&g_pci_drivers, driver, tailq); 176 } 177 178 #if RTE_VERSION >= RTE_VERSION_NUM(18, 5, 0, 0) 179 static void 180 spdk_pci_device_rte_hotremove(const char *device_name, 181 enum rte_dev_event_type event, 182 void *cb_arg) 183 { 184 struct spdk_pci_device *dev; 185 bool can_detach = false; 186 187 if (event != RTE_DEV_EVENT_REMOVE) { 188 return; 189 } 190 191 pthread_mutex_lock(&g_pci_mutex); 192 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 193 struct rte_pci_device *rte_dev = dev->dev_handle; 194 195 if (strcmp(rte_dev->name, device_name) == 0 && 196 !dev->internal.pending_removal) { 197 can_detach = !dev->internal.attached; 198 /* prevent any further attaches */ 199 dev->internal.pending_removal = true; 200 break; 201 } 202 } 203 pthread_mutex_unlock(&g_pci_mutex); 204 205 if (dev != NULL && can_detach) { 206 /* if device is not attached, we can remove it right away. */ 207 spdk_detach_rte(dev); 208 } 209 } 210 #endif 211 212 static void 213 cleanup_pci_devices(void) 214 { 215 struct spdk_pci_device *dev, *tmp; 216 217 pthread_mutex_lock(&g_pci_mutex); 218 /* cleanup removed devices */ 219 TAILQ_FOREACH_SAFE(dev, &g_pci_devices, internal.tailq, tmp) { 220 if (!dev->internal.removed) { 221 continue; 222 } 223 224 spdk_vtophys_pci_device_removed(dev->dev_handle); 225 TAILQ_REMOVE(&g_pci_devices, dev, internal.tailq); 226 free(dev); 227 } 228 229 /* add newly-attached devices */ 230 TAILQ_FOREACH_SAFE(dev, &g_pci_hotplugged_devices, internal.tailq, tmp) { 231 TAILQ_REMOVE(&g_pci_hotplugged_devices, dev, internal.tailq); 232 TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq); 233 spdk_vtophys_pci_device_added(dev->dev_handle); 234 } 235 pthread_mutex_unlock(&g_pci_mutex); 236 } 237 238 static void 239 _get_alarm_thread_cb(void *unused) 240 { 241 g_dpdk_tid = pthread_self(); 242 } 243 244 void 245 spdk_pci_init(void) 246 { 247 #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) 248 struct spdk_pci_driver *driver; 249 250 /* We need to pre-register pci drivers for the pci devices to be 251 * attachable in multi-process with DPDK 18.11+. 252 * 253 * DPDK 18.11+ does its best to ensure all devices are equally 254 * attached or detached in all processes within a shared memory group. 255 * For SPDK it means that if a device is hotplugged in the primary, 256 * then DPDK will automatically send an IPC hotplug request to all other 257 * processes. Those other processes may not have the same SPDK PCI 258 * driver registered and may fail to attach the device. DPDK will send 259 * back the failure status, and the the primary process will also fail 260 * to hotplug the device. To prevent that, we need to pre-register the 261 * pci drivers here. 262 */ 263 TAILQ_FOREACH(driver, &g_pci_drivers, tailq) { 264 assert(!driver->is_registered); 265 driver->is_registered = true; 266 rte_pci_register(&driver->driver); 267 } 268 #endif 269 270 #if RTE_VERSION >= RTE_VERSION_NUM(18, 5, 0, 0) 271 /* Register a single hotremove callback for all devices. */ 272 if (spdk_process_is_primary()) { 273 rte_dev_event_callback_register(NULL, spdk_pci_device_rte_hotremove, NULL); 274 } 275 #endif 276 277 rte_eal_alarm_set(1, _get_alarm_thread_cb, NULL); 278 /* alarms are executed in order, so this one will be always executed 279 * before any real hotremove alarms and we don't need to wait for it. 280 */ 281 } 282 283 void 284 spdk_pci_fini(void) 285 { 286 struct spdk_pci_device *dev; 287 char bdf[32]; 288 289 cleanup_pci_devices(); 290 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 291 if (dev->internal.attached) { 292 spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->addr); 293 fprintf(stderr, "Device %s is still attached at shutdown!\n", bdf); 294 } 295 } 296 297 #if RTE_VERSION >= RTE_VERSION_NUM(18, 5, 0, 0) 298 if (spdk_process_is_primary()) { 299 rte_dev_event_callback_unregister(NULL, spdk_pci_device_rte_hotremove, NULL); 300 } 301 #endif 302 } 303 304 int 305 spdk_pci_device_init(struct rte_pci_driver *_drv, 306 struct rte_pci_device *_dev) 307 { 308 struct spdk_pci_driver *driver = (struct spdk_pci_driver *)_drv; 309 struct spdk_pci_device *dev; 310 int rc; 311 312 #if RTE_VERSION < RTE_VERSION_NUM(18, 11, 0, 0) 313 if (!driver->cb_fn) { 314 /* Return a positive value to indicate that this device does 315 * not belong to this driver, but this isn't an error. 316 */ 317 return 1; 318 } 319 #endif 320 321 dev = calloc(1, sizeof(*dev)); 322 if (dev == NULL) { 323 return -1; 324 } 325 326 dev->dev_handle = _dev; 327 328 dev->addr.domain = _dev->addr.domain; 329 dev->addr.bus = _dev->addr.bus; 330 dev->addr.dev = _dev->addr.devid; 331 dev->addr.func = _dev->addr.function; 332 dev->id.vendor_id = _dev->id.vendor_id; 333 dev->id.device_id = _dev->id.device_id; 334 dev->id.subvendor_id = _dev->id.subsystem_vendor_id; 335 dev->id.subdevice_id = _dev->id.subsystem_device_id; 336 dev->socket_id = _dev->device.numa_node; 337 338 dev->map_bar = spdk_map_bar_rte; 339 dev->unmap_bar = spdk_unmap_bar_rte; 340 dev->cfg_read = spdk_cfg_read_rte; 341 dev->cfg_write = spdk_cfg_write_rte; 342 dev->detach = spdk_detach_rte; 343 344 dev->internal.driver = driver; 345 dev->internal.claim_fd = -1; 346 347 if (driver->cb_fn != NULL) { 348 rc = driver->cb_fn(driver->cb_arg, dev); 349 if (rc != 0) { 350 free(dev); 351 return rc; 352 } 353 dev->internal.attached = true; 354 } 355 356 pthread_mutex_lock(&g_pci_mutex); 357 TAILQ_INSERT_TAIL(&g_pci_hotplugged_devices, dev, internal.tailq); 358 pthread_mutex_unlock(&g_pci_mutex); 359 return 0; 360 } 361 362 int 363 spdk_pci_device_fini(struct rte_pci_device *_dev) 364 { 365 struct spdk_pci_device *dev; 366 367 pthread_mutex_lock(&g_pci_mutex); 368 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 369 if (dev->dev_handle == _dev) { 370 break; 371 } 372 } 373 374 if (dev == NULL || dev->internal.attached) { 375 /* The device might be still referenced somewhere in SPDK. */ 376 pthread_mutex_unlock(&g_pci_mutex); 377 return -1; 378 } 379 380 assert(!dev->internal.removed); 381 dev->internal.removed = true; 382 pthread_mutex_unlock(&g_pci_mutex); 383 return 0; 384 385 } 386 387 void 388 spdk_pci_device_detach(struct spdk_pci_device *dev) 389 { 390 assert(dev->internal.attached); 391 392 if (dev->internal.claim_fd >= 0) { 393 spdk_pci_device_unclaim(dev); 394 } 395 396 dev->internal.attached = false; 397 dev->detach(dev); 398 399 cleanup_pci_devices(); 400 } 401 402 int 403 spdk_pci_device_attach(struct spdk_pci_driver *driver, 404 spdk_pci_enum_cb enum_cb, 405 void *enum_ctx, struct spdk_pci_addr *pci_address) 406 { 407 struct spdk_pci_device *dev; 408 int rc; 409 char bdf[32]; 410 411 spdk_pci_addr_fmt(bdf, sizeof(bdf), pci_address); 412 413 cleanup_pci_devices(); 414 415 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 416 if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) { 417 break; 418 } 419 } 420 421 if (dev != NULL && dev->internal.driver == driver) { 422 pthread_mutex_lock(&g_pci_mutex); 423 if (dev->internal.attached || dev->internal.pending_removal) { 424 pthread_mutex_unlock(&g_pci_mutex); 425 return -1; 426 } 427 428 rc = enum_cb(enum_ctx, dev); 429 if (rc == 0) { 430 dev->internal.attached = true; 431 } 432 pthread_mutex_unlock(&g_pci_mutex); 433 return rc; 434 } 435 436 if (!driver->is_registered) { 437 driver->is_registered = true; 438 rte_pci_register(&driver->driver); 439 } 440 441 driver->cb_fn = enum_cb; 442 driver->cb_arg = enum_ctx; 443 444 #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) 445 int i = 0; 446 447 do { 448 rc = rte_eal_hotplug_add("pci", bdf, ""); 449 } while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT); 450 451 if (i > 1 && rc == -EEXIST) { 452 /* Even though the previous request timed out, the device 453 * was attached successfully. 454 */ 455 rc = 0; 456 } 457 #else 458 rc = rte_eal_dev_attach(bdf, ""); 459 #endif 460 461 driver->cb_arg = NULL; 462 driver->cb_fn = NULL; 463 464 cleanup_pci_devices(); 465 return rc == 0 ? 0 : -1; 466 } 467 468 /* Note: You can call spdk_pci_enumerate from more than one thread 469 * simultaneously safely, but you cannot call spdk_pci_enumerate 470 * and rte_eal_pci_probe simultaneously. 471 */ 472 int 473 spdk_pci_enumerate(struct spdk_pci_driver *driver, 474 spdk_pci_enum_cb enum_cb, 475 void *enum_ctx) 476 { 477 struct spdk_pci_device *dev; 478 int rc; 479 480 cleanup_pci_devices(); 481 482 pthread_mutex_lock(&g_pci_mutex); 483 TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { 484 if (dev->internal.attached || 485 dev->internal.driver != driver || 486 dev->internal.pending_removal) { 487 continue; 488 } 489 490 rc = enum_cb(enum_ctx, dev); 491 if (rc == 0) { 492 dev->internal.attached = true; 493 } else if (rc < 0) { 494 pthread_mutex_unlock(&g_pci_mutex); 495 return -1; 496 } 497 } 498 pthread_mutex_unlock(&g_pci_mutex); 499 500 if (!driver->is_registered) { 501 driver->is_registered = true; 502 rte_pci_register(&driver->driver); 503 } 504 505 driver->cb_fn = enum_cb; 506 driver->cb_arg = enum_ctx; 507 508 if (rte_bus_scan() != 0 || rte_bus_probe() != 0) { 509 driver->cb_arg = NULL; 510 driver->cb_fn = NULL; 511 return -1; 512 } 513 514 driver->cb_arg = NULL; 515 driver->cb_fn = NULL; 516 517 cleanup_pci_devices(); 518 return 0; 519 } 520 521 struct spdk_pci_device * 522 spdk_pci_get_first_device(void) 523 { 524 return TAILQ_FIRST(&g_pci_devices); 525 } 526 527 struct spdk_pci_device * 528 spdk_pci_get_next_device(struct spdk_pci_device *prev) 529 { 530 return TAILQ_NEXT(prev, internal.tailq); 531 } 532 533 int 534 spdk_pci_device_map_bar(struct spdk_pci_device *dev, uint32_t bar, 535 void **mapped_addr, uint64_t *phys_addr, uint64_t *size) 536 { 537 return dev->map_bar(dev, bar, mapped_addr, phys_addr, size); 538 } 539 540 int 541 spdk_pci_device_unmap_bar(struct spdk_pci_device *dev, uint32_t bar, void *addr) 542 { 543 return dev->unmap_bar(dev, bar, addr); 544 } 545 546 uint32_t 547 spdk_pci_device_get_domain(struct spdk_pci_device *dev) 548 { 549 return dev->addr.domain; 550 } 551 552 uint8_t 553 spdk_pci_device_get_bus(struct spdk_pci_device *dev) 554 { 555 return dev->addr.bus; 556 } 557 558 uint8_t 559 spdk_pci_device_get_dev(struct spdk_pci_device *dev) 560 { 561 return dev->addr.dev; 562 } 563 564 uint8_t 565 spdk_pci_device_get_func(struct spdk_pci_device *dev) 566 { 567 return dev->addr.func; 568 } 569 570 uint16_t 571 spdk_pci_device_get_vendor_id(struct spdk_pci_device *dev) 572 { 573 return dev->id.vendor_id; 574 } 575 576 uint16_t 577 spdk_pci_device_get_device_id(struct spdk_pci_device *dev) 578 { 579 return dev->id.device_id; 580 } 581 582 uint16_t 583 spdk_pci_device_get_subvendor_id(struct spdk_pci_device *dev) 584 { 585 return dev->id.subvendor_id; 586 } 587 588 uint16_t 589 spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev) 590 { 591 return dev->id.subdevice_id; 592 } 593 594 struct spdk_pci_id 595 spdk_pci_device_get_id(struct spdk_pci_device *dev) 596 { 597 return dev->id; 598 } 599 600 int 601 spdk_pci_device_get_socket_id(struct spdk_pci_device *dev) 602 { 603 return dev->socket_id; 604 } 605 606 int 607 spdk_pci_device_cfg_read(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 608 { 609 return dev->cfg_read(dev, value, len, offset); 610 } 611 612 int 613 spdk_pci_device_cfg_write(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 614 { 615 return dev->cfg_write(dev, value, len, offset); 616 } 617 618 int 619 spdk_pci_device_cfg_read8(struct spdk_pci_device *dev, uint8_t *value, uint32_t offset) 620 { 621 return spdk_pci_device_cfg_read(dev, value, 1, offset); 622 } 623 624 int 625 spdk_pci_device_cfg_write8(struct spdk_pci_device *dev, uint8_t value, uint32_t offset) 626 { 627 return spdk_pci_device_cfg_write(dev, &value, 1, offset); 628 } 629 630 int 631 spdk_pci_device_cfg_read16(struct spdk_pci_device *dev, uint16_t *value, uint32_t offset) 632 { 633 return spdk_pci_device_cfg_read(dev, value, 2, offset); 634 } 635 636 int 637 spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uint32_t offset) 638 { 639 return spdk_pci_device_cfg_write(dev, &value, 2, offset); 640 } 641 642 int 643 spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset) 644 { 645 return spdk_pci_device_cfg_read(dev, value, 4, offset); 646 } 647 648 int 649 spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset) 650 { 651 return spdk_pci_device_cfg_write(dev, &value, 4, offset); 652 } 653 654 int 655 spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t len) 656 { 657 int err; 658 uint32_t pos, header = 0; 659 uint32_t i, buf[2]; 660 661 if (len < 17) { 662 return -1; 663 } 664 665 err = spdk_pci_device_cfg_read32(dev, &header, PCI_CFG_SIZE); 666 if (err || !header) { 667 return -1; 668 } 669 670 pos = PCI_CFG_SIZE; 671 while (1) { 672 if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) { 673 if (pos) { 674 /* skip the header */ 675 pos += 4; 676 for (i = 0; i < 2; i++) { 677 err = spdk_pci_device_cfg_read32(dev, &buf[i], pos + 4 * i); 678 if (err) { 679 return -1; 680 } 681 } 682 snprintf(sn, len, "%08x%08x", buf[1], buf[0]); 683 return 0; 684 } 685 } 686 pos = (header >> 20) & 0xffc; 687 /* 0 if no other items exist */ 688 if (pos < PCI_CFG_SIZE) { 689 return -1; 690 } 691 err = spdk_pci_device_cfg_read32(dev, &header, pos); 692 if (err) { 693 return -1; 694 } 695 } 696 return -1; 697 } 698 699 struct spdk_pci_addr 700 spdk_pci_device_get_addr(struct spdk_pci_device *dev) 701 { 702 return dev->addr; 703 } 704 705 bool 706 spdk_pci_device_is_removed(struct spdk_pci_device *dev) 707 { 708 return dev->internal.pending_removal; 709 } 710 711 int 712 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2) 713 { 714 if (a1->domain > a2->domain) { 715 return 1; 716 } else if (a1->domain < a2->domain) { 717 return -1; 718 } else if (a1->bus > a2->bus) { 719 return 1; 720 } else if (a1->bus < a2->bus) { 721 return -1; 722 } else if (a1->dev > a2->dev) { 723 return 1; 724 } else if (a1->dev < a2->dev) { 725 return -1; 726 } else if (a1->func > a2->func) { 727 return 1; 728 } else if (a1->func < a2->func) { 729 return -1; 730 } 731 732 return 0; 733 } 734 735 #ifdef __linux__ 736 int 737 spdk_pci_device_claim(struct spdk_pci_device *dev) 738 { 739 int dev_fd; 740 char dev_name[64]; 741 int pid; 742 void *dev_map; 743 struct flock pcidev_lock = { 744 .l_type = F_WRLCK, 745 .l_whence = SEEK_SET, 746 .l_start = 0, 747 .l_len = 0, 748 }; 749 750 snprintf(dev_name, sizeof(dev_name), "/tmp/spdk_pci_lock_%04x:%02x:%02x.%x", 751 dev->addr.domain, dev->addr.bus, dev->addr.dev, dev->addr.func); 752 753 dev_fd = open(dev_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 754 if (dev_fd == -1) { 755 fprintf(stderr, "could not open %s\n", dev_name); 756 return -errno; 757 } 758 759 if (ftruncate(dev_fd, sizeof(int)) != 0) { 760 fprintf(stderr, "could not truncate %s\n", dev_name); 761 close(dev_fd); 762 return -errno; 763 } 764 765 dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, 766 MAP_SHARED, dev_fd, 0); 767 if (dev_map == MAP_FAILED) { 768 fprintf(stderr, "could not mmap dev %s (%d)\n", dev_name, errno); 769 close(dev_fd); 770 return -errno; 771 } 772 773 if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) { 774 pid = *(int *)dev_map; 775 fprintf(stderr, "Cannot create lock on device %s, probably" 776 " process %d has claimed it\n", dev_name, pid); 777 munmap(dev_map, sizeof(int)); 778 close(dev_fd); 779 /* F_SETLK returns unspecified errnos, normalize them */ 780 return -EACCES; 781 } 782 783 *(int *)dev_map = (int)getpid(); 784 munmap(dev_map, sizeof(int)); 785 dev->internal.claim_fd = dev_fd; 786 /* Keep dev_fd open to maintain the lock. */ 787 return 0; 788 } 789 790 void 791 spdk_pci_device_unclaim(struct spdk_pci_device *dev) 792 { 793 char dev_name[64]; 794 795 snprintf(dev_name, sizeof(dev_name), "/tmp/spdk_pci_lock_%04x:%02x:%02x.%x", 796 dev->addr.domain, dev->addr.bus, dev->addr.dev, dev->addr.func); 797 798 close(dev->internal.claim_fd); 799 dev->internal.claim_fd = -1; 800 unlink(dev_name); 801 } 802 #endif /* __linux__ */ 803 804 #ifdef __FreeBSD__ 805 int 806 spdk_pci_device_claim(struct spdk_pci_device *dev) 807 { 808 /* TODO */ 809 return 0; 810 } 811 812 void 813 spdk_pci_device_unclaim(struct spdk_pci_device *dev) 814 { 815 /* TODO */ 816 } 817 #endif /* __FreeBSD__ */ 818 819 int 820 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf) 821 { 822 unsigned domain, bus, dev, func; 823 824 if (addr == NULL || bdf == NULL) { 825 return -EINVAL; 826 } 827 828 if ((sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) || 829 (sscanf(bdf, "%x.%x.%x.%x", &domain, &bus, &dev, &func) == 4)) { 830 /* Matched a full address - all variables are initialized */ 831 } else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) { 832 func = 0; 833 } else if ((sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) || 834 (sscanf(bdf, "%x.%x.%x", &bus, &dev, &func) == 3)) { 835 domain = 0; 836 } else if ((sscanf(bdf, "%x:%x", &bus, &dev) == 2) || 837 (sscanf(bdf, "%x.%x", &bus, &dev) == 2)) { 838 domain = 0; 839 func = 0; 840 } else { 841 return -EINVAL; 842 } 843 844 if (bus > 0xFF || dev > 0x1F || func > 7) { 845 return -EINVAL; 846 } 847 848 addr->domain = domain; 849 addr->bus = bus; 850 addr->dev = dev; 851 addr->func = func; 852 853 return 0; 854 } 855 856 int 857 spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr) 858 { 859 int rc; 860 861 rc = snprintf(bdf, sz, "%04x:%02x:%02x.%x", 862 addr->domain, addr->bus, 863 addr->dev, addr->func); 864 865 if (rc > 0 && (size_t)rc < sz) { 866 return 0; 867 } 868 869 return -1; 870 } 871 872 void 873 spdk_pci_hook_device(struct spdk_pci_driver *drv, struct spdk_pci_device *dev) 874 { 875 assert(dev->map_bar != NULL); 876 assert(dev->unmap_bar != NULL); 877 assert(dev->cfg_read != NULL); 878 assert(dev->cfg_write != NULL); 879 assert(dev->detach != NULL); 880 dev->internal.driver = drv; 881 TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq); 882 } 883 884 void 885 spdk_pci_unhook_device(struct spdk_pci_device *dev) 886 { 887 assert(!dev->internal.attached); 888 TAILQ_REMOVE(&g_pci_devices, dev, internal.tailq); 889 } 890