1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "env_internal.h" 35 36 #include "spdk/env.h" 37 38 #define SYSFS_PCI_DRIVERS "/sys/bus/pci/drivers" 39 40 #define PCI_CFG_SIZE 256 41 #define PCI_EXT_CAP_ID_SN 0x03 42 43 int 44 spdk_pci_device_init(struct rte_pci_driver *driver, 45 struct rte_pci_device *device) 46 { 47 struct spdk_pci_enum_ctx *ctx = (struct spdk_pci_enum_ctx *)driver; 48 int rc; 49 50 if (!ctx->cb_fn) { 51 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 52 rte_pci_unmap_device(device); 53 #elif RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) 54 rte_eal_pci_unmap_device(device); 55 #endif 56 57 /* Return a positive value to indicate that this device does not belong to this driver, but 58 * this isn't an error. */ 59 return 1; 60 } 61 62 if (device->kdrv == RTE_KDRV_VFIO) { 63 /* 64 * TODO: This is a workaround for an issue where the device is not ready after VFIO reset. 65 * Figure out what is actually going wrong and remove this sleep. 66 */ 67 usleep(500 * 1000); 68 } 69 70 rc = ctx->cb_fn(ctx->cb_arg, (struct spdk_pci_device *)device); 71 if (rc != 0) { 72 return rc; 73 } 74 75 spdk_vtophys_pci_device_added(device); 76 return 0; 77 } 78 79 int 80 spdk_pci_device_fini(struct rte_pci_device *device) 81 { 82 spdk_vtophys_pci_device_removed(device); 83 return 0; 84 } 85 86 void 87 spdk_pci_device_detach(struct spdk_pci_device *device) 88 { 89 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) 90 #if RTE_VERSION < RTE_VERSION_NUM(17, 05, 0, 0) 91 rte_eal_device_remove(&device->device); 92 #endif 93 #endif 94 95 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 96 struct spdk_pci_addr addr; 97 char bdf[32]; 98 99 addr.domain = device->addr.domain; 100 addr.bus = device->addr.bus; 101 addr.dev = device->addr.devid; 102 addr.func = device->addr.function; 103 104 spdk_pci_addr_fmt(bdf, sizeof(bdf), &addr); 105 if (rte_eal_dev_detach(&device->device) < 0) { 106 fprintf(stderr, "Failed to detach PCI device %s (device already removed?).\n", bdf); 107 } 108 #elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 109 rte_pci_detach(&device->addr); 110 #else 111 rte_eal_pci_detach(&device->addr); 112 #endif 113 } 114 115 int 116 spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx, 117 spdk_pci_enum_cb enum_cb, 118 void *enum_ctx, struct spdk_pci_addr *pci_address) 119 { 120 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 121 char bdf[32]; 122 123 spdk_pci_addr_fmt(bdf, sizeof(bdf), pci_address); 124 #else 125 struct rte_pci_addr addr; 126 127 addr.domain = pci_address->domain; 128 addr.bus = pci_address->bus; 129 addr.devid = pci_address->dev; 130 addr.function = pci_address->func; 131 #endif 132 133 pthread_mutex_lock(&ctx->mtx); 134 135 if (!ctx->is_registered) { 136 ctx->is_registered = true; 137 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 138 rte_pci_register(&ctx->driver); 139 #else 140 rte_eal_pci_register(&ctx->driver); 141 #endif 142 } 143 144 ctx->cb_fn = enum_cb; 145 ctx->cb_arg = enum_ctx; 146 147 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 148 if (rte_eal_dev_attach(bdf, "") != 0) { 149 #elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 150 if (rte_pci_probe_one(&addr) != 0) { 151 #else 152 if (rte_eal_pci_probe_one(&addr) != 0) { 153 #endif 154 ctx->cb_arg = NULL; 155 ctx->cb_fn = NULL; 156 pthread_mutex_unlock(&ctx->mtx); 157 return -1; 158 } 159 160 ctx->cb_arg = NULL; 161 ctx->cb_fn = NULL; 162 pthread_mutex_unlock(&ctx->mtx); 163 164 return 0; 165 } 166 167 /* Note: You can call spdk_pci_enumerate from more than one thread 168 * simultaneously safely, but you cannot call spdk_pci_enumerate 169 * and rte_eal_pci_probe simultaneously. 170 */ 171 int 172 spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx, 173 spdk_pci_enum_cb enum_cb, 174 void *enum_ctx) 175 { 176 pthread_mutex_lock(&ctx->mtx); 177 178 if (!ctx->is_registered) { 179 ctx->is_registered = true; 180 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 181 rte_pci_register(&ctx->driver); 182 #else 183 rte_eal_pci_register(&ctx->driver); 184 #endif 185 } 186 187 ctx->cb_fn = enum_cb; 188 ctx->cb_arg = enum_ctx; 189 190 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 191 if (rte_bus_probe() != 0) { 192 #elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 193 if (rte_pci_probe() != 0) { 194 #else 195 if (rte_eal_pci_probe() != 0) { 196 #endif 197 ctx->cb_arg = NULL; 198 ctx->cb_fn = NULL; 199 pthread_mutex_unlock(&ctx->mtx); 200 return -1; 201 } 202 203 ctx->cb_arg = NULL; 204 ctx->cb_fn = NULL; 205 pthread_mutex_unlock(&ctx->mtx); 206 207 return 0; 208 } 209 210 int 211 spdk_pci_device_map_bar(struct spdk_pci_device *device, uint32_t bar, 212 void **mapped_addr, uint64_t *phys_addr, uint64_t *size) 213 { 214 struct rte_pci_device *dev = device; 215 216 *mapped_addr = dev->mem_resource[bar].addr; 217 *phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr; 218 *size = (uint64_t)dev->mem_resource[bar].len; 219 220 return 0; 221 } 222 223 int 224 spdk_pci_device_unmap_bar(struct spdk_pci_device *device, uint32_t bar, void *addr) 225 { 226 return 0; 227 } 228 229 uint32_t 230 spdk_pci_device_get_domain(struct spdk_pci_device *dev) 231 { 232 return dev->addr.domain; 233 } 234 235 uint8_t 236 spdk_pci_device_get_bus(struct spdk_pci_device *dev) 237 { 238 return dev->addr.bus; 239 } 240 241 uint8_t 242 spdk_pci_device_get_dev(struct spdk_pci_device *dev) 243 { 244 return dev->addr.devid; 245 } 246 247 uint8_t 248 spdk_pci_device_get_func(struct spdk_pci_device *dev) 249 { 250 return dev->addr.function; 251 } 252 253 uint16_t 254 spdk_pci_device_get_vendor_id(struct spdk_pci_device *dev) 255 { 256 return dev->id.vendor_id; 257 } 258 259 uint16_t 260 spdk_pci_device_get_device_id(struct spdk_pci_device *dev) 261 { 262 return dev->id.device_id; 263 } 264 265 uint16_t 266 spdk_pci_device_get_subvendor_id(struct spdk_pci_device *dev) 267 { 268 return dev->id.subsystem_vendor_id; 269 } 270 271 uint16_t 272 spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev) 273 { 274 return dev->id.subsystem_device_id; 275 } 276 277 struct spdk_pci_id 278 spdk_pci_device_get_id(struct spdk_pci_device *pci_dev) 279 { 280 struct spdk_pci_id pci_id; 281 282 pci_id.vendor_id = spdk_pci_device_get_vendor_id(pci_dev); 283 pci_id.device_id = spdk_pci_device_get_device_id(pci_dev); 284 pci_id.subvendor_id = spdk_pci_device_get_subvendor_id(pci_dev); 285 pci_id.subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev); 286 287 return pci_id; 288 } 289 290 int 291 spdk_pci_device_get_socket_id(struct spdk_pci_device *pci_dev) 292 { 293 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) 294 return pci_dev->device.numa_node; 295 #else 296 return pci_dev->numa_node; 297 #endif 298 } 299 300 int 301 spdk_pci_device_cfg_read(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 302 { 303 int rc; 304 305 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 306 rc = rte_pci_read_config(dev, value, len, offset); 307 #else 308 rc = rte_eal_pci_read_config(dev, value, len, offset); 309 #endif 310 return (rc > 0 && (uint32_t) rc == len) ? 0 : -1; 311 } 312 313 int 314 spdk_pci_device_cfg_write(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset) 315 { 316 int rc; 317 318 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) 319 rc = rte_pci_write_config(dev, value, len, offset); 320 #else 321 rc = rte_eal_pci_write_config(dev, value, len, offset); 322 #endif 323 return (rc > 0 && (uint32_t) rc == len) ? 0 : -1; 324 } 325 326 int 327 spdk_pci_device_cfg_read8(struct spdk_pci_device *dev, uint8_t *value, uint32_t offset) 328 { 329 return spdk_pci_device_cfg_read(dev, value, 1, offset); 330 } 331 332 int 333 spdk_pci_device_cfg_write8(struct spdk_pci_device *dev, uint8_t value, uint32_t offset) 334 { 335 return spdk_pci_device_cfg_write(dev, &value, 1, offset); 336 } 337 338 int 339 spdk_pci_device_cfg_read16(struct spdk_pci_device *dev, uint16_t *value, uint32_t offset) 340 { 341 return spdk_pci_device_cfg_read(dev, value, 2, offset); 342 } 343 344 int 345 spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uint32_t offset) 346 { 347 return spdk_pci_device_cfg_write(dev, &value, 2, offset); 348 } 349 350 int 351 spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset) 352 { 353 return spdk_pci_device_cfg_read(dev, value, 4, offset); 354 } 355 356 int 357 spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset) 358 { 359 return spdk_pci_device_cfg_write(dev, &value, 4, offset); 360 } 361 362 int 363 spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t len) 364 { 365 int err; 366 uint32_t pos, header = 0; 367 uint32_t i, buf[2]; 368 369 if (len < 17) { 370 return -1; 371 } 372 373 err = spdk_pci_device_cfg_read32(dev, &header, PCI_CFG_SIZE); 374 if (err || !header) { 375 return -1; 376 } 377 378 pos = PCI_CFG_SIZE; 379 while (1) { 380 if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) { 381 if (pos) { 382 /* skip the header */ 383 pos += 4; 384 for (i = 0; i < 2; i++) { 385 err = spdk_pci_device_cfg_read32(dev, &buf[i], pos + 4 * i); 386 if (err) { 387 return -1; 388 } 389 } 390 snprintf(sn, len, "%08x%08x", buf[1], buf[0]); 391 return 0; 392 } 393 } 394 pos = (header >> 20) & 0xffc; 395 /* 0 if no other items exist */ 396 if (pos < PCI_CFG_SIZE) { 397 return -1; 398 } 399 err = spdk_pci_device_cfg_read32(dev, &header, pos); 400 if (err) { 401 return -1; 402 } 403 } 404 return -1; 405 } 406 407 struct spdk_pci_addr 408 spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev) 409 { 410 struct spdk_pci_addr pci_addr; 411 412 pci_addr.domain = spdk_pci_device_get_domain(pci_dev); 413 pci_addr.bus = spdk_pci_device_get_bus(pci_dev); 414 pci_addr.dev = spdk_pci_device_get_dev(pci_dev); 415 pci_addr.func = spdk_pci_device_get_func(pci_dev); 416 417 return pci_addr; 418 } 419 420 int 421 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2) 422 { 423 if (a1->domain > a2->domain) { 424 return 1; 425 } else if (a1->domain < a2->domain) { 426 return -1; 427 } else if (a1->bus > a2->bus) { 428 return 1; 429 } else if (a1->bus < a2->bus) { 430 return -1; 431 } else if (a1->dev > a2->dev) { 432 return 1; 433 } else if (a1->dev < a2->dev) { 434 return -1; 435 } else if (a1->func > a2->func) { 436 return 1; 437 } else if (a1->func < a2->func) { 438 return -1; 439 } 440 441 return 0; 442 } 443 444 #ifdef __linux__ 445 int 446 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr) 447 { 448 int dev_fd; 449 char dev_name[64]; 450 int pid; 451 void *dev_map; 452 struct flock pcidev_lock = { 453 .l_type = F_WRLCK, 454 .l_whence = SEEK_SET, 455 .l_start = 0, 456 .l_len = 0, 457 }; 458 459 snprintf(dev_name, sizeof(dev_name), "/tmp/spdk_pci_lock_%04x:%02x:%02x.%x", pci_addr->domain, 460 pci_addr->bus, 461 pci_addr->dev, pci_addr->func); 462 463 dev_fd = open(dev_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 464 if (dev_fd == -1) { 465 fprintf(stderr, "could not open %s\n", dev_name); 466 return -1; 467 } 468 469 if (ftruncate(dev_fd, sizeof(int)) != 0) { 470 fprintf(stderr, "could not truncate %s\n", dev_name); 471 close(dev_fd); 472 return -1; 473 } 474 475 dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, 476 MAP_SHARED, dev_fd, 0); 477 if (dev_map == MAP_FAILED) { 478 fprintf(stderr, "could not mmap dev %s (%d)\n", dev_name, errno); 479 close(dev_fd); 480 return -1; 481 } 482 483 if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) { 484 pid = *(int *)dev_map; 485 fprintf(stderr, "Cannot create lock on device %s, probably" 486 " process %d has claimed it\n", dev_name, pid); 487 munmap(dev_map, sizeof(int)); 488 close(dev_fd); 489 return -1; 490 } 491 492 *(int *)dev_map = (int)getpid(); 493 munmap(dev_map, sizeof(int)); 494 /* Keep dev_fd open to maintain the lock. */ 495 return dev_fd; 496 } 497 #endif /* __linux__ */ 498 499 #ifdef __FreeBSD__ 500 int 501 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr) 502 { 503 /* TODO */ 504 return 0; 505 } 506 #endif /* __FreeBSD__ */ 507 508 int 509 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf) 510 { 511 unsigned domain, bus, dev, func; 512 513 if (addr == NULL || bdf == NULL) { 514 return -EINVAL; 515 } 516 517 if ((sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) || 518 (sscanf(bdf, "%x.%x.%x.%x", &domain, &bus, &dev, &func) == 4)) { 519 /* Matched a full address - all variables are initialized */ 520 } else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) { 521 func = 0; 522 } else if ((sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) || 523 (sscanf(bdf, "%x.%x.%x", &bus, &dev, &func) == 3)) { 524 domain = 0; 525 } else if ((sscanf(bdf, "%x:%x", &bus, &dev) == 2) || 526 (sscanf(bdf, "%x.%x", &bus, &dev) == 2)) { 527 domain = 0; 528 func = 0; 529 } else { 530 return -EINVAL; 531 } 532 533 if (bus > 0xFF || dev > 0x1F || func > 7) { 534 return -EINVAL; 535 } 536 537 addr->domain = domain; 538 addr->bus = bus; 539 addr->dev = dev; 540 addr->func = func; 541 542 return 0; 543 } 544 545 int 546 spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr) 547 { 548 int rc; 549 550 rc = snprintf(bdf, sz, "%04x:%02x:%02x.%x", 551 addr->domain, addr->bus, 552 addr->dev, addr->func); 553 554 if (rc > 0 && (size_t)rc < sz) { 555 return 0; 556 } 557 558 return -1; 559 } 560