1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <unistd.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <fcntl.h> 39 #include <dirent.h> 40 #include <sys/mman.h> 41 #include <sys/stat.h> 42 #include <sys/types.h> 43 #include <pthread.h> 44 #include <stdbool.h> 45 46 #include <rte_config.h> 47 #include <rte_pci.h> 48 #include <rte_version.h> 49 50 #define spdk_pci_device rte_pci_device 51 52 #ifdef __FreeBSD__ 53 #include <sys/pciio.h> 54 #endif 55 56 #include "spdk/env.h" 57 #include "spdk/pci_ids.h" 58 59 #define SYSFS_PCI_DRIVERS "/sys/bus/pci/drivers" 60 61 #define PCI_CFG_SIZE 256 62 #define PCI_EXT_CAP_ID_SN 0x03 63 64 struct spdk_pci_enum_ctx { 65 struct rte_pci_driver driver; 66 spdk_pci_enum_cb enum_cb; 67 void *enum_ctx; 68 }; 69 70 static struct rte_pci_id nvme_pci_driver_id[] = { 71 #if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1) 72 { 73 .class_id = SPDK_PCI_CLASS_NVME, 74 .vendor_id = PCI_ANY_ID, 75 .device_id = PCI_ANY_ID, 76 .subsystem_vendor_id = PCI_ANY_ID, 77 .subsystem_device_id = PCI_ANY_ID, 78 }, 79 #else 80 {RTE_PCI_DEVICE(0x8086, 0x0953)}, 81 #endif 82 { .vendor_id = 0, /* sentinel */ }, 83 }; 84 85 #define SPDK_IOAT_PCI_DEVICE(DEVICE_ID) RTE_PCI_DEVICE(SPDK_PCI_VID_INTEL, DEVICE_ID) 86 static struct rte_pci_id ioat_driver_id[] = { 87 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB0)}, 88 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB1)}, 89 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB2)}, 90 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB3)}, 91 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB4)}, 92 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB5)}, 93 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB6)}, 94 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB7)}, 95 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_SNB8)}, 96 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB0)}, 97 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB1)}, 98 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB2)}, 99 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB3)}, 100 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB4)}, 101 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB5)}, 102 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB6)}, 103 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB7)}, 104 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB8)}, 105 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_IVB9)}, 106 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW0)}, 107 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW2)}, 108 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW3)}, 109 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW4)}, 110 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW5)}, 111 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW6)}, 112 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW7)}, 113 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW8)}, 114 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_HSW9)}, 115 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BWD0)}, 116 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BWD1)}, 117 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BWD2)}, 118 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BWD3)}, 119 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDXDE0)}, 120 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDXDE1)}, 121 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDXDE2)}, 122 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDXDE3)}, 123 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX0)}, 124 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX1)}, 125 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX2)}, 126 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX3)}, 127 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX4)}, 128 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX5)}, 129 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX6)}, 130 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX7)}, 131 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX8)}, 132 {SPDK_IOAT_PCI_DEVICE(PCI_DEVICE_ID_INTEL_IOAT_BDX9)}, 133 { .vendor_id = 0, /* sentinel */ }, 134 }; 135 136 static int 137 spdk_pci_device_init(struct rte_pci_driver *driver, 138 struct rte_pci_device *device) 139 { 140 struct spdk_pci_enum_ctx *ctx = (struct spdk_pci_enum_ctx *)driver; 141 142 if (device->kdrv == RTE_KDRV_VFIO) { 143 /* 144 * TODO: This is a workaround for an issue where the device is not ready after VFIO reset. 145 * Figure out what is actually going wrong and remove this sleep. 146 */ 147 usleep(500 * 1000); 148 } 149 150 return ctx->enum_cb(ctx->enum_ctx, (struct spdk_pci_device *)device); 151 } 152 153 static int 154 spdk_pci_device_fini(struct rte_pci_device *device) 155 { 156 return 0; 157 } 158 159 int 160 spdk_pci_enumerate(enum spdk_pci_device_type type, 161 spdk_pci_enum_cb enum_cb, 162 void *enum_ctx) 163 { 164 struct spdk_pci_enum_ctx ctx = {}; 165 int rc; 166 const char *name; 167 168 if (type == SPDK_PCI_DEVICE_NVME) { 169 name = "SPDK NVMe"; 170 ctx.driver.id_table = nvme_pci_driver_id; 171 } else if (type == SPDK_PCI_DEVICE_IOAT) { 172 name = "SPDK IOAT"; 173 ctx.driver.id_table = ioat_driver_id; 174 } else { 175 return -1; 176 } 177 178 ctx.enum_cb = enum_cb; 179 ctx.enum_ctx = enum_ctx; 180 ctx.driver.drv_flags = RTE_PCI_DRV_NEED_MAPPING; 181 182 #if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) 183 ctx.driver.probe = spdk_pci_device_init; 184 ctx.driver.remove = spdk_pci_device_fini; 185 ctx.driver.driver.name = name; 186 #else 187 ctx.driver.devinit = spdk_pci_device_init; 188 ctx.driver.devuninit = spdk_pci_device_fini; 189 ctx.driver.name = name; 190 #endif 191 192 rte_eal_pci_register(&ctx.driver); 193 rc = rte_eal_pci_probe(); 194 rte_eal_pci_unregister(&ctx.driver); 195 196 return rc; 197 } 198 199 int 200 spdk_pci_device_map_bar(struct spdk_pci_device *device, uint32_t bar, 201 void **mapped_addr, uint64_t *phys_addr, uint64_t *size) 202 { 203 struct rte_pci_device *dev = device; 204 205 *mapped_addr = dev->mem_resource[bar].addr; 206 *phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr; 207 *size = (uint64_t)dev->mem_resource[bar].len; 208 209 return 0; 210 } 211 212 int 213 spdk_pci_device_unmap_bar(struct spdk_pci_device *device, uint32_t bar, void *addr) 214 { 215 return 0; 216 } 217 218 uint16_t 219 spdk_pci_device_get_domain(struct spdk_pci_device *dev) 220 { 221 return dev->addr.domain; 222 } 223 224 uint8_t 225 spdk_pci_device_get_bus(struct spdk_pci_device *dev) 226 { 227 return dev->addr.bus; 228 } 229 230 uint8_t 231 spdk_pci_device_get_dev(struct spdk_pci_device *dev) 232 { 233 return dev->addr.devid; 234 } 235 236 uint8_t 237 spdk_pci_device_get_func(struct spdk_pci_device *dev) 238 { 239 return dev->addr.function; 240 } 241 242 uint16_t 243 spdk_pci_device_get_vendor_id(struct spdk_pci_device *dev) 244 { 245 return dev->id.vendor_id; 246 } 247 248 uint16_t 249 spdk_pci_device_get_device_id(struct spdk_pci_device *dev) 250 { 251 return dev->id.device_id; 252 } 253 254 uint16_t 255 spdk_pci_device_get_subvendor_id(struct spdk_pci_device *dev) 256 { 257 return dev->id.subsystem_vendor_id; 258 } 259 260 uint16_t 261 spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev) 262 { 263 return dev->id.subsystem_device_id; 264 } 265 266 struct spdk_pci_id 267 spdk_pci_device_get_id(struct spdk_pci_device *pci_dev) 268 { 269 struct spdk_pci_id pci_id; 270 271 pci_id.vendor_id = spdk_pci_device_get_vendor_id(pci_dev); 272 pci_id.device_id = spdk_pci_device_get_device_id(pci_dev); 273 pci_id.subvendor_id = spdk_pci_device_get_subvendor_id(pci_dev); 274 pci_id.subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev); 275 276 return pci_id; 277 } 278 279 int 280 spdk_pci_device_cfg_read8(struct spdk_pci_device *dev, uint8_t *value, uint32_t offset) 281 { 282 return rte_eal_pci_read_config(dev, value, 1, offset) == 1 ? 0 : -1; 283 } 284 285 int 286 spdk_pci_device_cfg_write8(struct spdk_pci_device *dev, uint8_t value, uint32_t offset) 287 { 288 return rte_eal_pci_write_config(dev, &value, 1, offset) == 1 ? 0 : -1; 289 } 290 291 int 292 spdk_pci_device_cfg_read16(struct spdk_pci_device *dev, uint16_t *value, uint32_t offset) 293 { 294 return rte_eal_pci_read_config(dev, value, 2, offset) == 2 ? 0 : -1; 295 } 296 297 int 298 spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uint32_t offset) 299 { 300 return rte_eal_pci_write_config(dev, &value, 2, offset) == 2 ? 0 : -1; 301 } 302 303 int 304 spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset) 305 { 306 return rte_eal_pci_read_config(dev, value, 4, offset) == 4 ? 0 : -1; 307 } 308 309 int 310 spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset) 311 { 312 return rte_eal_pci_write_config(dev, &value, 4, offset) == 4 ? 0 : -1; 313 } 314 315 int 316 spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t len) 317 { 318 int err; 319 uint32_t pos, header = 0; 320 uint32_t i, buf[2]; 321 322 if (len < 17) 323 return -1; 324 325 err = spdk_pci_device_cfg_read32(dev, &header, PCI_CFG_SIZE); 326 if (err || !header) 327 return -1; 328 329 pos = PCI_CFG_SIZE; 330 while (1) { 331 if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) { 332 if (pos) { 333 /*skip the header*/ 334 pos += 4; 335 for (i = 0; i < 2; i++) { 336 err = spdk_pci_device_cfg_read32(dev, &buf[i], pos + 4 * i); 337 if (err) 338 return -1; 339 } 340 sprintf(sn, "%08x%08x", buf[1], buf[0]); 341 return 0; 342 } 343 } 344 pos = (header >> 20) & 0xffc; 345 /*0 if no other items exist*/ 346 if (pos < PCI_CFG_SIZE) 347 return -1; 348 err = spdk_pci_device_cfg_read32(dev, &header, pos); 349 if (err) 350 return -1; 351 } 352 return -1; 353 } 354 355 struct spdk_pci_addr 356 spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev) 357 { 358 struct spdk_pci_addr pci_addr; 359 360 pci_addr.domain = spdk_pci_device_get_domain(pci_dev); 361 pci_addr.bus = spdk_pci_device_get_bus(pci_dev); 362 pci_addr.dev = spdk_pci_device_get_dev(pci_dev); 363 pci_addr.func = spdk_pci_device_get_func(pci_dev); 364 365 return pci_addr; 366 } 367 368 int 369 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2) 370 { 371 if (a1->domain > a2->domain) { 372 return 1; 373 } else if (a1->domain < a2->domain) { 374 return -1; 375 } else if (a1->bus > a2->bus) { 376 return 1; 377 } else if (a1->bus < a2->bus) { 378 return -1; 379 } else if (a1->dev > a2->dev) { 380 return 1; 381 } else if (a1->dev < a2->dev) { 382 return -1; 383 } else if (a1->func > a2->func) { 384 return 1; 385 } else if (a1->func < a2->func) { 386 return -1; 387 } 388 389 return 0; 390 } 391 392 #ifdef __linux__ 393 int 394 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr) 395 { 396 int dev_fd; 397 char shm_name[64]; 398 int pid; 399 void *dev_map; 400 struct flock pcidev_lock = { 401 .l_type = F_WRLCK, 402 .l_whence = SEEK_SET, 403 .l_start = 0, 404 .l_len = 0, 405 }; 406 407 sprintf(shm_name, PCI_PRI_FMT, pci_addr->domain, pci_addr->bus, pci_addr->dev, pci_addr->func); 408 409 dev_fd = shm_open(shm_name, O_RDWR | O_CREAT, 0600); 410 if (dev_fd == -1) { 411 fprintf(stderr, "could not shm_open %s\n", shm_name); 412 return -1; 413 } 414 415 if (ftruncate(dev_fd, sizeof(int)) != 0) { 416 fprintf(stderr, "could not truncate shm %s\n", shm_name); 417 close(dev_fd); 418 return -1; 419 } 420 421 dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, 422 MAP_SHARED, dev_fd, 0); 423 if (dev_map == NULL) { 424 fprintf(stderr, "could not mmap shm %s\n", shm_name); 425 close(dev_fd); 426 return -1; 427 } 428 429 if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) { 430 pid = *(int *)dev_map; 431 fprintf(stderr, "Cannot create lock on device %s, probably" 432 " process %d has claimed it\n", shm_name, pid); 433 munmap(dev_map, sizeof(int)); 434 close(dev_fd); 435 return -1; 436 } 437 438 *(int *)dev_map = (int)getpid(); 439 munmap(dev_map, sizeof(int)); 440 /* Keep dev_fd open to maintain the lock. */ 441 return 0; 442 } 443 #endif /* __linux__ */ 444 445 #ifdef __FreeBSD__ 446 int 447 spdk_pci_device_claim(const struct spdk_pci_addr *pci_addr) 448 { 449 /* TODO */ 450 return 0; 451 } 452 #endif /* __FreeBSD__ */ 453 454 int 455 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf) 456 { 457 unsigned domain, bus, dev, func; 458 459 if (addr == NULL || bdf == NULL) { 460 return -EINVAL; 461 } 462 463 if (sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) { 464 /* Matched a full address - all variables are initialized */ 465 } else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) { 466 func = 0; 467 } else if (sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) { 468 domain = 0; 469 } else if (sscanf(bdf, "%x:%x", &bus, &dev) == 2) { 470 domain = 0; 471 func = 0; 472 } else { 473 return -EINVAL; 474 } 475 476 if (domain > 0xFFFF || bus > 0xFF || dev > 0x1F || func > 7) { 477 return -EINVAL; 478 } 479 480 addr->domain = domain; 481 addr->bus = bus; 482 addr->dev = dev; 483 addr->func = func; 484 485 return 0; 486 } 487