1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/env.h" 8 #include "spdk/thread.h" 9 #include "spdk/log.h" 10 #include "spdk/util.h" 11 #include "spdk/memory.h" 12 #include "spdk/cpuset.h" 13 #include "spdk/likely.h" 14 #include "spdk/vfu_target.h" 15 16 #include "tgt_internal.h" 17 18 struct tgt_pci_device_ops { 19 struct spdk_vfu_endpoint_ops ops; 20 TAILQ_ENTRY(tgt_pci_device_ops) link; 21 }; 22 23 static struct spdk_cpuset g_tgt_core_mask; 24 static pthread_mutex_t g_endpoint_lock = PTHREAD_MUTEX_INITIALIZER; 25 static TAILQ_HEAD(, spdk_vfu_endpoint) g_endpoint = TAILQ_HEAD_INITIALIZER(g_endpoint); 26 static TAILQ_HEAD(, tgt_pci_device_ops) g_pci_device_ops = TAILQ_HEAD_INITIALIZER(g_pci_device_ops); 27 static char g_endpoint_path_dirname[PATH_MAX] = ""; 28 29 static struct spdk_vfu_endpoint_ops * 30 tgt_get_pci_device_ops(const char *device_type_name) 31 { 32 struct tgt_pci_device_ops *pci_ops, *tmp; 33 bool exist = false; 34 35 pthread_mutex_lock(&g_endpoint_lock); 36 TAILQ_FOREACH_SAFE(pci_ops, &g_pci_device_ops, link, tmp) { 37 if (!strncmp(device_type_name, pci_ops->ops.name, SPDK_VFU_MAX_NAME_LEN)) { 38 exist = true; 39 break; 40 } 41 } 42 pthread_mutex_unlock(&g_endpoint_lock); 43 44 if (exist) { 45 return &pci_ops->ops; 46 } 47 return NULL; 48 } 49 50 int 51 spdk_vfu_register_endpoint_ops(struct spdk_vfu_endpoint_ops *ops) 52 { 53 struct tgt_pci_device_ops *pci_ops; 54 struct spdk_vfu_endpoint_ops *tmp; 55 56 tmp = tgt_get_pci_device_ops(ops->name); 57 if (tmp) { 58 return -EEXIST; 59 } 60 61 pci_ops = calloc(1, sizeof(*pci_ops)); 62 if (!pci_ops) { 63 return -ENOMEM; 64 } 65 pci_ops->ops = *ops; 66 67 pthread_mutex_lock(&g_endpoint_lock); 68 TAILQ_INSERT_TAIL(&g_pci_device_ops, pci_ops, link); 69 pthread_mutex_unlock(&g_endpoint_lock); 70 71 return 0; 72 } 73 74 static char * 75 tgt_get_base_path(void) 76 { 77 return g_endpoint_path_dirname; 78 } 79 80 int 81 spdk_vfu_set_socket_path(const char *basename) 82 { 83 int ret; 84 85 if (basename && strlen(basename) > 0) { 86 ret = snprintf(g_endpoint_path_dirname, sizeof(g_endpoint_path_dirname) - 2, "%s", basename); 87 if (ret <= 0) { 88 return -EINVAL; 89 } 90 if ((size_t)ret >= sizeof(g_endpoint_path_dirname) - 2) { 91 SPDK_ERRLOG("Char dev dir path length %d is too long\n", ret); 92 return -EINVAL; 93 } 94 95 if (g_endpoint_path_dirname[ret - 1] != '/') { 96 g_endpoint_path_dirname[ret] = '/'; 97 g_endpoint_path_dirname[ret + 1] = '\0'; 98 } 99 } 100 101 return 0; 102 } 103 104 struct spdk_vfu_endpoint * 105 spdk_vfu_get_endpoint_by_name(const char *name) 106 { 107 struct spdk_vfu_endpoint *endpoint, *tmp; 108 bool exist = false; 109 110 pthread_mutex_lock(&g_endpoint_lock); 111 TAILQ_FOREACH_SAFE(endpoint, &g_endpoint, link, tmp) { 112 if (!strncmp(name, endpoint->name, SPDK_VFU_MAX_NAME_LEN)) { 113 exist = true; 114 break; 115 } 116 } 117 pthread_mutex_unlock(&g_endpoint_lock); 118 119 if (exist) { 120 return endpoint; 121 } 122 return NULL; 123 } 124 125 static int 126 tgt_vfu_ctx_poller(void *ctx) 127 { 128 struct spdk_vfu_endpoint *endpoint = ctx; 129 vfu_ctx_t *vfu_ctx = endpoint->vfu_ctx; 130 int ret; 131 132 ret = vfu_run_ctx(vfu_ctx); 133 if (spdk_unlikely(ret == -1)) { 134 if (errno == EBUSY) { 135 return SPDK_POLLER_IDLE; 136 } 137 138 if (errno == ENOTCONN) { 139 spdk_poller_unregister(&endpoint->vfu_ctx_poller); 140 if (endpoint->ops.detach_device) { 141 endpoint->ops.detach_device(endpoint); 142 } 143 endpoint->is_attached = false; 144 return SPDK_POLLER_BUSY; 145 } 146 } 147 148 return ret != 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 149 } 150 151 static int 152 tgt_accept_poller(void *ctx) 153 { 154 struct spdk_vfu_endpoint *endpoint = ctx; 155 int ret; 156 157 if (endpoint->is_attached) { 158 return SPDK_POLLER_IDLE; 159 } 160 161 ret = vfu_attach_ctx(endpoint->vfu_ctx); 162 if (ret == 0) { 163 ret = endpoint->ops.attach_device(endpoint); 164 if (!ret) { 165 SPDK_NOTICELOG("%s: attached successfully\n", spdk_vfu_get_endpoint_id(endpoint)); 166 /* Polling socket too frequently will cause performance issue */ 167 endpoint->vfu_ctx_poller = SPDK_POLLER_REGISTER(tgt_vfu_ctx_poller, endpoint, 1000); 168 endpoint->is_attached = true; 169 } 170 return SPDK_POLLER_BUSY; 171 } 172 173 if (errno == EAGAIN || errno == EWOULDBLOCK) { 174 return SPDK_POLLER_IDLE; 175 } 176 177 return SPDK_POLLER_BUSY; 178 } 179 180 static void 181 tgt_log_cb(vfu_ctx_t *vfu_ctx, int level, char const *msg) 182 { 183 struct spdk_vfu_endpoint *endpoint = vfu_get_private(vfu_ctx); 184 185 if (level >= LOG_DEBUG) { 186 SPDK_DEBUGLOG(vfu, "%s: %s\n", spdk_vfu_get_endpoint_id(endpoint), msg); 187 } else if (level >= LOG_INFO) { 188 SPDK_INFOLOG(vfu, "%s: %s\n", spdk_vfu_get_endpoint_id(endpoint), msg); 189 } else if (level >= LOG_NOTICE) { 190 SPDK_NOTICELOG("%s: %s\n", spdk_vfu_get_endpoint_id(endpoint), msg); 191 } else if (level >= LOG_WARNING) { 192 SPDK_WARNLOG("%s: %s\n", spdk_vfu_get_endpoint_id(endpoint), msg); 193 } else { 194 SPDK_ERRLOG("%s: %s\n", spdk_vfu_get_endpoint_id(endpoint), msg); 195 } 196 } 197 198 static int 199 tgt_get_log_level(void) 200 { 201 int level; 202 203 if (SPDK_DEBUGLOG_FLAG_ENABLED("vfu")) { 204 return LOG_DEBUG; 205 } 206 207 level = spdk_log_to_syslog_level(spdk_log_get_level()); 208 if (level < 0) { 209 return LOG_ERR; 210 } 211 212 return level; 213 } 214 215 static void 216 init_pci_config_space(vfu_pci_config_space_t *p, uint16_t ipin) 217 { 218 /* MLBAR */ 219 p->hdr.bars[0].raw = 0x0; 220 /* MUBAR */ 221 p->hdr.bars[1].raw = 0x0; 222 223 /* vendor specific, let's set them to zero for now */ 224 p->hdr.bars[3].raw = 0x0; 225 p->hdr.bars[4].raw = 0x0; 226 p->hdr.bars[5].raw = 0x0; 227 228 /* enable INTx */ 229 p->hdr.intr.ipin = ipin; 230 } 231 232 static void 233 tgt_memory_region_add_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 234 { 235 struct spdk_vfu_endpoint *endpoint = vfu_get_private(vfu_ctx); 236 void *map_start, *map_end; 237 int ret; 238 239 if (!info->vaddr) { 240 return; 241 } 242 243 map_start = info->mapping.iov_base; 244 map_end = info->mapping.iov_base + info->mapping.iov_len; 245 246 if (((uintptr_t)info->mapping.iov_base & MASK_2MB) || 247 (info->mapping.iov_len & MASK_2MB)) { 248 SPDK_DEBUGLOG(vfu, "Invalid memory region vaddr %p, IOVA %p-%p\n", 249 info->vaddr, map_start, map_end); 250 return; 251 } 252 253 if (info->prot == (PROT_WRITE | PROT_READ)) { 254 ret = spdk_mem_register(info->mapping.iov_base, info->mapping.iov_len); 255 if (ret) { 256 SPDK_ERRLOG("Memory region register %p-%p failed, ret=%d\n", 257 map_start, map_end, ret); 258 } 259 } 260 261 if (endpoint->ops.post_memory_add) { 262 endpoint->ops.post_memory_add(endpoint, map_start, map_end); 263 } 264 } 265 266 static void 267 tgt_memory_region_remove_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 268 { 269 struct spdk_vfu_endpoint *endpoint = vfu_get_private(vfu_ctx); 270 void *map_start, *map_end; 271 int ret = 0; 272 273 if (!info->vaddr) { 274 return; 275 } 276 277 map_start = info->mapping.iov_base; 278 map_end = info->mapping.iov_base + info->mapping.iov_len; 279 280 if (((uintptr_t)info->mapping.iov_base & MASK_2MB) || 281 (info->mapping.iov_len & MASK_2MB)) { 282 SPDK_DEBUGLOG(vfu, "Invalid memory region vaddr %p, IOVA %p-%p\n", 283 info->vaddr, map_start, map_end); 284 return; 285 } 286 287 if (endpoint->ops.pre_memory_remove) { 288 endpoint->ops.pre_memory_remove(endpoint, map_start, map_end); 289 } 290 291 if (info->prot == (PROT_WRITE | PROT_READ)) { 292 ret = spdk_mem_unregister(info->mapping.iov_base, info->mapping.iov_len); 293 if (ret) { 294 SPDK_ERRLOG("Memory region unregister %p-%p failed, ret=%d\n", 295 map_start, map_end, ret); 296 } 297 } 298 } 299 300 static int 301 tgt_device_quiesce_cb(vfu_ctx_t *vfu_ctx) 302 { 303 struct spdk_vfu_endpoint *endpoint = vfu_get_private(vfu_ctx); 304 int ret; 305 306 assert(endpoint->ops.quiesce_device); 307 ret = endpoint->ops.quiesce_device(endpoint); 308 if (ret) { 309 errno = EBUSY; 310 ret = -1; 311 } 312 313 return ret; 314 } 315 316 static int 317 tgt_device_reset_cb(vfu_ctx_t *vfu_ctx, vfu_reset_type_t type) 318 { 319 struct spdk_vfu_endpoint *endpoint = vfu_get_private(vfu_ctx); 320 321 SPDK_DEBUGLOG(vfu, "Device reset type %u\n", type); 322 323 assert(endpoint->ops.reset_device); 324 return endpoint->ops.reset_device(endpoint); 325 } 326 327 static int 328 tgt_endpoint_realize(struct spdk_vfu_endpoint *endpoint) 329 { 330 int ret; 331 uint8_t buf[512]; 332 struct vsc *vendor_cap; 333 ssize_t cap_offset; 334 uint16_t vendor_cap_idx, cap_size, sparse_mmap_idx; 335 struct spdk_vfu_pci_device pci_dev; 336 uint8_t region_idx; 337 338 assert(endpoint->ops.get_device_info); 339 ret = endpoint->ops.get_device_info(endpoint, &pci_dev); 340 if (ret) { 341 SPDK_ERRLOG("%s: failed to get pci device info\n", spdk_vfu_get_endpoint_id(endpoint)); 342 return ret; 343 } 344 345 endpoint->vfu_ctx = vfu_create_ctx(VFU_TRANS_SOCK, endpoint->uuid, LIBVFIO_USER_FLAG_ATTACH_NB, 346 endpoint, VFU_DEV_TYPE_PCI); 347 if (endpoint->vfu_ctx == NULL) { 348 SPDK_ERRLOG("%s: error creating libvfio-user context\n", spdk_vfu_get_endpoint_id(endpoint)); 349 return -EFAULT; 350 } 351 vfu_setup_log(endpoint->vfu_ctx, tgt_log_cb, tgt_get_log_level()); 352 353 ret = vfu_pci_init(endpoint->vfu_ctx, VFU_PCI_TYPE_EXPRESS, PCI_HEADER_TYPE_NORMAL, 0); 354 if (ret < 0) { 355 SPDK_ERRLOG("vfu_ctx %p failed to initialize PCI\n", endpoint->vfu_ctx); 356 goto error; 357 } 358 359 vfu_pci_set_id(endpoint->vfu_ctx, pci_dev.id.vid, pci_dev.id.did, pci_dev.id.ssvid, 360 pci_dev.id.ssid); 361 vfu_pci_set_class(endpoint->vfu_ctx, pci_dev.class.bcc, pci_dev.class.scc, pci_dev.class.pi); 362 363 /* Add Vendor Capabilities */ 364 for (vendor_cap_idx = 0; vendor_cap_idx < pci_dev.nr_vendor_caps; vendor_cap_idx++) { 365 memset(buf, 0, sizeof(buf)); 366 cap_size = endpoint->ops.get_vendor_capability(endpoint, buf, 256, vendor_cap_idx); 367 if (cap_size) { 368 vendor_cap = (struct vsc *)buf; 369 assert(vendor_cap->hdr.id == PCI_CAP_ID_VNDR); 370 assert(vendor_cap->size == cap_size); 371 372 cap_offset = vfu_pci_add_capability(endpoint->vfu_ctx, 0, 0, vendor_cap); 373 if (cap_offset < 0) { 374 SPDK_ERRLOG("vfu_ctx %p failed add vendor capability\n", endpoint->vfu_ctx); 375 ret = -EFAULT; 376 goto error; 377 } 378 } 379 } 380 381 /* Add Standard PCI Capabilities */ 382 cap_offset = vfu_pci_add_capability(endpoint->vfu_ctx, 0, 0, &pci_dev.pmcap); 383 if (cap_offset < 0) { 384 SPDK_ERRLOG("vfu_ctx %p failed add pmcap\n", endpoint->vfu_ctx); 385 ret = -EFAULT; 386 goto error; 387 } 388 SPDK_DEBUGLOG(vfu, "%s PM cap_offset %ld\n", spdk_vfu_get_endpoint_id(endpoint), cap_offset); 389 390 cap_offset = vfu_pci_add_capability(endpoint->vfu_ctx, 0, 0, &pci_dev.pxcap); 391 if (cap_offset < 0) { 392 SPDK_ERRLOG("vfu_ctx %p failed add pxcap\n", endpoint->vfu_ctx); 393 ret = -EFAULT; 394 goto error; 395 } 396 SPDK_DEBUGLOG(vfu, "%s PX cap_offset %ld\n", spdk_vfu_get_endpoint_id(endpoint), cap_offset); 397 398 cap_offset = vfu_pci_add_capability(endpoint->vfu_ctx, 0, 0, &pci_dev.msixcap); 399 if (cap_offset < 0) { 400 SPDK_ERRLOG("vfu_ctx %p failed add msixcap\n", endpoint->vfu_ctx); 401 ret = -EFAULT; 402 goto error; 403 } 404 SPDK_DEBUGLOG(vfu, "%s MSIX cap_offset %ld\n", spdk_vfu_get_endpoint_id(endpoint), cap_offset); 405 406 /* Setup PCI Regions */ 407 for (region_idx = 0; region_idx < VFU_PCI_DEV_NUM_REGIONS; region_idx++) { 408 struct spdk_vfu_pci_region *region = &pci_dev.regions[region_idx]; 409 struct iovec sparse_mmap[SPDK_VFU_MAXIMUM_SPARSE_MMAP_REGIONS]; 410 if (!region->len) { 411 continue; 412 } 413 414 if (region->nr_sparse_mmaps) { 415 assert(region->nr_sparse_mmaps <= SPDK_VFU_MAXIMUM_SPARSE_MMAP_REGIONS); 416 for (sparse_mmap_idx = 0; sparse_mmap_idx < region->nr_sparse_mmaps; sparse_mmap_idx++) { 417 sparse_mmap[sparse_mmap_idx].iov_base = (void *)region->mmaps[sparse_mmap_idx].offset; 418 sparse_mmap[sparse_mmap_idx].iov_len = region->mmaps[sparse_mmap_idx].len; 419 } 420 } 421 422 ret = vfu_setup_region(endpoint->vfu_ctx, region_idx, region->len, region->access_cb, region->flags, 423 region->nr_sparse_mmaps ? sparse_mmap : NULL, region->nr_sparse_mmaps, 424 region->fd, region->offset); 425 if (ret) { 426 SPDK_ERRLOG("vfu_ctx %p failed to setup region %u\n", endpoint->vfu_ctx, region_idx); 427 goto error; 428 } 429 SPDK_DEBUGLOG(vfu, "%s: region %u, len 0x%"PRIx64", callback %p, nr sparse mmaps %u, fd %d\n", 430 spdk_vfu_get_endpoint_id(endpoint), region_idx, region->len, region->access_cb, 431 region->nr_sparse_mmaps, region->fd); 432 } 433 434 ret = vfu_setup_device_dma(endpoint->vfu_ctx, tgt_memory_region_add_cb, 435 tgt_memory_region_remove_cb); 436 if (ret < 0) { 437 SPDK_ERRLOG("vfu_ctx %p failed to setup dma callback\n", endpoint->vfu_ctx); 438 goto error; 439 } 440 441 if (endpoint->ops.reset_device) { 442 ret = vfu_setup_device_reset_cb(endpoint->vfu_ctx, tgt_device_reset_cb); 443 if (ret < 0) { 444 SPDK_ERRLOG("vfu_ctx %p failed to setup reset callback\n", endpoint->vfu_ctx); 445 goto error; 446 } 447 } 448 449 if (endpoint->ops.quiesce_device) { 450 vfu_setup_device_quiesce_cb(endpoint->vfu_ctx, tgt_device_quiesce_cb); 451 } 452 453 ret = vfu_setup_device_nr_irqs(endpoint->vfu_ctx, VFU_DEV_INTX_IRQ, pci_dev.nr_int_irqs); 454 if (ret < 0) { 455 SPDK_ERRLOG("vfu_ctx %p failed to setup INTX\n", endpoint->vfu_ctx); 456 goto error; 457 } 458 459 ret = vfu_setup_device_nr_irqs(endpoint->vfu_ctx, VFU_DEV_MSIX_IRQ, pci_dev.nr_msix_irqs); 460 if (ret < 0) { 461 SPDK_ERRLOG("vfu_ctx %p failed to setup MSIX\n", endpoint->vfu_ctx); 462 goto error; 463 } 464 465 ret = vfu_realize_ctx(endpoint->vfu_ctx); 466 if (ret < 0) { 467 SPDK_ERRLOG("vfu_ctx %p failed to realize\n", endpoint->vfu_ctx); 468 goto error; 469 } 470 471 endpoint->pci_config_space = vfu_pci_get_config_space(endpoint->vfu_ctx); 472 assert(endpoint->pci_config_space != NULL); 473 init_pci_config_space(endpoint->pci_config_space, pci_dev.intr_ipin); 474 475 assert(cap_offset != 0); 476 endpoint->msix = (struct msixcap *)((uint8_t *)endpoint->pci_config_space + cap_offset); 477 478 return 0; 479 480 error: 481 if (endpoint->vfu_ctx) { 482 vfu_destroy_ctx(endpoint->vfu_ctx); 483 } 484 return ret; 485 } 486 487 static int 488 vfu_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask) 489 { 490 int rc; 491 struct spdk_cpuset negative_vfu_mask; 492 493 if (cpumask == NULL) { 494 return -1; 495 } 496 497 if (mask == NULL) { 498 spdk_cpuset_copy(cpumask, &g_tgt_core_mask); 499 return 0; 500 } 501 502 rc = spdk_cpuset_parse(cpumask, mask); 503 if (rc < 0) { 504 SPDK_ERRLOG("invalid cpumask %s\n", mask); 505 return -1; 506 } 507 508 spdk_cpuset_copy(&negative_vfu_mask, &g_tgt_core_mask); 509 spdk_cpuset_negate(&negative_vfu_mask); 510 spdk_cpuset_and(&negative_vfu_mask, cpumask); 511 512 if (spdk_cpuset_count(&negative_vfu_mask) != 0) { 513 SPDK_ERRLOG("one of selected cpu is outside of core mask(=%s)\n", 514 spdk_cpuset_fmt(&g_tgt_core_mask)); 515 return -1; 516 } 517 518 spdk_cpuset_and(cpumask, &g_tgt_core_mask); 519 520 if (spdk_cpuset_count(cpumask) == 0) { 521 SPDK_ERRLOG("no cpu is selected among core mask(=%s)\n", 522 spdk_cpuset_fmt(&g_tgt_core_mask)); 523 return -1; 524 } 525 526 return 0; 527 } 528 529 static void 530 tgt_endpoint_start_thread(void *arg1) 531 { 532 struct spdk_vfu_endpoint *endpoint = arg1; 533 534 endpoint->accept_poller = SPDK_POLLER_REGISTER(tgt_accept_poller, endpoint, 1000); 535 assert(endpoint->accept_poller != NULL); 536 } 537 538 static void 539 tgt_endpoint_thread_exit(void *arg1) 540 { 541 struct spdk_vfu_endpoint *endpoint = arg1; 542 543 spdk_poller_unregister(&endpoint->accept_poller); 544 spdk_poller_unregister(&endpoint->vfu_ctx_poller); 545 546 /* Ensure the attached device is stopped before destorying the vfu context */ 547 if (endpoint->ops.detach_device) { 548 endpoint->ops.detach_device(endpoint); 549 } 550 551 if (endpoint->vfu_ctx) { 552 vfu_destroy_ctx(endpoint->vfu_ctx); 553 } 554 555 endpoint->ops.destruct(endpoint); 556 free(endpoint); 557 558 spdk_thread_exit(spdk_get_thread()); 559 } 560 561 int 562 spdk_vfu_create_endpoint(const char *endpoint_name, const char *cpumask_str, 563 const char *dev_type_name) 564 { 565 char *basename; 566 char uuid[PATH_MAX] = ""; 567 struct spdk_cpuset cpumask = {}; 568 struct spdk_vfu_endpoint *endpoint; 569 struct spdk_vfu_endpoint_ops *ops; 570 int ret = 0; 571 572 ret = vfu_parse_core_mask(cpumask_str, &cpumask); 573 if (ret) { 574 return ret; 575 } 576 577 if (strlen(endpoint_name) >= SPDK_VFU_MAX_NAME_LEN - 1) { 578 return -ENAMETOOLONG; 579 } 580 581 if (spdk_vfu_get_endpoint_by_name(endpoint_name)) { 582 SPDK_ERRLOG("%s already exist\n", endpoint_name); 583 return -EEXIST; 584 } 585 586 /* Find supported PCI device type */ 587 ops = tgt_get_pci_device_ops(dev_type_name); 588 if (!ops) { 589 SPDK_ERRLOG("Request %s device type isn't registered\n", dev_type_name); 590 return -ENOTSUP; 591 } 592 593 basename = tgt_get_base_path(); 594 if (snprintf(uuid, sizeof(uuid), "%s%s", basename, endpoint_name) >= (int)sizeof(uuid)) { 595 SPDK_ERRLOG("Resulting socket path for endpoint %s is too long: %s%s\n", 596 endpoint_name, basename, endpoint_name); 597 return -EINVAL; 598 } 599 600 endpoint = calloc(1, sizeof(*endpoint)); 601 if (!endpoint) { 602 return -ENOMEM; 603 } 604 605 endpoint->endpoint_ctx = ops->init(endpoint, basename, endpoint_name); 606 if (!endpoint->endpoint_ctx) { 607 free(endpoint); 608 return -EINVAL; 609 } 610 endpoint->ops = *ops; 611 snprintf(endpoint->name, SPDK_VFU_MAX_NAME_LEN, "%s", endpoint_name); 612 snprintf(endpoint->uuid, sizeof(uuid), "%s", uuid); 613 614 SPDK_DEBUGLOG(vfu, "Construct endpoint %s\n", endpoint_name); 615 /* Endpoint realize */ 616 ret = tgt_endpoint_realize(endpoint); 617 if (ret) { 618 endpoint->ops.destruct(endpoint); 619 free(endpoint); 620 return ret; 621 } 622 623 endpoint->thread = spdk_thread_create(endpoint_name, &cpumask); 624 if (!endpoint->thread) { 625 endpoint->ops.destruct(endpoint); 626 vfu_destroy_ctx(endpoint->vfu_ctx); 627 free(endpoint); 628 return -EFAULT; 629 } 630 631 pthread_mutex_lock(&g_endpoint_lock); 632 TAILQ_INSERT_TAIL(&g_endpoint, endpoint, link); 633 pthread_mutex_unlock(&g_endpoint_lock); 634 635 spdk_thread_send_msg(endpoint->thread, tgt_endpoint_start_thread, endpoint); 636 637 return 0; 638 } 639 640 int 641 spdk_vfu_delete_endpoint(const char *endpoint_name) 642 { 643 struct spdk_vfu_endpoint *endpoint; 644 645 endpoint = spdk_vfu_get_endpoint_by_name(endpoint_name); 646 if (!endpoint) { 647 SPDK_ERRLOG("%s doesn't exist\n", endpoint_name); 648 return -ENOENT; 649 } 650 651 SPDK_NOTICELOG("Destruct endpoint %s\n", endpoint_name); 652 653 pthread_mutex_lock(&g_endpoint_lock); 654 TAILQ_REMOVE(&g_endpoint, endpoint, link); 655 pthread_mutex_unlock(&g_endpoint_lock); 656 spdk_thread_send_msg(endpoint->thread, tgt_endpoint_thread_exit, endpoint); 657 658 return 0; 659 } 660 661 const char * 662 spdk_vfu_get_endpoint_id(struct spdk_vfu_endpoint *endpoint) 663 { 664 return endpoint->uuid; 665 } 666 667 const char * 668 spdk_vfu_get_endpoint_name(struct spdk_vfu_endpoint *endpoint) 669 { 670 return endpoint->name; 671 } 672 673 vfu_ctx_t * 674 spdk_vfu_get_vfu_ctx(struct spdk_vfu_endpoint *endpoint) 675 { 676 return endpoint->vfu_ctx; 677 } 678 679 void * 680 spdk_vfu_get_endpoint_private(struct spdk_vfu_endpoint *endpoint) 681 { 682 return endpoint->endpoint_ctx; 683 } 684 685 bool 686 spdk_vfu_endpoint_msix_enabled(struct spdk_vfu_endpoint *endpoint) 687 { 688 return endpoint->msix->mxc.mxe; 689 } 690 691 bool 692 spdk_vfu_endpoint_intx_enabled(struct spdk_vfu_endpoint *endpoint) 693 { 694 return !endpoint->pci_config_space->hdr.cmd.id; 695 } 696 697 void * 698 spdk_vfu_endpoint_get_pci_config(struct spdk_vfu_endpoint *endpoint) 699 { 700 return (void *)endpoint->pci_config_space; 701 } 702 703 void 704 spdk_vfu_init(spdk_vfu_init_cb init_cb) 705 { 706 uint32_t i; 707 size_t len; 708 709 if (g_endpoint_path_dirname[0] == '\0') { 710 if (getcwd(g_endpoint_path_dirname, sizeof(g_endpoint_path_dirname) - 2) == NULL) { 711 SPDK_ERRLOG("getcwd failed\n"); 712 return; 713 } 714 715 len = strlen(g_endpoint_path_dirname); 716 if (g_endpoint_path_dirname[len - 1] != '/') { 717 g_endpoint_path_dirname[len] = '/'; 718 g_endpoint_path_dirname[len + 1] = '\0'; 719 } 720 } 721 722 spdk_cpuset_zero(&g_tgt_core_mask); 723 SPDK_ENV_FOREACH_CORE(i) { 724 spdk_cpuset_set_cpu(&g_tgt_core_mask, i, true); 725 } 726 727 init_cb(0); 728 } 729 730 void * 731 spdk_vfu_map_one(struct spdk_vfu_endpoint *endpoint, uint64_t addr, uint64_t len, dma_sg_t *sg, 732 struct iovec *iov, 733 int prot) 734 { 735 int ret; 736 737 assert(endpoint != NULL); 738 assert(endpoint->vfu_ctx != NULL); 739 assert(sg != NULL); 740 assert(iov != NULL); 741 742 ret = vfu_addr_to_sgl(endpoint->vfu_ctx, (void *)(uintptr_t)addr, len, sg, 1, prot); 743 if (ret < 0) { 744 return NULL; 745 } 746 747 ret = vfu_sgl_get(endpoint->vfu_ctx, sg, iov, 1, 0); 748 if (ret != 0) { 749 return NULL; 750 } 751 752 assert(iov->iov_base != NULL); 753 return iov->iov_base; 754 } 755 756 void 757 spdk_vfu_unmap_sg(struct spdk_vfu_endpoint *endpoint, dma_sg_t *sg, struct iovec *iov, int iovcnt) 758 { 759 assert(endpoint != NULL); 760 assert(endpoint->vfu_ctx != NULL); 761 assert(sg != NULL); 762 assert(iov != NULL); 763 764 vfu_sgl_put(endpoint->vfu_ctx, sg, iov, iovcnt); 765 } 766 767 void 768 spdk_vfu_fini(spdk_vfu_fini_cb fini_cb) 769 { 770 struct spdk_vfu_endpoint *endpoint, *tmp; 771 struct tgt_pci_device_ops *ops, *ops_tmp; 772 773 pthread_mutex_lock(&g_endpoint_lock); 774 TAILQ_FOREACH_SAFE(ops, &g_pci_device_ops, link, ops_tmp) { 775 TAILQ_REMOVE(&g_pci_device_ops, ops, link); 776 free(ops); 777 } 778 779 TAILQ_FOREACH_SAFE(endpoint, &g_endpoint, link, tmp) { 780 TAILQ_REMOVE(&g_endpoint, endpoint, link); 781 spdk_thread_send_msg(endpoint->thread, tgt_endpoint_thread_exit, endpoint); 782 } 783 pthread_mutex_unlock(&g_endpoint_lock); 784 785 fini_cb(); 786 } 787 SPDK_LOG_REGISTER_COMPONENT(vfu) 788