1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/env.h" 37 #include "spdk/likely.h" 38 #include "spdk/string.h" 39 #include "spdk/util.h" 40 #include "spdk/barrier.h" 41 #include "spdk/vhost.h" 42 #include "vhost_internal.h" 43 44 #include "spdk_internal/memory.h" 45 46 struct vhost_poll_group { 47 struct spdk_thread *thread; 48 unsigned ref; 49 TAILQ_ENTRY(vhost_poll_group) tailq; 50 }; 51 52 static TAILQ_HEAD(, vhost_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); 53 54 /* Temporary cpuset for poll group assignment */ 55 static struct spdk_cpuset *g_tmp_cpuset; 56 57 /* Path to folder where character device will be created. Can be set by user. */ 58 static char dev_dirname[PATH_MAX] = ""; 59 60 /* Thread performing all vhost management operations */ 61 static struct spdk_thread *g_vhost_init_thread; 62 63 static spdk_vhost_fini_cb g_fini_cpl_cb; 64 65 /** 66 * DPDK calls our callbacks synchronously but the work those callbacks 67 * perform needs to be async. Luckily, all DPDK callbacks are called on 68 * a DPDK-internal pthread, so we'll just wait on a semaphore in there. 69 */ 70 static sem_t g_dpdk_sem; 71 72 /** Return code for the current DPDK callback */ 73 static int g_dpdk_response; 74 75 struct spdk_vhost_session_fn_ctx { 76 /** Device pointer obtained before enqueuing the event */ 77 struct spdk_vhost_dev *vdev; 78 79 /** ID of the session to send event to. */ 80 uint32_t vsession_id; 81 82 /** User callback function to be executed on given thread. */ 83 spdk_vhost_session_fn cb_fn; 84 85 /** Custom user context */ 86 void *user_ctx; 87 }; 88 89 static int new_connection(int vid); 90 static int start_device(int vid); 91 static void stop_device(int vid); 92 static void destroy_connection(int vid); 93 94 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 95 static int get_config(int vid, uint8_t *config, uint32_t len); 96 static int set_config(int vid, uint8_t *config, uint32_t offset, 97 uint32_t size, uint32_t flags); 98 #endif 99 100 const struct vhost_device_ops g_spdk_vhost_ops = { 101 .new_device = start_device, 102 .destroy_device = stop_device, 103 .new_connection = new_connection, 104 .destroy_connection = destroy_connection, 105 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 106 .get_config = get_config, 107 .set_config = set_config, 108 .vhost_nvme_admin_passthrough = spdk_vhost_nvme_admin_passthrough, 109 .vhost_nvme_set_cq_call = spdk_vhost_nvme_set_cq_call, 110 .vhost_nvme_get_cap = spdk_vhost_nvme_get_cap, 111 .vhost_nvme_set_bar_mr = spdk_vhost_nvme_set_bar_mr, 112 #endif 113 }; 114 115 static TAILQ_HEAD(, spdk_vhost_dev) g_spdk_vhost_devices = TAILQ_HEAD_INITIALIZER( 116 g_spdk_vhost_devices); 117 static pthread_mutex_t g_spdk_vhost_mutex = PTHREAD_MUTEX_INITIALIZER; 118 119 void *spdk_vhost_gpa_to_vva(struct spdk_vhost_session *vsession, uint64_t addr, uint64_t len) 120 { 121 void *vva; 122 uint64_t newlen; 123 124 newlen = len; 125 vva = (void *)rte_vhost_va_from_guest_pa(vsession->mem, addr, &newlen); 126 if (newlen != len) { 127 return NULL; 128 } 129 130 return vva; 131 132 } 133 134 static void 135 spdk_vhost_log_req_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, 136 uint16_t req_id) 137 { 138 struct vring_desc *desc, *desc_table; 139 uint32_t desc_table_size; 140 int rc; 141 142 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 143 return; 144 } 145 146 rc = spdk_vhost_vq_get_desc(vsession, virtqueue, req_id, &desc, &desc_table, &desc_table_size); 147 if (spdk_unlikely(rc != 0)) { 148 SPDK_ERRLOG("Can't log used ring descriptors!\n"); 149 return; 150 } 151 152 do { 153 if (spdk_vhost_vring_desc_is_wr(desc)) { 154 /* To be honest, only pages realy touched should be logged, but 155 * doing so would require tracking those changes in each backed. 156 * Also backend most likely will touch all/most of those pages so 157 * for lets assume we touched all pages passed to as writeable buffers. */ 158 rte_vhost_log_write(vsession->vid, desc->addr, desc->len); 159 } 160 spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); 161 } while (desc); 162 } 163 164 static void 165 spdk_vhost_log_used_vring_elem(struct spdk_vhost_session *vsession, 166 struct spdk_vhost_virtqueue *virtqueue, 167 uint16_t idx) 168 { 169 uint64_t offset, len; 170 uint16_t vq_idx; 171 172 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 173 return; 174 } 175 176 offset = offsetof(struct vring_used, ring[idx]); 177 len = sizeof(virtqueue->vring.used->ring[idx]); 178 vq_idx = virtqueue - vsession->virtqueue; 179 180 rte_vhost_log_used_vring(vsession->vid, vq_idx, offset, len); 181 } 182 183 static void 184 spdk_vhost_log_used_vring_idx(struct spdk_vhost_session *vsession, 185 struct spdk_vhost_virtqueue *virtqueue) 186 { 187 uint64_t offset, len; 188 uint16_t vq_idx; 189 190 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 191 return; 192 } 193 194 offset = offsetof(struct vring_used, idx); 195 len = sizeof(virtqueue->vring.used->idx); 196 vq_idx = virtqueue - vsession->virtqueue; 197 198 rte_vhost_log_used_vring(vsession->vid, vq_idx, offset, len); 199 } 200 201 /* 202 * Get available requests from avail ring. 203 */ 204 uint16_t 205 spdk_vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *virtqueue, uint16_t *reqs, 206 uint16_t reqs_len) 207 { 208 struct rte_vhost_vring *vring = &virtqueue->vring; 209 struct vring_avail *avail = vring->avail; 210 uint16_t size_mask = vring->size - 1; 211 uint16_t last_idx = virtqueue->last_avail_idx, avail_idx = avail->idx; 212 uint16_t count, i; 213 214 count = avail_idx - last_idx; 215 if (spdk_likely(count == 0)) { 216 return 0; 217 } 218 219 if (spdk_unlikely(count > vring->size)) { 220 /* TODO: the queue is unrecoverably broken and should be marked so. 221 * For now we will fail silently and report there are no new avail entries. 222 */ 223 return 0; 224 } 225 226 count = spdk_min(count, reqs_len); 227 virtqueue->last_avail_idx += count; 228 for (i = 0; i < count; i++) { 229 reqs[i] = vring->avail->ring[(last_idx + i) & size_mask]; 230 } 231 232 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 233 "AVAIL: last_idx=%"PRIu16" avail_idx=%"PRIu16" count=%"PRIu16"\n", 234 last_idx, avail_idx, count); 235 236 return count; 237 } 238 239 static bool 240 spdk_vhost_vring_desc_is_indirect(struct vring_desc *cur_desc) 241 { 242 return !!(cur_desc->flags & VRING_DESC_F_INDIRECT); 243 } 244 245 int 246 spdk_vhost_vq_get_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, 247 uint16_t req_idx, struct vring_desc **desc, struct vring_desc **desc_table, 248 uint32_t *desc_table_size) 249 { 250 if (spdk_unlikely(req_idx >= virtqueue->vring.size)) { 251 return -1; 252 } 253 254 *desc = &virtqueue->vring.desc[req_idx]; 255 256 if (spdk_vhost_vring_desc_is_indirect(*desc)) { 257 *desc_table_size = (*desc)->len / sizeof(**desc); 258 *desc_table = spdk_vhost_gpa_to_vva(vsession, (*desc)->addr, 259 sizeof(**desc) * *desc_table_size); 260 *desc = *desc_table; 261 if (*desc == NULL) { 262 return -1; 263 } 264 265 return 0; 266 } 267 268 *desc_table = virtqueue->vring.desc; 269 *desc_table_size = virtqueue->vring.size; 270 271 return 0; 272 } 273 274 int 275 spdk_vhost_vq_used_signal(struct spdk_vhost_session *vsession, 276 struct spdk_vhost_virtqueue *virtqueue) 277 { 278 if (virtqueue->used_req_cnt == 0) { 279 return 0; 280 } 281 282 virtqueue->req_cnt += virtqueue->used_req_cnt; 283 virtqueue->used_req_cnt = 0; 284 285 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 286 "Queue %td - USED RING: sending IRQ: last used %"PRIu16"\n", 287 virtqueue - vsession->virtqueue, virtqueue->last_used_idx); 288 289 if (rte_vhost_vring_call(vsession->vid, virtqueue->vring_idx) == 0) { 290 /* interrupt signalled */ 291 return 1; 292 } else { 293 /* interrupt not signalled */ 294 return 0; 295 } 296 } 297 298 299 static void 300 check_session_io_stats(struct spdk_vhost_session *vsession, uint64_t now) 301 { 302 struct spdk_vhost_virtqueue *virtqueue; 303 uint32_t irq_delay_base = vsession->coalescing_delay_time_base; 304 uint32_t io_threshold = vsession->coalescing_io_rate_threshold; 305 int32_t irq_delay; 306 uint32_t req_cnt; 307 uint16_t q_idx; 308 309 if (now < vsession->next_stats_check_time) { 310 return; 311 } 312 313 vsession->next_stats_check_time = now + vsession->stats_check_interval; 314 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 315 virtqueue = &vsession->virtqueue[q_idx]; 316 317 req_cnt = virtqueue->req_cnt + virtqueue->used_req_cnt; 318 if (req_cnt <= io_threshold) { 319 continue; 320 } 321 322 irq_delay = (irq_delay_base * (req_cnt - io_threshold)) / io_threshold; 323 virtqueue->irq_delay_time = (uint32_t) spdk_max(0, irq_delay); 324 325 virtqueue->req_cnt = 0; 326 virtqueue->next_event_time = now; 327 } 328 } 329 330 void 331 spdk_vhost_session_used_signal(struct spdk_vhost_session *vsession) 332 { 333 struct spdk_vhost_virtqueue *virtqueue; 334 uint64_t now; 335 uint16_t q_idx; 336 337 if (vsession->coalescing_delay_time_base == 0) { 338 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 339 virtqueue = &vsession->virtqueue[q_idx]; 340 341 if (virtqueue->vring.desc == NULL || 342 (virtqueue->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 343 continue; 344 } 345 346 spdk_vhost_vq_used_signal(vsession, virtqueue); 347 } 348 } else { 349 now = spdk_get_ticks(); 350 check_session_io_stats(vsession, now); 351 352 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 353 virtqueue = &vsession->virtqueue[q_idx]; 354 355 /* No need for event right now */ 356 if (now < virtqueue->next_event_time || 357 (virtqueue->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 358 continue; 359 } 360 361 if (!spdk_vhost_vq_used_signal(vsession, virtqueue)) { 362 continue; 363 } 364 365 /* Syscall is quite long so update time */ 366 now = spdk_get_ticks(); 367 virtqueue->next_event_time = now + virtqueue->irq_delay_time; 368 } 369 } 370 } 371 372 static int 373 spdk_vhost_session_set_coalescing(struct spdk_vhost_dev *vdev, 374 struct spdk_vhost_session *vsession, void *ctx) 375 { 376 if (vdev == NULL || vsession == NULL) { 377 /* nothing to do */ 378 return 0; 379 } 380 381 vsession->coalescing_delay_time_base = 382 vdev->coalescing_delay_us * spdk_get_ticks_hz() / 1000000ULL; 383 vsession->coalescing_io_rate_threshold = 384 vdev->coalescing_iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; 385 return 0; 386 } 387 388 int 389 spdk_vhost_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us, 390 uint32_t iops_threshold) 391 { 392 uint64_t delay_time_base = delay_base_us * spdk_get_ticks_hz() / 1000000ULL; 393 uint32_t io_rate = iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; 394 395 if (delay_time_base >= UINT32_MAX) { 396 SPDK_ERRLOG("Delay time of %"PRIu32" is to big\n", delay_base_us); 397 return -EINVAL; 398 } else if (io_rate == 0) { 399 SPDK_ERRLOG("IOPS rate of %"PRIu32" is too low. Min is %u\n", io_rate, 400 1000U / SPDK_VHOST_STATS_CHECK_INTERVAL_MS); 401 return -EINVAL; 402 } 403 404 vdev->coalescing_delay_us = delay_base_us; 405 vdev->coalescing_iops_threshold = iops_threshold; 406 407 spdk_vhost_dev_foreach_session(vdev, spdk_vhost_session_set_coalescing, NULL); 408 return 0; 409 } 410 411 void 412 spdk_vhost_get_coalescing(struct spdk_vhost_dev *vdev, uint32_t *delay_base_us, 413 uint32_t *iops_threshold) 414 { 415 if (delay_base_us) { 416 *delay_base_us = vdev->coalescing_delay_us; 417 } 418 419 if (iops_threshold) { 420 *iops_threshold = vdev->coalescing_iops_threshold; 421 } 422 } 423 424 /* 425 * Enqueue id and len to used ring. 426 */ 427 void 428 spdk_vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, 429 struct spdk_vhost_virtqueue *virtqueue, 430 uint16_t id, uint32_t len) 431 { 432 struct rte_vhost_vring *vring = &virtqueue->vring; 433 struct vring_used *used = vring->used; 434 uint16_t last_idx = virtqueue->last_used_idx & (vring->size - 1); 435 436 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 437 "Queue %td - USED RING: last_idx=%"PRIu16" req id=%"PRIu16" len=%"PRIu32"\n", 438 virtqueue - vsession->virtqueue, virtqueue->last_used_idx, id, len); 439 440 spdk_vhost_log_req_desc(vsession, virtqueue, id); 441 442 virtqueue->last_used_idx++; 443 used->ring[last_idx].id = id; 444 used->ring[last_idx].len = len; 445 446 /* Ensure the used ring is updated before we log it or increment used->idx. */ 447 spdk_smp_wmb(); 448 449 spdk_vhost_log_used_vring_elem(vsession, virtqueue, last_idx); 450 * (volatile uint16_t *) &used->idx = virtqueue->last_used_idx; 451 spdk_vhost_log_used_vring_idx(vsession, virtqueue); 452 453 /* Ensure all our used ring changes are visible to the guest at the time 454 * of interrupt. 455 * TODO: this is currently an sfence on x86. For other architectures we 456 * will most likely need an smp_mb(), but smp_mb() is an overkill for x86. 457 */ 458 spdk_wmb(); 459 460 virtqueue->used_req_cnt++; 461 } 462 463 int 464 spdk_vhost_vring_desc_get_next(struct vring_desc **desc, 465 struct vring_desc *desc_table, uint32_t desc_table_size) 466 { 467 struct vring_desc *old_desc = *desc; 468 uint16_t next_idx; 469 470 if ((old_desc->flags & VRING_DESC_F_NEXT) == 0) { 471 *desc = NULL; 472 return 0; 473 } 474 475 next_idx = old_desc->next; 476 if (spdk_unlikely(next_idx >= desc_table_size)) { 477 *desc = NULL; 478 return -1; 479 } 480 481 *desc = &desc_table[next_idx]; 482 return 0; 483 } 484 485 bool 486 spdk_vhost_vring_desc_is_wr(struct vring_desc *cur_desc) 487 { 488 return !!(cur_desc->flags & VRING_DESC_F_WRITE); 489 } 490 491 int 492 spdk_vhost_vring_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, 493 uint16_t *iov_index, const struct vring_desc *desc) 494 { 495 uint64_t len; 496 uint64_t remaining = desc->len; 497 uintptr_t payload = desc->addr; 498 uintptr_t vva; 499 500 do { 501 if (*iov_index >= SPDK_VHOST_IOVS_MAX) { 502 SPDK_ERRLOG("SPDK_VHOST_IOVS_MAX(%d) reached\n", SPDK_VHOST_IOVS_MAX); 503 return -1; 504 } 505 len = remaining; 506 vva = (uintptr_t)rte_vhost_va_from_guest_pa(vsession->mem, payload, &len); 507 if (vva == 0 || len == 0) { 508 SPDK_ERRLOG("gpa_to_vva(%p) == NULL\n", (void *)payload); 509 return -1; 510 } 511 iov[*iov_index].iov_base = (void *)vva; 512 iov[*iov_index].iov_len = len; 513 remaining -= len; 514 payload += len; 515 (*iov_index)++; 516 } while (remaining); 517 518 return 0; 519 } 520 521 static struct spdk_vhost_session * 522 spdk_vhost_session_find_by_id(struct spdk_vhost_dev *vdev, unsigned id) 523 { 524 struct spdk_vhost_session *vsession; 525 526 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 527 if (vsession->id == id) { 528 return vsession; 529 } 530 } 531 532 return NULL; 533 } 534 535 struct spdk_vhost_session * 536 spdk_vhost_session_find_by_vid(int vid) 537 { 538 struct spdk_vhost_dev *vdev; 539 struct spdk_vhost_session *vsession; 540 541 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 542 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 543 if (vsession->vid == vid) { 544 return vsession; 545 } 546 } 547 } 548 549 return NULL; 550 } 551 552 #define SHIFT_2MB 21 553 #define SIZE_2MB (1ULL << SHIFT_2MB) 554 #define FLOOR_2MB(x) (((uintptr_t)x) / SIZE_2MB) << SHIFT_2MB 555 #define CEIL_2MB(x) ((((uintptr_t)x) + SIZE_2MB - 1) / SIZE_2MB) << SHIFT_2MB 556 557 static void 558 spdk_vhost_session_mem_register(struct spdk_vhost_session *vsession) 559 { 560 struct rte_vhost_mem_region *region; 561 uint32_t i; 562 uint64_t previous_start = UINT64_MAX; 563 564 for (i = 0; i < vsession->mem->nregions; i++) { 565 uint64_t start, end, len; 566 region = &vsession->mem->regions[i]; 567 start = FLOOR_2MB(region->mmap_addr); 568 end = CEIL_2MB(region->mmap_addr + region->mmap_size); 569 if (start == previous_start) { 570 start += (size_t) SIZE_2MB; 571 } 572 previous_start = start; 573 len = end - start; 574 SPDK_INFOLOG(SPDK_LOG_VHOST, "Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n", 575 start, len); 576 577 if (spdk_mem_register((void *)start, len) != 0) { 578 SPDK_WARNLOG("Failed to register memory region %"PRIu32". Future vtophys translation might fail.\n", 579 i); 580 continue; 581 } 582 } 583 } 584 585 static void 586 spdk_vhost_session_mem_unregister(struct spdk_vhost_session *vsession) 587 { 588 struct rte_vhost_mem_region *region; 589 uint32_t i; 590 uint64_t previous_start = UINT64_MAX; 591 592 for (i = 0; i < vsession->mem->nregions; i++) { 593 uint64_t start, end, len; 594 region = &vsession->mem->regions[i]; 595 start = FLOOR_2MB(region->mmap_addr); 596 end = CEIL_2MB(region->mmap_addr + region->mmap_size); 597 if (start == previous_start) { 598 start += (size_t) SIZE_2MB; 599 } 600 previous_start = start; 601 len = end - start; 602 603 if (spdk_vtophys((void *) start, NULL) == SPDK_VTOPHYS_ERROR) { 604 continue; /* region has not been registered */ 605 } 606 607 if (spdk_mem_unregister((void *)start, len) != 0) { 608 assert(false); 609 } 610 } 611 612 } 613 614 struct spdk_vhost_dev * 615 spdk_vhost_dev_next(struct spdk_vhost_dev *vdev) 616 { 617 if (vdev == NULL) { 618 return TAILQ_FIRST(&g_spdk_vhost_devices); 619 } 620 621 return TAILQ_NEXT(vdev, tailq); 622 } 623 624 struct spdk_vhost_dev * 625 spdk_vhost_dev_find(const char *ctrlr_name) 626 { 627 struct spdk_vhost_dev *vdev; 628 size_t dev_dirname_len = strlen(dev_dirname); 629 630 if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) { 631 ctrlr_name += dev_dirname_len; 632 } 633 634 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 635 if (strcmp(vdev->name, ctrlr_name) == 0) { 636 return vdev; 637 } 638 } 639 640 return NULL; 641 } 642 643 static int 644 spdk_vhost_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask) 645 { 646 int rc; 647 648 if (cpumask == NULL) { 649 return -1; 650 } 651 652 if (mask == NULL) { 653 spdk_cpuset_copy(cpumask, spdk_app_get_core_mask()); 654 return 0; 655 } 656 657 rc = spdk_app_parse_core_mask(mask, cpumask); 658 if (rc < 0) { 659 SPDK_ERRLOG("invalid cpumask %s\n", mask); 660 return -1; 661 } 662 663 if (spdk_cpuset_count(cpumask) == 0) { 664 SPDK_ERRLOG("no cpu is selected among reactor mask(=%s)\n", 665 spdk_cpuset_fmt(spdk_app_get_core_mask())); 666 return -1; 667 } 668 669 return 0; 670 } 671 672 static void * 673 _start_rte_driver(void *arg) 674 { 675 char *path = arg; 676 677 if (rte_vhost_driver_start(path) != 0) { 678 return NULL; 679 } 680 681 return path; 682 } 683 684 int 685 spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, 686 const struct spdk_vhost_dev_backend *backend) 687 { 688 char path[PATH_MAX]; 689 struct stat file_stat; 690 struct spdk_cpuset *cpumask; 691 int rc; 692 693 assert(vdev); 694 if (name == NULL) { 695 SPDK_ERRLOG("Can't register controller with no name\n"); 696 return -EINVAL; 697 } 698 699 cpumask = spdk_cpuset_alloc(); 700 if (!cpumask) { 701 SPDK_ERRLOG("spdk_cpuset_alloc failed\n"); 702 return -ENOMEM; 703 } 704 705 if (spdk_vhost_parse_core_mask(mask_str, cpumask) != 0) { 706 SPDK_ERRLOG("cpumask %s is invalid (app mask is 0x%s)\n", 707 mask_str, spdk_cpuset_fmt(spdk_app_get_core_mask())); 708 rc = -EINVAL; 709 goto out; 710 } 711 712 if (spdk_vhost_dev_find(name)) { 713 SPDK_ERRLOG("vhost controller %s already exists.\n", name); 714 rc = -EEXIST; 715 goto out; 716 } 717 718 if (snprintf(path, sizeof(path), "%s%s", dev_dirname, name) >= (int)sizeof(path)) { 719 SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", name, dev_dirname, 720 name); 721 rc = -EINVAL; 722 goto out; 723 } 724 725 /* Register vhost driver to handle vhost messages. */ 726 if (stat(path, &file_stat) != -1) { 727 if (!S_ISSOCK(file_stat.st_mode)) { 728 SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " 729 "The file already exists and is not a socket.\n", 730 path); 731 rc = -EIO; 732 goto out; 733 } else if (unlink(path) != 0) { 734 SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " 735 "The socket already exists and failed to unlink.\n", 736 path); 737 rc = -EIO; 738 goto out; 739 } 740 } 741 742 if (rte_vhost_driver_register(path, 0) != 0) { 743 SPDK_ERRLOG("Could not register controller %s with vhost library\n", name); 744 SPDK_ERRLOG("Check if domain socket %s already exists\n", path); 745 rc = -EIO; 746 goto out; 747 } 748 if (rte_vhost_driver_set_features(path, backend->virtio_features) || 749 rte_vhost_driver_disable_features(path, backend->disabled_features)) { 750 SPDK_ERRLOG("Couldn't set vhost features for controller %s\n", name); 751 752 rte_vhost_driver_unregister(path); 753 rc = -EIO; 754 goto out; 755 } 756 757 if (rte_vhost_driver_callback_register(path, &g_spdk_vhost_ops) != 0) { 758 rte_vhost_driver_unregister(path); 759 SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", name); 760 rc = -EIO; 761 goto out; 762 } 763 764 vdev->name = strdup(name); 765 vdev->path = strdup(path); 766 if (vdev->name == NULL || vdev->path == NULL) { 767 free(vdev->name); 768 free(vdev->path); 769 rte_vhost_driver_unregister(path); 770 rc = -EIO; 771 goto out; 772 } 773 774 vdev->cpumask = cpumask; 775 vdev->registered = true; 776 vdev->backend = backend; 777 TAILQ_INIT(&vdev->vsessions); 778 TAILQ_INSERT_TAIL(&g_spdk_vhost_devices, vdev, tailq); 779 780 spdk_vhost_set_coalescing(vdev, SPDK_VHOST_COALESCING_DELAY_BASE_US, 781 SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD); 782 783 spdk_vhost_dev_install_rte_compat_hooks(vdev); 784 785 /* The following might start a POSIX thread that polls for incoming 786 * socket connections and calls backend->start/stop_device. These backend 787 * callbacks are also protected by the global SPDK vhost mutex, so we're 788 * safe with not initializing the vdev just yet. 789 */ 790 if (spdk_call_unaffinitized(_start_rte_driver, path) == NULL) { 791 SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", 792 name, errno, spdk_strerror(errno)); 793 rte_vhost_driver_unregister(path); 794 TAILQ_REMOVE(&g_spdk_vhost_devices, vdev, tailq); 795 free(vdev->name); 796 free(vdev->path); 797 rc = -EIO; 798 goto out; 799 } 800 801 SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: new controller added\n", vdev->name); 802 return 0; 803 804 out: 805 spdk_cpuset_free(cpumask); 806 return rc; 807 } 808 809 int 810 spdk_vhost_dev_unregister(struct spdk_vhost_dev *vdev) 811 { 812 if (!TAILQ_EMPTY(&vdev->vsessions)) { 813 SPDK_ERRLOG("Controller %s has still valid connection.\n", vdev->name); 814 return -EBUSY; 815 } 816 817 if (vdev->registered && rte_vhost_driver_unregister(vdev->path) != 0) { 818 SPDK_ERRLOG("Could not unregister controller %s with vhost library\n" 819 "Check if domain socket %s still exists\n", 820 vdev->name, vdev->path); 821 return -EIO; 822 } 823 824 SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: removed\n", vdev->name); 825 826 free(vdev->name); 827 free(vdev->path); 828 spdk_cpuset_free(vdev->cpumask); 829 TAILQ_REMOVE(&g_spdk_vhost_devices, vdev, tailq); 830 return 0; 831 } 832 833 static struct spdk_vhost_session * 834 spdk_vhost_session_next(struct spdk_vhost_dev *vdev, unsigned prev_id) 835 { 836 struct spdk_vhost_session *vsession; 837 838 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 839 if (vsession->id > prev_id) { 840 return vsession; 841 } 842 } 843 844 return NULL; 845 } 846 847 const char * 848 spdk_vhost_dev_get_name(struct spdk_vhost_dev *vdev) 849 { 850 assert(vdev != NULL); 851 return vdev->name; 852 } 853 854 const struct spdk_cpuset * 855 spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *vdev) 856 { 857 assert(vdev != NULL); 858 return vdev->cpumask; 859 } 860 861 struct vhost_poll_group * 862 spdk_vhost_get_poll_group(struct spdk_cpuset *cpumask) 863 { 864 struct vhost_poll_group *pg, *selected_pg; 865 uint32_t min_ctrlrs; 866 867 min_ctrlrs = INT_MAX; 868 selected_pg = TAILQ_FIRST(&g_poll_groups); 869 870 TAILQ_FOREACH(pg, &g_poll_groups, tailq) { 871 spdk_cpuset_copy(g_tmp_cpuset, cpumask); 872 spdk_cpuset_and(g_tmp_cpuset, spdk_thread_get_cpumask(pg->thread)); 873 874 /* ignore threads which could be relocated to a non-masked cpu. */ 875 if (!spdk_cpuset_equal(g_tmp_cpuset, spdk_thread_get_cpumask(pg->thread))) { 876 continue; 877 } 878 879 if (pg->ref < min_ctrlrs) { 880 selected_pg = pg; 881 min_ctrlrs = pg->ref; 882 } 883 } 884 885 assert(selected_pg != NULL); 886 assert(selected_pg->ref < UINT_MAX); 887 selected_pg->ref++; 888 return selected_pg; 889 } 890 891 void 892 spdk_vhost_put_poll_group(struct vhost_poll_group *pg) 893 { 894 assert(pg->ref > 0); 895 pg->ref--; 896 } 897 898 void 899 spdk_vhost_session_start_done(struct spdk_vhost_session *vsession, int response) 900 { 901 if (response == 0) { 902 vsession->started = true; 903 assert(vsession->vdev->active_session_num < UINT32_MAX); 904 vsession->vdev->active_session_num++; 905 } 906 907 g_dpdk_response = response; 908 sem_post(&g_dpdk_sem); 909 } 910 911 void 912 spdk_vhost_session_stop_done(struct spdk_vhost_session *vsession, int response) 913 { 914 if (response == 0) { 915 vsession->started = false; 916 assert(vsession->vdev->active_session_num > 0); 917 vsession->vdev->active_session_num--; 918 } 919 920 g_dpdk_response = response; 921 sem_post(&g_dpdk_sem); 922 } 923 924 static void 925 spdk_vhost_event_cb(void *arg1) 926 { 927 struct spdk_vhost_session_fn_ctx *ctx = arg1; 928 struct spdk_vhost_session *vsession; 929 930 if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { 931 spdk_thread_send_msg(spdk_get_thread(), spdk_vhost_event_cb, arg1); 932 return; 933 } 934 935 vsession = spdk_vhost_session_find_by_id(ctx->vdev, ctx->vsession_id); 936 ctx->cb_fn(ctx->vdev, vsession, NULL); 937 pthread_mutex_unlock(&g_spdk_vhost_mutex); 938 } 939 940 static void spdk_vhost_external_event_foreach_continue(struct spdk_vhost_dev *vdev, 941 struct spdk_vhost_session *vsession, 942 spdk_vhost_session_fn fn, void *arg); 943 944 static void 945 spdk_vhost_event_async_foreach_fn(void *arg1) 946 { 947 struct spdk_vhost_session_fn_ctx *ctx = arg1; 948 struct spdk_vhost_session *vsession = NULL; 949 struct spdk_vhost_dev *vdev = ctx->vdev; 950 int rc; 951 952 if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { 953 spdk_thread_send_msg(spdk_get_thread(), 954 spdk_vhost_event_async_foreach_fn, arg1); 955 return; 956 } 957 958 vsession = spdk_vhost_session_find_by_id(vdev, ctx->vsession_id); 959 if (vsession == NULL || !vsession->initialized) { 960 /* The session must have been removed in the meantime, so we 961 * just skip it in our foreach chain 962 */ 963 goto out_unlock_continue; 964 } 965 966 967 if (vsession->started && vsession->poll_group->thread != spdk_get_thread()) { 968 /* if session has been relocated to other thread, it is no longer thread-safe 969 * to access its contents here. Even though we're running under the global 970 * vhost mutex, the session itself (and its pollers) are not. We need to chase 971 * the session thread as many times as necessary. 972 */ 973 spdk_thread_send_msg(vsession->poll_group->thread, 974 spdk_vhost_event_async_foreach_fn, arg1); 975 pthread_mutex_unlock(&g_spdk_vhost_mutex); 976 return; 977 } 978 979 rc = ctx->cb_fn(vdev, vsession, ctx->user_ctx); 980 if (rc < 0) { 981 goto out_unlock; 982 } 983 984 out_unlock_continue: 985 vsession = spdk_vhost_session_next(vdev, ctx->vsession_id); 986 spdk_vhost_external_event_foreach_continue(vdev, vsession, ctx->cb_fn, ctx->user_ctx); 987 out_unlock: 988 pthread_mutex_unlock(&g_spdk_vhost_mutex); 989 free(ctx); 990 } 991 992 int 993 spdk_vhost_session_send_event(struct vhost_poll_group *pg, 994 struct spdk_vhost_session *vsession, 995 spdk_vhost_session_fn cb_fn, unsigned timeout_sec, 996 const char *errmsg) 997 { 998 struct spdk_vhost_session_fn_ctx ev_ctx = {0}; 999 struct timespec timeout; 1000 int rc; 1001 1002 ev_ctx.vdev = vsession->vdev; 1003 ev_ctx.vsession_id = vsession->id; 1004 ev_ctx.cb_fn = cb_fn; 1005 1006 vsession->poll_group = pg; 1007 vsession->event_ctx = &ev_ctx; 1008 spdk_thread_send_msg(pg->thread, spdk_vhost_event_cb, &ev_ctx); 1009 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1010 1011 clock_gettime(CLOCK_REALTIME, &timeout); 1012 timeout.tv_sec += timeout_sec; 1013 1014 rc = sem_timedwait(&g_dpdk_sem, &timeout); 1015 if (rc != 0) { 1016 SPDK_ERRLOG("Timeout waiting for event: %s.\n", errmsg); 1017 sem_wait(&g_dpdk_sem); 1018 } 1019 1020 pthread_mutex_lock(&g_spdk_vhost_mutex); 1021 vsession->event_ctx = NULL; 1022 return g_dpdk_response; 1023 } 1024 1025 static int 1026 spdk_vhost_event_async_send_foreach_continue(struct spdk_vhost_session *vsession, 1027 spdk_vhost_session_fn cb_fn, void *arg) 1028 { 1029 struct spdk_vhost_dev *vdev = vsession->vdev; 1030 struct spdk_vhost_session_fn_ctx *ev_ctx; 1031 1032 ev_ctx = calloc(1, sizeof(*ev_ctx)); 1033 if (ev_ctx == NULL) { 1034 SPDK_ERRLOG("Failed to alloc vhost event.\n"); 1035 assert(false); 1036 return -ENOMEM; 1037 } 1038 1039 ev_ctx->vdev = vdev; 1040 ev_ctx->vsession_id = vsession->id; 1041 ev_ctx->cb_fn = cb_fn; 1042 ev_ctx->user_ctx = arg; 1043 1044 spdk_thread_send_msg(vsession->poll_group->thread, 1045 spdk_vhost_event_async_foreach_fn, ev_ctx); 1046 return 0; 1047 } 1048 1049 static void 1050 _stop_session(struct spdk_vhost_session *vsession) 1051 { 1052 struct spdk_vhost_dev *vdev = vsession->vdev; 1053 struct spdk_vhost_virtqueue *q; 1054 int rc; 1055 uint16_t i; 1056 1057 rc = vdev->backend->stop_session(vsession); 1058 if (rc != 0) { 1059 SPDK_ERRLOG("Couldn't stop device with vid %d.\n", vsession->vid); 1060 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1061 return; 1062 } 1063 1064 for (i = 0; i < vsession->max_queues; i++) { 1065 q = &vsession->virtqueue[i]; 1066 if (q->vring.desc == NULL) { 1067 continue; 1068 } 1069 rte_vhost_set_vring_base(vsession->vid, i, q->last_avail_idx, q->last_used_idx); 1070 } 1071 1072 spdk_vhost_session_mem_unregister(vsession); 1073 free(vsession->mem); 1074 } 1075 1076 static void 1077 stop_device(int vid) 1078 { 1079 struct spdk_vhost_session *vsession; 1080 1081 pthread_mutex_lock(&g_spdk_vhost_mutex); 1082 vsession = spdk_vhost_session_find_by_vid(vid); 1083 if (vsession == NULL) { 1084 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1085 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1086 return; 1087 } 1088 1089 if (!vsession->started) { 1090 /* already stopped, nothing to do */ 1091 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1092 return; 1093 } 1094 1095 _stop_session(vsession); 1096 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1097 } 1098 1099 static int 1100 start_device(int vid) 1101 { 1102 struct spdk_vhost_dev *vdev; 1103 struct spdk_vhost_session *vsession; 1104 int rc = -1; 1105 uint16_t i; 1106 1107 pthread_mutex_lock(&g_spdk_vhost_mutex); 1108 1109 vsession = spdk_vhost_session_find_by_vid(vid); 1110 if (vsession == NULL) { 1111 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1112 goto out; 1113 } 1114 1115 vdev = vsession->vdev; 1116 if (vsession->started) { 1117 /* already started, nothing to do */ 1118 rc = 0; 1119 goto out; 1120 } 1121 1122 vsession->max_queues = 0; 1123 memset(vsession->virtqueue, 0, sizeof(vsession->virtqueue)); 1124 for (i = 0; i < SPDK_VHOST_MAX_VQUEUES; i++) { 1125 struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; 1126 1127 q->vring_idx = -1; 1128 if (rte_vhost_get_vhost_vring(vid, i, &q->vring)) { 1129 continue; 1130 } 1131 q->vring_idx = i; 1132 1133 if (q->vring.desc == NULL || q->vring.size == 0) { 1134 continue; 1135 } 1136 1137 if (rte_vhost_get_vring_base(vsession->vid, i, &q->last_avail_idx, &q->last_used_idx)) { 1138 q->vring.desc = NULL; 1139 continue; 1140 } 1141 1142 /* Disable notifications. */ 1143 if (rte_vhost_enable_guest_notification(vid, i, 0) != 0) { 1144 SPDK_ERRLOG("vhost device %d: Failed to disable guest notification on queue %"PRIu16"\n", vid, i); 1145 goto out; 1146 } 1147 1148 vsession->max_queues = i + 1; 1149 } 1150 1151 if (rte_vhost_get_negotiated_features(vid, &vsession->negotiated_features) != 0) { 1152 SPDK_ERRLOG("vhost device %d: Failed to get negotiated driver features\n", vid); 1153 goto out; 1154 } 1155 1156 if (rte_vhost_get_mem_table(vid, &vsession->mem) != 0) { 1157 SPDK_ERRLOG("vhost device %d: Failed to get guest memory table\n", vid); 1158 goto out; 1159 } 1160 1161 /* 1162 * Not sure right now but this look like some kind of QEMU bug and guest IO 1163 * might be frozed without kicking all queues after live-migration. This look like 1164 * the previous vhost instance failed to effectively deliver all interrupts before 1165 * the GET_VRING_BASE message. This shouldn't harm guest since spurious interrupts 1166 * should be ignored by guest virtio driver. 1167 * 1168 * Tested on QEMU 2.10.91 and 2.11.50. 1169 */ 1170 for (i = 0; i < vsession->max_queues; i++) { 1171 struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; 1172 1173 if (q->vring.desc != NULL && q->vring.size > 0) { 1174 rte_vhost_vring_call(vsession->vid, q->vring_idx); 1175 } 1176 } 1177 1178 spdk_vhost_session_set_coalescing(vdev, vsession, NULL); 1179 spdk_vhost_session_mem_register(vsession); 1180 vsession->initialized = true; 1181 rc = vdev->backend->start_session(vsession); 1182 if (rc != 0) { 1183 spdk_vhost_session_mem_unregister(vsession); 1184 free(vsession->mem); 1185 goto out; 1186 } 1187 1188 out: 1189 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1190 return rc; 1191 } 1192 1193 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 1194 static int 1195 get_config(int vid, uint8_t *config, uint32_t len) 1196 { 1197 struct spdk_vhost_session *vsession; 1198 struct spdk_vhost_dev *vdev; 1199 int rc = -1; 1200 1201 pthread_mutex_lock(&g_spdk_vhost_mutex); 1202 vsession = spdk_vhost_session_find_by_vid(vid); 1203 if (vsession == NULL) { 1204 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1205 goto out; 1206 } 1207 1208 vdev = vsession->vdev; 1209 if (vdev->backend->vhost_get_config) { 1210 rc = vdev->backend->vhost_get_config(vdev, config, len); 1211 } 1212 1213 out: 1214 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1215 return rc; 1216 } 1217 1218 static int 1219 set_config(int vid, uint8_t *config, uint32_t offset, uint32_t size, uint32_t flags) 1220 { 1221 struct spdk_vhost_session *vsession; 1222 struct spdk_vhost_dev *vdev; 1223 int rc = -1; 1224 1225 pthread_mutex_lock(&g_spdk_vhost_mutex); 1226 vsession = spdk_vhost_session_find_by_vid(vid); 1227 if (vsession == NULL) { 1228 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1229 goto out; 1230 } 1231 1232 vdev = vsession->vdev; 1233 if (vdev->backend->vhost_set_config) { 1234 rc = vdev->backend->vhost_set_config(vdev, config, offset, size, flags); 1235 } 1236 1237 out: 1238 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1239 return rc; 1240 } 1241 #endif 1242 1243 int 1244 spdk_vhost_set_socket_path(const char *basename) 1245 { 1246 int ret; 1247 1248 if (basename && strlen(basename) > 0) { 1249 ret = snprintf(dev_dirname, sizeof(dev_dirname) - 2, "%s", basename); 1250 if (ret <= 0) { 1251 return -EINVAL; 1252 } 1253 if ((size_t)ret >= sizeof(dev_dirname) - 2) { 1254 SPDK_ERRLOG("Char dev dir path length %d is too long\n", ret); 1255 return -EINVAL; 1256 } 1257 1258 if (dev_dirname[ret - 1] != '/') { 1259 dev_dirname[ret] = '/'; 1260 dev_dirname[ret + 1] = '\0'; 1261 } 1262 } 1263 1264 return 0; 1265 } 1266 1267 void 1268 spdk_vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) 1269 { 1270 assert(vdev->backend->dump_info_json != NULL); 1271 vdev->backend->dump_info_json(vdev, w); 1272 } 1273 1274 int 1275 spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev) 1276 { 1277 if (vdev->pending_async_op_num) { 1278 return -EBUSY; 1279 } 1280 1281 return vdev->backend->remove_device(vdev); 1282 } 1283 1284 static int 1285 new_connection(int vid) 1286 { 1287 struct spdk_vhost_dev *vdev; 1288 struct spdk_vhost_session *vsession; 1289 char ifname[PATH_MAX]; 1290 1291 pthread_mutex_lock(&g_spdk_vhost_mutex); 1292 1293 if (rte_vhost_get_ifname(vid, ifname, PATH_MAX) < 0) { 1294 SPDK_ERRLOG("Couldn't get a valid ifname for device with vid %d\n", vid); 1295 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1296 return -1; 1297 } 1298 1299 vdev = spdk_vhost_dev_find(ifname); 1300 if (vdev == NULL) { 1301 SPDK_ERRLOG("Couldn't find device with vid %d to create connection for.\n", vid); 1302 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1303 return -1; 1304 } 1305 1306 /* We expect sessions inside vdev->vsessions to be sorted in ascending 1307 * order in regard of vsession->id. For now we always set id = vsessions_cnt++ 1308 * and append each session to the very end of the vsessions list. 1309 * This is required for spdk_vhost_dev_foreach_session() to work. 1310 */ 1311 if (vdev->vsessions_num == UINT_MAX) { 1312 assert(false); 1313 return -EINVAL; 1314 } 1315 1316 if (posix_memalign((void **)&vsession, SPDK_CACHE_LINE_SIZE, sizeof(*vsession) + 1317 vdev->backend->session_ctx_size)) { 1318 SPDK_ERRLOG("vsession alloc failed\n"); 1319 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1320 return -1; 1321 } 1322 memset(vsession, 0, sizeof(*vsession) + vdev->backend->session_ctx_size); 1323 1324 vsession->vdev = vdev; 1325 vsession->id = vdev->vsessions_num++; 1326 vsession->vid = vid; 1327 vsession->poll_group = NULL; 1328 vsession->started = false; 1329 vsession->initialized = false; 1330 vsession->next_stats_check_time = 0; 1331 vsession->stats_check_interval = SPDK_VHOST_STATS_CHECK_INTERVAL_MS * 1332 spdk_get_ticks_hz() / 1000UL; 1333 TAILQ_INSERT_TAIL(&vdev->vsessions, vsession, tailq); 1334 1335 spdk_vhost_session_install_rte_compat_hooks(vsession); 1336 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1337 return 0; 1338 } 1339 1340 static void 1341 destroy_connection(int vid) 1342 { 1343 struct spdk_vhost_session *vsession; 1344 1345 pthread_mutex_lock(&g_spdk_vhost_mutex); 1346 vsession = spdk_vhost_session_find_by_vid(vid); 1347 if (vsession == NULL) { 1348 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1349 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1350 return; 1351 } 1352 1353 if (vsession->started) { 1354 _stop_session(vsession); 1355 } 1356 1357 TAILQ_REMOVE(&vsession->vdev->vsessions, vsession, tailq); 1358 free(vsession); 1359 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1360 } 1361 1362 static void 1363 spdk_vhost_external_event_foreach_continue(struct spdk_vhost_dev *vdev, 1364 struct spdk_vhost_session *vsession, 1365 spdk_vhost_session_fn fn, void *arg) 1366 { 1367 int rc; 1368 1369 if (vsession == NULL) { 1370 goto out_finish_foreach; 1371 } 1372 1373 while (!vsession->started) { 1374 if (vsession->initialized) { 1375 rc = fn(vdev, vsession, arg); 1376 if (rc < 0) { 1377 return; 1378 } 1379 } 1380 1381 vsession = spdk_vhost_session_next(vdev, vsession->id); 1382 if (vsession == NULL) { 1383 goto out_finish_foreach; 1384 } 1385 } 1386 1387 spdk_vhost_event_async_send_foreach_continue(vsession, fn, arg); 1388 return; 1389 1390 out_finish_foreach: 1391 /* there are no more sessions to iterate through, so call the 1392 * fn one last time with vsession == NULL 1393 */ 1394 assert(vdev->pending_async_op_num > 0); 1395 vdev->pending_async_op_num--; 1396 fn(vdev, NULL, arg); 1397 } 1398 1399 void 1400 spdk_vhost_dev_foreach_session(struct spdk_vhost_dev *vdev, 1401 spdk_vhost_session_fn fn, void *arg) 1402 { 1403 struct spdk_vhost_session *vsession = TAILQ_FIRST(&vdev->vsessions); 1404 1405 assert(vdev->pending_async_op_num < UINT32_MAX); 1406 vdev->pending_async_op_num++; 1407 spdk_vhost_external_event_foreach_continue(vdev, vsession, fn, arg); 1408 } 1409 1410 void 1411 spdk_vhost_lock(void) 1412 { 1413 pthread_mutex_lock(&g_spdk_vhost_mutex); 1414 } 1415 1416 int 1417 spdk_vhost_trylock(void) 1418 { 1419 return -pthread_mutex_trylock(&g_spdk_vhost_mutex); 1420 } 1421 1422 void 1423 spdk_vhost_unlock(void) 1424 { 1425 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1426 } 1427 1428 static void 1429 vhost_create_poll_group_done(void *ctx) 1430 { 1431 spdk_vhost_init_cb init_cb = ctx; 1432 int ret; 1433 1434 if (TAILQ_EMPTY(&g_poll_groups)) { 1435 /* No threads? Iteration failed? */ 1436 init_cb(-ECHILD); 1437 return; 1438 } 1439 1440 ret = spdk_vhost_scsi_controller_construct(); 1441 if (ret != 0) { 1442 SPDK_ERRLOG("Cannot construct vhost controllers\n"); 1443 goto out; 1444 } 1445 1446 ret = spdk_vhost_blk_controller_construct(); 1447 if (ret != 0) { 1448 SPDK_ERRLOG("Cannot construct vhost block controllers\n"); 1449 goto out; 1450 } 1451 1452 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 1453 ret = spdk_vhost_nvme_controller_construct(); 1454 if (ret != 0) { 1455 SPDK_ERRLOG("Cannot construct vhost NVMe controllers\n"); 1456 goto out; 1457 } 1458 #endif 1459 1460 out: 1461 init_cb(ret); 1462 } 1463 1464 static void 1465 vhost_create_poll_group(void *ctx) 1466 { 1467 struct vhost_poll_group *pg; 1468 1469 pg = calloc(1, sizeof(*pg)); 1470 if (!pg) { 1471 SPDK_ERRLOG("Not enough memory to allocate poll groups\n"); 1472 spdk_app_stop(-ENOMEM); 1473 return; 1474 } 1475 1476 pg->thread = spdk_get_thread(); 1477 TAILQ_INSERT_TAIL(&g_poll_groups, pg, tailq); 1478 } 1479 1480 void 1481 spdk_vhost_init(spdk_vhost_init_cb init_cb) 1482 { 1483 size_t len; 1484 int ret; 1485 1486 g_vhost_init_thread = spdk_get_thread(); 1487 assert(g_vhost_init_thread != NULL); 1488 1489 if (dev_dirname[0] == '\0') { 1490 if (getcwd(dev_dirname, sizeof(dev_dirname) - 1) == NULL) { 1491 SPDK_ERRLOG("getcwd failed (%d): %s\n", errno, spdk_strerror(errno)); 1492 ret = -1; 1493 goto err_out; 1494 } 1495 1496 len = strlen(dev_dirname); 1497 if (dev_dirname[len - 1] != '/') { 1498 dev_dirname[len] = '/'; 1499 dev_dirname[len + 1] = '\0'; 1500 } 1501 } 1502 1503 g_tmp_cpuset = spdk_cpuset_alloc(); 1504 if (g_tmp_cpuset == NULL) { 1505 ret = -1; 1506 goto err_out; 1507 } 1508 1509 ret = sem_init(&g_dpdk_sem, 0, 0); 1510 if (ret != 0) { 1511 SPDK_ERRLOG("Failed to initialize semaphore for rte_vhost pthread.\n"); 1512 spdk_cpuset_free(g_tmp_cpuset); 1513 ret = -1; 1514 goto err_out; 1515 } 1516 1517 spdk_for_each_thread(vhost_create_poll_group, 1518 init_cb, 1519 vhost_create_poll_group_done); 1520 return; 1521 err_out: 1522 init_cb(ret); 1523 } 1524 1525 static void 1526 _spdk_vhost_fini(void *arg1) 1527 { 1528 struct spdk_vhost_dev *vdev, *tmp; 1529 struct vhost_poll_group *pg, *tpg; 1530 1531 spdk_vhost_lock(); 1532 vdev = spdk_vhost_dev_next(NULL); 1533 while (vdev != NULL) { 1534 tmp = spdk_vhost_dev_next(vdev); 1535 spdk_vhost_dev_remove(vdev); 1536 /* don't care if it fails, there's nothing we can do for now */ 1537 vdev = tmp; 1538 } 1539 spdk_vhost_unlock(); 1540 1541 /* All devices are removed now. */ 1542 sem_destroy(&g_dpdk_sem); 1543 spdk_cpuset_free(g_tmp_cpuset); 1544 TAILQ_FOREACH_SAFE(pg, &g_poll_groups, tailq, tpg) { 1545 TAILQ_REMOVE(&g_poll_groups, pg, tailq); 1546 free(pg); 1547 } 1548 g_fini_cpl_cb(); 1549 } 1550 1551 static void * 1552 session_shutdown(void *arg) 1553 { 1554 struct spdk_vhost_dev *vdev = NULL; 1555 1556 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 1557 rte_vhost_driver_unregister(vdev->path); 1558 vdev->registered = false; 1559 } 1560 1561 SPDK_INFOLOG(SPDK_LOG_VHOST, "Exiting\n"); 1562 spdk_thread_send_msg(g_vhost_init_thread, _spdk_vhost_fini, NULL); 1563 return NULL; 1564 } 1565 1566 void 1567 spdk_vhost_fini(spdk_vhost_fini_cb fini_cb) 1568 { 1569 pthread_t tid; 1570 int rc; 1571 1572 assert(spdk_get_thread() == g_vhost_init_thread); 1573 g_fini_cpl_cb = fini_cb; 1574 1575 /* rte_vhost API for removing sockets is not asynchronous. Since it may call SPDK 1576 * ops for stopping a device or removing a connection, we need to call it from 1577 * a separate thread to avoid deadlock. 1578 */ 1579 rc = pthread_create(&tid, NULL, &session_shutdown, NULL); 1580 if (rc < 0) { 1581 SPDK_ERRLOG("Failed to start session shutdown thread (%d): %s\n", rc, spdk_strerror(rc)); 1582 abort(); 1583 } 1584 pthread_detach(tid); 1585 } 1586 1587 void 1588 spdk_vhost_config_json(struct spdk_json_write_ctx *w) 1589 { 1590 struct spdk_vhost_dev *vdev; 1591 uint32_t delay_base_us; 1592 uint32_t iops_threshold; 1593 1594 spdk_json_write_array_begin(w); 1595 1596 spdk_vhost_lock(); 1597 vdev = spdk_vhost_dev_next(NULL); 1598 while (vdev != NULL) { 1599 vdev->backend->write_config_json(vdev, w); 1600 1601 spdk_vhost_get_coalescing(vdev, &delay_base_us, &iops_threshold); 1602 if (delay_base_us) { 1603 spdk_json_write_object_begin(w); 1604 spdk_json_write_named_string(w, "method", "set_vhost_controller_coalescing"); 1605 1606 spdk_json_write_named_object_begin(w, "params"); 1607 spdk_json_write_named_string(w, "ctrlr", vdev->name); 1608 spdk_json_write_named_uint32(w, "delay_base_us", delay_base_us); 1609 spdk_json_write_named_uint32(w, "iops_threshold", iops_threshold); 1610 spdk_json_write_object_end(w); 1611 1612 spdk_json_write_object_end(w); 1613 } 1614 vdev = spdk_vhost_dev_next(vdev); 1615 } 1616 spdk_vhost_unlock(); 1617 1618 spdk_json_write_array_end(w); 1619 } 1620 1621 SPDK_LOG_REGISTER_COMPONENT("vhost", SPDK_LOG_VHOST) 1622 SPDK_LOG_REGISTER_COMPONENT("vhost_ring", SPDK_LOG_VHOST_RING) 1623