1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/env.h" 37 #include "spdk/likely.h" 38 #include "spdk/string.h" 39 #include "spdk/util.h" 40 #include "spdk/barrier.h" 41 #include "spdk/vhost.h" 42 #include "vhost_internal.h" 43 44 #include "spdk_internal/memory.h" 45 46 struct vhost_poll_group { 47 struct spdk_thread *thread; 48 unsigned ref; 49 TAILQ_ENTRY(vhost_poll_group) tailq; 50 }; 51 52 static TAILQ_HEAD(, vhost_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); 53 54 /* Temporary cpuset for poll group assignment */ 55 static struct spdk_cpuset *g_tmp_cpuset; 56 57 /* Path to folder where character device will be created. Can be set by user. */ 58 static char dev_dirname[PATH_MAX] = ""; 59 60 /* Thread performing all vhost management operations */ 61 static struct spdk_thread *g_vhost_init_thread; 62 63 static spdk_vhost_fini_cb g_fini_cpl_cb; 64 65 /** 66 * DPDK calls our callbacks synchronously but the work those callbacks 67 * perform needs to be async. Luckily, all DPDK callbacks are called on 68 * a DPDK-internal pthread, so we'll just wait on a semaphore in there. 69 */ 70 static sem_t g_dpdk_sem; 71 72 /** Return code for the current DPDK callback */ 73 static int g_dpdk_response; 74 75 struct spdk_vhost_session_fn_ctx { 76 /** Device pointer obtained before enqueuing the event */ 77 struct spdk_vhost_dev *vdev; 78 79 /** ID of the session to send event to. */ 80 uint32_t vsession_id; 81 82 /** User callback function to be executed on given thread. */ 83 spdk_vhost_session_fn cb_fn; 84 85 /** Custom user context */ 86 void *user_ctx; 87 }; 88 89 static int new_connection(int vid); 90 static int start_device(int vid); 91 static void stop_device(int vid); 92 static void destroy_connection(int vid); 93 94 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 95 static int get_config(int vid, uint8_t *config, uint32_t len); 96 static int set_config(int vid, uint8_t *config, uint32_t offset, 97 uint32_t size, uint32_t flags); 98 #endif 99 100 const struct vhost_device_ops g_spdk_vhost_ops = { 101 .new_device = start_device, 102 .destroy_device = stop_device, 103 .new_connection = new_connection, 104 .destroy_connection = destroy_connection, 105 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 106 .get_config = get_config, 107 .set_config = set_config, 108 .vhost_nvme_admin_passthrough = spdk_vhost_nvme_admin_passthrough, 109 .vhost_nvme_set_cq_call = spdk_vhost_nvme_set_cq_call, 110 .vhost_nvme_get_cap = spdk_vhost_nvme_get_cap, 111 .vhost_nvme_set_bar_mr = spdk_vhost_nvme_set_bar_mr, 112 #endif 113 }; 114 115 static TAILQ_HEAD(, spdk_vhost_dev) g_spdk_vhost_devices = TAILQ_HEAD_INITIALIZER( 116 g_spdk_vhost_devices); 117 static pthread_mutex_t g_spdk_vhost_mutex = PTHREAD_MUTEX_INITIALIZER; 118 119 void *spdk_vhost_gpa_to_vva(struct spdk_vhost_session *vsession, uint64_t addr, uint64_t len) 120 { 121 void *vva; 122 uint64_t newlen; 123 124 newlen = len; 125 vva = (void *)rte_vhost_va_from_guest_pa(vsession->mem, addr, &newlen); 126 if (newlen != len) { 127 return NULL; 128 } 129 130 return vva; 131 132 } 133 134 static void 135 spdk_vhost_log_req_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, 136 uint16_t req_id) 137 { 138 struct vring_desc *desc, *desc_table; 139 uint32_t desc_table_size; 140 int rc; 141 142 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 143 return; 144 } 145 146 rc = spdk_vhost_vq_get_desc(vsession, virtqueue, req_id, &desc, &desc_table, &desc_table_size); 147 if (spdk_unlikely(rc != 0)) { 148 SPDK_ERRLOG("Can't log used ring descriptors!\n"); 149 return; 150 } 151 152 do { 153 if (spdk_vhost_vring_desc_is_wr(desc)) { 154 /* To be honest, only pages realy touched should be logged, but 155 * doing so would require tracking those changes in each backed. 156 * Also backend most likely will touch all/most of those pages so 157 * for lets assume we touched all pages passed to as writeable buffers. */ 158 rte_vhost_log_write(vsession->vid, desc->addr, desc->len); 159 } 160 spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); 161 } while (desc); 162 } 163 164 static void 165 spdk_vhost_log_used_vring_elem(struct spdk_vhost_session *vsession, 166 struct spdk_vhost_virtqueue *virtqueue, 167 uint16_t idx) 168 { 169 uint64_t offset, len; 170 uint16_t vq_idx; 171 172 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 173 return; 174 } 175 176 offset = offsetof(struct vring_used, ring[idx]); 177 len = sizeof(virtqueue->vring.used->ring[idx]); 178 vq_idx = virtqueue - vsession->virtqueue; 179 180 rte_vhost_log_used_vring(vsession->vid, vq_idx, offset, len); 181 } 182 183 static void 184 spdk_vhost_log_used_vring_idx(struct spdk_vhost_session *vsession, 185 struct spdk_vhost_virtqueue *virtqueue) 186 { 187 uint64_t offset, len; 188 uint16_t vq_idx; 189 190 if (spdk_likely(!spdk_vhost_dev_has_feature(vsession, VHOST_F_LOG_ALL))) { 191 return; 192 } 193 194 offset = offsetof(struct vring_used, idx); 195 len = sizeof(virtqueue->vring.used->idx); 196 vq_idx = virtqueue - vsession->virtqueue; 197 198 rte_vhost_log_used_vring(vsession->vid, vq_idx, offset, len); 199 } 200 201 /* 202 * Get available requests from avail ring. 203 */ 204 uint16_t 205 spdk_vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *virtqueue, uint16_t *reqs, 206 uint16_t reqs_len) 207 { 208 struct rte_vhost_vring *vring = &virtqueue->vring; 209 struct vring_avail *avail = vring->avail; 210 uint16_t size_mask = vring->size - 1; 211 uint16_t last_idx = virtqueue->last_avail_idx, avail_idx = avail->idx; 212 uint16_t count, i; 213 214 count = avail_idx - last_idx; 215 if (spdk_likely(count == 0)) { 216 return 0; 217 } 218 219 if (spdk_unlikely(count > vring->size)) { 220 /* TODO: the queue is unrecoverably broken and should be marked so. 221 * For now we will fail silently and report there are no new avail entries. 222 */ 223 return 0; 224 } 225 226 count = spdk_min(count, reqs_len); 227 virtqueue->last_avail_idx += count; 228 for (i = 0; i < count; i++) { 229 reqs[i] = vring->avail->ring[(last_idx + i) & size_mask]; 230 } 231 232 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 233 "AVAIL: last_idx=%"PRIu16" avail_idx=%"PRIu16" count=%"PRIu16"\n", 234 last_idx, avail_idx, count); 235 236 return count; 237 } 238 239 static bool 240 spdk_vhost_vring_desc_is_indirect(struct vring_desc *cur_desc) 241 { 242 return !!(cur_desc->flags & VRING_DESC_F_INDIRECT); 243 } 244 245 int 246 spdk_vhost_vq_get_desc(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, 247 uint16_t req_idx, struct vring_desc **desc, struct vring_desc **desc_table, 248 uint32_t *desc_table_size) 249 { 250 if (spdk_unlikely(req_idx >= virtqueue->vring.size)) { 251 return -1; 252 } 253 254 *desc = &virtqueue->vring.desc[req_idx]; 255 256 if (spdk_vhost_vring_desc_is_indirect(*desc)) { 257 *desc_table_size = (*desc)->len / sizeof(**desc); 258 *desc_table = spdk_vhost_gpa_to_vva(vsession, (*desc)->addr, 259 sizeof(**desc) * *desc_table_size); 260 *desc = *desc_table; 261 if (*desc == NULL) { 262 return -1; 263 } 264 265 return 0; 266 } 267 268 *desc_table = virtqueue->vring.desc; 269 *desc_table_size = virtqueue->vring.size; 270 271 return 0; 272 } 273 274 int 275 spdk_vhost_vq_used_signal(struct spdk_vhost_session *vsession, 276 struct spdk_vhost_virtqueue *virtqueue) 277 { 278 if (virtqueue->used_req_cnt == 0) { 279 return 0; 280 } 281 282 virtqueue->req_cnt += virtqueue->used_req_cnt; 283 virtqueue->used_req_cnt = 0; 284 285 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 286 "Queue %td - USED RING: sending IRQ: last used %"PRIu16"\n", 287 virtqueue - vsession->virtqueue, virtqueue->last_used_idx); 288 289 if (rte_vhost_vring_call(vsession->vid, virtqueue->vring_idx) == 0) { 290 /* interrupt signalled */ 291 return 1; 292 } else { 293 /* interrupt not signalled */ 294 return 0; 295 } 296 } 297 298 299 static void 300 check_session_io_stats(struct spdk_vhost_session *vsession, uint64_t now) 301 { 302 struct spdk_vhost_virtqueue *virtqueue; 303 uint32_t irq_delay_base = vsession->coalescing_delay_time_base; 304 uint32_t io_threshold = vsession->coalescing_io_rate_threshold; 305 int32_t irq_delay; 306 uint32_t req_cnt; 307 uint16_t q_idx; 308 309 if (now < vsession->next_stats_check_time) { 310 return; 311 } 312 313 vsession->next_stats_check_time = now + vsession->stats_check_interval; 314 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 315 virtqueue = &vsession->virtqueue[q_idx]; 316 317 req_cnt = virtqueue->req_cnt + virtqueue->used_req_cnt; 318 if (req_cnt <= io_threshold) { 319 continue; 320 } 321 322 irq_delay = (irq_delay_base * (req_cnt - io_threshold)) / io_threshold; 323 virtqueue->irq_delay_time = (uint32_t) spdk_max(0, irq_delay); 324 325 virtqueue->req_cnt = 0; 326 virtqueue->next_event_time = now; 327 } 328 } 329 330 void 331 spdk_vhost_session_used_signal(struct spdk_vhost_session *vsession) 332 { 333 struct spdk_vhost_virtqueue *virtqueue; 334 uint64_t now; 335 uint16_t q_idx; 336 337 if (vsession->coalescing_delay_time_base == 0) { 338 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 339 virtqueue = &vsession->virtqueue[q_idx]; 340 341 if (virtqueue->vring.desc == NULL || 342 (virtqueue->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 343 continue; 344 } 345 346 spdk_vhost_vq_used_signal(vsession, virtqueue); 347 } 348 } else { 349 now = spdk_get_ticks(); 350 check_session_io_stats(vsession, now); 351 352 for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) { 353 virtqueue = &vsession->virtqueue[q_idx]; 354 355 /* No need for event right now */ 356 if (now < virtqueue->next_event_time || 357 (virtqueue->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 358 continue; 359 } 360 361 if (!spdk_vhost_vq_used_signal(vsession, virtqueue)) { 362 continue; 363 } 364 365 /* Syscall is quite long so update time */ 366 now = spdk_get_ticks(); 367 virtqueue->next_event_time = now + virtqueue->irq_delay_time; 368 } 369 } 370 } 371 372 static int 373 spdk_vhost_session_set_coalescing(struct spdk_vhost_dev *vdev, 374 struct spdk_vhost_session *vsession, void *ctx) 375 { 376 if (vdev == NULL || vsession == NULL) { 377 /* nothing to do */ 378 return 0; 379 } 380 381 vsession->coalescing_delay_time_base = 382 vdev->coalescing_delay_us * spdk_get_ticks_hz() / 1000000ULL; 383 vsession->coalescing_io_rate_threshold = 384 vdev->coalescing_iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; 385 return 0; 386 } 387 388 static int 389 vhost_dev_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us, 390 uint32_t iops_threshold) 391 { 392 uint64_t delay_time_base = delay_base_us * spdk_get_ticks_hz() / 1000000ULL; 393 uint32_t io_rate = iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U; 394 395 if (delay_time_base >= UINT32_MAX) { 396 SPDK_ERRLOG("Delay time of %"PRIu32" is to big\n", delay_base_us); 397 return -EINVAL; 398 } else if (io_rate == 0) { 399 SPDK_ERRLOG("IOPS rate of %"PRIu32" is too low. Min is %u\n", io_rate, 400 1000U / SPDK_VHOST_STATS_CHECK_INTERVAL_MS); 401 return -EINVAL; 402 } 403 404 vdev->coalescing_delay_us = delay_base_us; 405 vdev->coalescing_iops_threshold = iops_threshold; 406 return 0; 407 } 408 409 int 410 spdk_vhost_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us, 411 uint32_t iops_threshold) 412 { 413 int rc; 414 415 rc = vhost_dev_set_coalescing(vdev, delay_base_us, iops_threshold); 416 if (rc != 0) { 417 return rc; 418 } 419 420 spdk_vhost_dev_foreach_session(vdev, spdk_vhost_session_set_coalescing, NULL); 421 return 0; 422 } 423 424 void 425 spdk_vhost_get_coalescing(struct spdk_vhost_dev *vdev, uint32_t *delay_base_us, 426 uint32_t *iops_threshold) 427 { 428 if (delay_base_us) { 429 *delay_base_us = vdev->coalescing_delay_us; 430 } 431 432 if (iops_threshold) { 433 *iops_threshold = vdev->coalescing_iops_threshold; 434 } 435 } 436 437 /* 438 * Enqueue id and len to used ring. 439 */ 440 void 441 spdk_vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, 442 struct spdk_vhost_virtqueue *virtqueue, 443 uint16_t id, uint32_t len) 444 { 445 struct rte_vhost_vring *vring = &virtqueue->vring; 446 struct vring_used *used = vring->used; 447 uint16_t last_idx = virtqueue->last_used_idx & (vring->size - 1); 448 449 SPDK_DEBUGLOG(SPDK_LOG_VHOST_RING, 450 "Queue %td - USED RING: last_idx=%"PRIu16" req id=%"PRIu16" len=%"PRIu32"\n", 451 virtqueue - vsession->virtqueue, virtqueue->last_used_idx, id, len); 452 453 spdk_vhost_log_req_desc(vsession, virtqueue, id); 454 455 virtqueue->last_used_idx++; 456 used->ring[last_idx].id = id; 457 used->ring[last_idx].len = len; 458 459 /* Ensure the used ring is updated before we log it or increment used->idx. */ 460 spdk_smp_wmb(); 461 462 spdk_vhost_log_used_vring_elem(vsession, virtqueue, last_idx); 463 * (volatile uint16_t *) &used->idx = virtqueue->last_used_idx; 464 spdk_vhost_log_used_vring_idx(vsession, virtqueue); 465 466 /* Ensure all our used ring changes are visible to the guest at the time 467 * of interrupt. 468 * TODO: this is currently an sfence on x86. For other architectures we 469 * will most likely need an smp_mb(), but smp_mb() is an overkill for x86. 470 */ 471 spdk_wmb(); 472 473 virtqueue->used_req_cnt++; 474 } 475 476 int 477 spdk_vhost_vring_desc_get_next(struct vring_desc **desc, 478 struct vring_desc *desc_table, uint32_t desc_table_size) 479 { 480 struct vring_desc *old_desc = *desc; 481 uint16_t next_idx; 482 483 if ((old_desc->flags & VRING_DESC_F_NEXT) == 0) { 484 *desc = NULL; 485 return 0; 486 } 487 488 next_idx = old_desc->next; 489 if (spdk_unlikely(next_idx >= desc_table_size)) { 490 *desc = NULL; 491 return -1; 492 } 493 494 *desc = &desc_table[next_idx]; 495 return 0; 496 } 497 498 bool 499 spdk_vhost_vring_desc_is_wr(struct vring_desc *cur_desc) 500 { 501 return !!(cur_desc->flags & VRING_DESC_F_WRITE); 502 } 503 504 int 505 spdk_vhost_vring_desc_to_iov(struct spdk_vhost_session *vsession, struct iovec *iov, 506 uint16_t *iov_index, const struct vring_desc *desc) 507 { 508 uint64_t len; 509 uint64_t remaining = desc->len; 510 uintptr_t payload = desc->addr; 511 uintptr_t vva; 512 513 do { 514 if (*iov_index >= SPDK_VHOST_IOVS_MAX) { 515 SPDK_ERRLOG("SPDK_VHOST_IOVS_MAX(%d) reached\n", SPDK_VHOST_IOVS_MAX); 516 return -1; 517 } 518 len = remaining; 519 vva = (uintptr_t)rte_vhost_va_from_guest_pa(vsession->mem, payload, &len); 520 if (vva == 0 || len == 0) { 521 SPDK_ERRLOG("gpa_to_vva(%p) == NULL\n", (void *)payload); 522 return -1; 523 } 524 iov[*iov_index].iov_base = (void *)vva; 525 iov[*iov_index].iov_len = len; 526 remaining -= len; 527 payload += len; 528 (*iov_index)++; 529 } while (remaining); 530 531 return 0; 532 } 533 534 static struct spdk_vhost_session * 535 spdk_vhost_session_find_by_id(struct spdk_vhost_dev *vdev, unsigned id) 536 { 537 struct spdk_vhost_session *vsession; 538 539 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 540 if (vsession->id == id) { 541 return vsession; 542 } 543 } 544 545 return NULL; 546 } 547 548 struct spdk_vhost_session * 549 spdk_vhost_session_find_by_vid(int vid) 550 { 551 struct spdk_vhost_dev *vdev; 552 struct spdk_vhost_session *vsession; 553 554 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 555 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 556 if (vsession->vid == vid) { 557 return vsession; 558 } 559 } 560 } 561 562 return NULL; 563 } 564 565 #define SHIFT_2MB 21 566 #define SIZE_2MB (1ULL << SHIFT_2MB) 567 #define FLOOR_2MB(x) (((uintptr_t)x) / SIZE_2MB) << SHIFT_2MB 568 #define CEIL_2MB(x) ((((uintptr_t)x) + SIZE_2MB - 1) / SIZE_2MB) << SHIFT_2MB 569 570 static void 571 spdk_vhost_session_mem_register(struct spdk_vhost_session *vsession) 572 { 573 struct rte_vhost_mem_region *region; 574 uint32_t i; 575 uint64_t previous_start = UINT64_MAX; 576 577 for (i = 0; i < vsession->mem->nregions; i++) { 578 uint64_t start, end, len; 579 region = &vsession->mem->regions[i]; 580 start = FLOOR_2MB(region->mmap_addr); 581 end = CEIL_2MB(region->mmap_addr + region->mmap_size); 582 if (start == previous_start) { 583 start += (size_t) SIZE_2MB; 584 } 585 previous_start = start; 586 len = end - start; 587 SPDK_INFOLOG(SPDK_LOG_VHOST, "Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n", 588 start, len); 589 590 if (spdk_mem_register((void *)start, len) != 0) { 591 SPDK_WARNLOG("Failed to register memory region %"PRIu32". Future vtophys translation might fail.\n", 592 i); 593 continue; 594 } 595 } 596 } 597 598 static void 599 spdk_vhost_session_mem_unregister(struct spdk_vhost_session *vsession) 600 { 601 struct rte_vhost_mem_region *region; 602 uint32_t i; 603 uint64_t previous_start = UINT64_MAX; 604 605 for (i = 0; i < vsession->mem->nregions; i++) { 606 uint64_t start, end, len; 607 region = &vsession->mem->regions[i]; 608 start = FLOOR_2MB(region->mmap_addr); 609 end = CEIL_2MB(region->mmap_addr + region->mmap_size); 610 if (start == previous_start) { 611 start += (size_t) SIZE_2MB; 612 } 613 previous_start = start; 614 len = end - start; 615 616 if (spdk_vtophys((void *) start, NULL) == SPDK_VTOPHYS_ERROR) { 617 continue; /* region has not been registered */ 618 } 619 620 if (spdk_mem_unregister((void *)start, len) != 0) { 621 assert(false); 622 } 623 } 624 625 } 626 627 struct spdk_vhost_dev * 628 spdk_vhost_dev_next(struct spdk_vhost_dev *vdev) 629 { 630 if (vdev == NULL) { 631 return TAILQ_FIRST(&g_spdk_vhost_devices); 632 } 633 634 return TAILQ_NEXT(vdev, tailq); 635 } 636 637 struct spdk_vhost_dev * 638 spdk_vhost_dev_find(const char *ctrlr_name) 639 { 640 struct spdk_vhost_dev *vdev; 641 size_t dev_dirname_len = strlen(dev_dirname); 642 643 if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) { 644 ctrlr_name += dev_dirname_len; 645 } 646 647 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 648 if (strcmp(vdev->name, ctrlr_name) == 0) { 649 return vdev; 650 } 651 } 652 653 return NULL; 654 } 655 656 static int 657 spdk_vhost_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask) 658 { 659 int rc; 660 661 if (cpumask == NULL) { 662 return -1; 663 } 664 665 if (mask == NULL) { 666 spdk_cpuset_copy(cpumask, spdk_app_get_core_mask()); 667 return 0; 668 } 669 670 rc = spdk_app_parse_core_mask(mask, cpumask); 671 if (rc < 0) { 672 SPDK_ERRLOG("invalid cpumask %s\n", mask); 673 return -1; 674 } 675 676 if (spdk_cpuset_count(cpumask) == 0) { 677 SPDK_ERRLOG("no cpu is selected among reactor mask(=%s)\n", 678 spdk_cpuset_fmt(spdk_app_get_core_mask())); 679 return -1; 680 } 681 682 return 0; 683 } 684 685 static void * 686 _start_rte_driver(void *arg) 687 { 688 char *path = arg; 689 690 if (rte_vhost_driver_start(path) != 0) { 691 return NULL; 692 } 693 694 return path; 695 } 696 697 int 698 spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, 699 const struct spdk_vhost_dev_backend *backend) 700 { 701 char path[PATH_MAX]; 702 struct stat file_stat; 703 struct spdk_cpuset *cpumask; 704 int rc; 705 706 assert(vdev); 707 if (name == NULL) { 708 SPDK_ERRLOG("Can't register controller with no name\n"); 709 return -EINVAL; 710 } 711 712 cpumask = spdk_cpuset_alloc(); 713 if (!cpumask) { 714 SPDK_ERRLOG("spdk_cpuset_alloc failed\n"); 715 return -ENOMEM; 716 } 717 718 if (spdk_vhost_parse_core_mask(mask_str, cpumask) != 0) { 719 SPDK_ERRLOG("cpumask %s is invalid (app mask is 0x%s)\n", 720 mask_str, spdk_cpuset_fmt(spdk_app_get_core_mask())); 721 rc = -EINVAL; 722 goto out; 723 } 724 725 if (spdk_vhost_dev_find(name)) { 726 SPDK_ERRLOG("vhost controller %s already exists.\n", name); 727 rc = -EEXIST; 728 goto out; 729 } 730 731 if (snprintf(path, sizeof(path), "%s%s", dev_dirname, name) >= (int)sizeof(path)) { 732 SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", name, dev_dirname, 733 name); 734 rc = -EINVAL; 735 goto out; 736 } 737 738 /* Register vhost driver to handle vhost messages. */ 739 if (stat(path, &file_stat) != -1) { 740 if (!S_ISSOCK(file_stat.st_mode)) { 741 SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " 742 "The file already exists and is not a socket.\n", 743 path); 744 rc = -EIO; 745 goto out; 746 } else if (unlink(path) != 0) { 747 SPDK_ERRLOG("Cannot create a domain socket at path \"%s\": " 748 "The socket already exists and failed to unlink.\n", 749 path); 750 rc = -EIO; 751 goto out; 752 } 753 } 754 755 if (rte_vhost_driver_register(path, 0) != 0) { 756 SPDK_ERRLOG("Could not register controller %s with vhost library\n", name); 757 SPDK_ERRLOG("Check if domain socket %s already exists\n", path); 758 rc = -EIO; 759 goto out; 760 } 761 if (rte_vhost_driver_set_features(path, backend->virtio_features) || 762 rte_vhost_driver_disable_features(path, backend->disabled_features)) { 763 SPDK_ERRLOG("Couldn't set vhost features for controller %s\n", name); 764 765 rte_vhost_driver_unregister(path); 766 rc = -EIO; 767 goto out; 768 } 769 770 if (rte_vhost_driver_callback_register(path, &g_spdk_vhost_ops) != 0) { 771 rte_vhost_driver_unregister(path); 772 SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", name); 773 rc = -EIO; 774 goto out; 775 } 776 777 vdev->name = strdup(name); 778 vdev->path = strdup(path); 779 if (vdev->name == NULL || vdev->path == NULL) { 780 free(vdev->name); 781 free(vdev->path); 782 rte_vhost_driver_unregister(path); 783 rc = -EIO; 784 goto out; 785 } 786 787 vdev->cpumask = cpumask; 788 vdev->registered = true; 789 vdev->backend = backend; 790 TAILQ_INIT(&vdev->vsessions); 791 TAILQ_INSERT_TAIL(&g_spdk_vhost_devices, vdev, tailq); 792 793 vhost_dev_set_coalescing(vdev, SPDK_VHOST_COALESCING_DELAY_BASE_US, 794 SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD); 795 796 spdk_vhost_dev_install_rte_compat_hooks(vdev); 797 798 /* The following might start a POSIX thread that polls for incoming 799 * socket connections and calls backend->start/stop_device. These backend 800 * callbacks are also protected by the global SPDK vhost mutex, so we're 801 * safe with not initializing the vdev just yet. 802 */ 803 if (spdk_call_unaffinitized(_start_rte_driver, path) == NULL) { 804 SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", 805 name, errno, spdk_strerror(errno)); 806 rte_vhost_driver_unregister(path); 807 TAILQ_REMOVE(&g_spdk_vhost_devices, vdev, tailq); 808 free(vdev->name); 809 free(vdev->path); 810 rc = -EIO; 811 goto out; 812 } 813 814 SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: new controller added\n", vdev->name); 815 return 0; 816 817 out: 818 spdk_cpuset_free(cpumask); 819 return rc; 820 } 821 822 int 823 spdk_vhost_dev_unregister(struct spdk_vhost_dev *vdev) 824 { 825 if (!TAILQ_EMPTY(&vdev->vsessions)) { 826 SPDK_ERRLOG("Controller %s has still valid connection.\n", vdev->name); 827 return -EBUSY; 828 } 829 830 if (vdev->registered && rte_vhost_driver_unregister(vdev->path) != 0) { 831 SPDK_ERRLOG("Could not unregister controller %s with vhost library\n" 832 "Check if domain socket %s still exists\n", 833 vdev->name, vdev->path); 834 return -EIO; 835 } 836 837 SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: removed\n", vdev->name); 838 839 free(vdev->name); 840 free(vdev->path); 841 spdk_cpuset_free(vdev->cpumask); 842 TAILQ_REMOVE(&g_spdk_vhost_devices, vdev, tailq); 843 return 0; 844 } 845 846 static struct spdk_vhost_session * 847 spdk_vhost_session_next(struct spdk_vhost_dev *vdev, unsigned prev_id) 848 { 849 struct spdk_vhost_session *vsession; 850 851 TAILQ_FOREACH(vsession, &vdev->vsessions, tailq) { 852 if (vsession->id > prev_id) { 853 return vsession; 854 } 855 } 856 857 return NULL; 858 } 859 860 const char * 861 spdk_vhost_dev_get_name(struct spdk_vhost_dev *vdev) 862 { 863 assert(vdev != NULL); 864 return vdev->name; 865 } 866 867 const struct spdk_cpuset * 868 spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *vdev) 869 { 870 assert(vdev != NULL); 871 return vdev->cpumask; 872 } 873 874 struct vhost_poll_group * 875 spdk_vhost_get_poll_group(struct spdk_cpuset *cpumask) 876 { 877 struct vhost_poll_group *pg, *selected_pg; 878 uint32_t min_ctrlrs; 879 880 min_ctrlrs = INT_MAX; 881 selected_pg = TAILQ_FIRST(&g_poll_groups); 882 883 TAILQ_FOREACH(pg, &g_poll_groups, tailq) { 884 spdk_cpuset_copy(g_tmp_cpuset, cpumask); 885 spdk_cpuset_and(g_tmp_cpuset, spdk_thread_get_cpumask(pg->thread)); 886 887 /* ignore threads which could be relocated to a non-masked cpu. */ 888 if (!spdk_cpuset_equal(g_tmp_cpuset, spdk_thread_get_cpumask(pg->thread))) { 889 continue; 890 } 891 892 if (pg->ref < min_ctrlrs) { 893 selected_pg = pg; 894 min_ctrlrs = pg->ref; 895 } 896 } 897 898 assert(selected_pg != NULL); 899 assert(selected_pg->ref < UINT_MAX); 900 selected_pg->ref++; 901 return selected_pg; 902 } 903 904 void 905 spdk_vhost_put_poll_group(struct vhost_poll_group *pg) 906 { 907 assert(pg->ref > 0); 908 pg->ref--; 909 } 910 911 void 912 spdk_vhost_session_start_done(struct spdk_vhost_session *vsession, int response) 913 { 914 if (response == 0) { 915 vsession->started = true; 916 assert(vsession->vdev->active_session_num < UINT32_MAX); 917 vsession->vdev->active_session_num++; 918 } 919 920 g_dpdk_response = response; 921 sem_post(&g_dpdk_sem); 922 } 923 924 void 925 spdk_vhost_session_stop_done(struct spdk_vhost_session *vsession, int response) 926 { 927 if (response == 0) { 928 vsession->started = false; 929 assert(vsession->vdev->active_session_num > 0); 930 vsession->vdev->active_session_num--; 931 } 932 933 g_dpdk_response = response; 934 sem_post(&g_dpdk_sem); 935 } 936 937 static void 938 spdk_vhost_event_cb(void *arg1) 939 { 940 struct spdk_vhost_session_fn_ctx *ctx = arg1; 941 struct spdk_vhost_session *vsession; 942 943 if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { 944 spdk_thread_send_msg(spdk_get_thread(), spdk_vhost_event_cb, arg1); 945 return; 946 } 947 948 vsession = spdk_vhost_session_find_by_id(ctx->vdev, ctx->vsession_id); 949 ctx->cb_fn(ctx->vdev, vsession, NULL); 950 pthread_mutex_unlock(&g_spdk_vhost_mutex); 951 } 952 953 int 954 spdk_vhost_session_send_event(struct vhost_poll_group *pg, 955 struct spdk_vhost_session *vsession, 956 spdk_vhost_session_fn cb_fn, unsigned timeout_sec, 957 const char *errmsg) 958 { 959 struct spdk_vhost_session_fn_ctx ev_ctx = {0}; 960 struct timespec timeout; 961 int rc; 962 963 ev_ctx.vdev = vsession->vdev; 964 ev_ctx.vsession_id = vsession->id; 965 ev_ctx.cb_fn = cb_fn; 966 967 vsession->poll_group = pg; 968 spdk_thread_send_msg(pg->thread, spdk_vhost_event_cb, &ev_ctx); 969 pthread_mutex_unlock(&g_spdk_vhost_mutex); 970 971 clock_gettime(CLOCK_REALTIME, &timeout); 972 timeout.tv_sec += timeout_sec; 973 974 rc = sem_timedwait(&g_dpdk_sem, &timeout); 975 if (rc != 0) { 976 SPDK_ERRLOG("Timeout waiting for event: %s.\n", errmsg); 977 sem_wait(&g_dpdk_sem); 978 } 979 980 pthread_mutex_lock(&g_spdk_vhost_mutex); 981 return g_dpdk_response; 982 } 983 984 static void foreach_session_continue(struct spdk_vhost_session_fn_ctx *ev_ctx, 985 struct spdk_vhost_session *vsession); 986 987 static void 988 foreach_session_finish_cb(void *arg1) 989 { 990 struct spdk_vhost_session_fn_ctx *ctx = arg1; 991 struct spdk_vhost_dev *vdev = ctx->vdev; 992 993 if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { 994 spdk_thread_send_msg(spdk_get_thread(), 995 foreach_session_finish_cb, arg1); 996 return; 997 } 998 999 assert(vdev->pending_async_op_num > 0); 1000 vdev->pending_async_op_num--; 1001 /* Call fn one last time with vsession == NULL */ 1002 ctx->cb_fn(vdev, NULL, ctx->user_ctx); 1003 1004 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1005 free(ctx); 1006 } 1007 1008 static void 1009 foreach_session_continue_cb(void *arg1) 1010 { 1011 struct spdk_vhost_session_fn_ctx *ctx = arg1; 1012 struct spdk_vhost_session *vsession = NULL; 1013 struct spdk_vhost_dev *vdev = ctx->vdev; 1014 int rc; 1015 1016 if (pthread_mutex_trylock(&g_spdk_vhost_mutex) != 0) { 1017 spdk_thread_send_msg(spdk_get_thread(), 1018 foreach_session_continue_cb, arg1); 1019 return; 1020 } 1021 1022 vsession = spdk_vhost_session_find_by_id(vdev, ctx->vsession_id); 1023 if (vsession == NULL || !vsession->initialized) { 1024 /* The session must have been removed in the meantime, so we 1025 * just skip it in our foreach chain 1026 */ 1027 goto out_unlock_continue; 1028 } 1029 1030 if (vsession->started && vsession->poll_group->thread != spdk_get_thread()) { 1031 /* if session has been relocated to other thread, it is no longer thread-safe 1032 * to access its contents here. Even though we're running under the global 1033 * vhost mutex, the session itself (and its pollers) are not. We need to chase 1034 * the session thread as many times as necessary. 1035 */ 1036 spdk_thread_send_msg(vsession->poll_group->thread, 1037 foreach_session_continue_cb, arg1); 1038 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1039 return; 1040 } 1041 1042 rc = ctx->cb_fn(vdev, vsession, ctx->user_ctx); 1043 if (rc < 0) { 1044 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1045 free(ctx); 1046 return; 1047 } 1048 1049 out_unlock_continue: 1050 vsession = spdk_vhost_session_next(vdev, ctx->vsession_id); 1051 foreach_session_continue(ctx, vsession); 1052 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1053 } 1054 1055 static void 1056 foreach_session_continue(struct spdk_vhost_session_fn_ctx *ev_ctx, 1057 struct spdk_vhost_session *vsession) 1058 { 1059 struct spdk_vhost_dev *vdev = ev_ctx->vdev; 1060 int rc; 1061 1062 while (vsession != NULL && !vsession->started) { 1063 if (vsession->initialized) { 1064 rc = ev_ctx->cb_fn(vdev, vsession, ev_ctx->user_ctx); 1065 if (rc < 0) { 1066 return; 1067 } 1068 } 1069 1070 vsession = spdk_vhost_session_next(vdev, vsession->id); 1071 } 1072 1073 if (vsession != NULL) { 1074 ev_ctx->vsession_id = vsession->id; 1075 spdk_thread_send_msg(vsession->poll_group->thread, 1076 foreach_session_continue_cb, ev_ctx); 1077 } else { 1078 ev_ctx->vsession_id = UINT32_MAX; 1079 spdk_thread_send_msg(g_vhost_init_thread, 1080 foreach_session_finish_cb, ev_ctx); 1081 } 1082 } 1083 1084 void 1085 spdk_vhost_dev_foreach_session(struct spdk_vhost_dev *vdev, 1086 spdk_vhost_session_fn fn, void *arg) 1087 { 1088 struct spdk_vhost_session *vsession = TAILQ_FIRST(&vdev->vsessions); 1089 struct spdk_vhost_session_fn_ctx *ev_ctx; 1090 1091 ev_ctx = calloc(1, sizeof(*ev_ctx)); 1092 if (ev_ctx == NULL) { 1093 SPDK_ERRLOG("Failed to alloc vhost event.\n"); 1094 assert(false); 1095 return; 1096 } 1097 1098 ev_ctx->vdev = vdev; 1099 ev_ctx->cb_fn = fn; 1100 ev_ctx->user_ctx = arg; 1101 1102 assert(vdev->pending_async_op_num < UINT32_MAX); 1103 vdev->pending_async_op_num++; 1104 foreach_session_continue(ev_ctx, vsession); 1105 } 1106 1107 static void 1108 _stop_session(struct spdk_vhost_session *vsession) 1109 { 1110 struct spdk_vhost_dev *vdev = vsession->vdev; 1111 struct spdk_vhost_virtqueue *q; 1112 int rc; 1113 uint16_t i; 1114 1115 rc = vdev->backend->stop_session(vsession); 1116 if (rc != 0) { 1117 SPDK_ERRLOG("Couldn't stop device with vid %d.\n", vsession->vid); 1118 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1119 return; 1120 } 1121 1122 for (i = 0; i < vsession->max_queues; i++) { 1123 q = &vsession->virtqueue[i]; 1124 if (q->vring.desc == NULL) { 1125 continue; 1126 } 1127 rte_vhost_set_vring_base(vsession->vid, i, q->last_avail_idx, q->last_used_idx); 1128 } 1129 1130 spdk_vhost_session_mem_unregister(vsession); 1131 free(vsession->mem); 1132 } 1133 1134 static void 1135 stop_device(int vid) 1136 { 1137 struct spdk_vhost_session *vsession; 1138 1139 pthread_mutex_lock(&g_spdk_vhost_mutex); 1140 vsession = spdk_vhost_session_find_by_vid(vid); 1141 if (vsession == NULL) { 1142 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1143 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1144 return; 1145 } 1146 1147 if (!vsession->started) { 1148 /* already stopped, nothing to do */ 1149 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1150 return; 1151 } 1152 1153 _stop_session(vsession); 1154 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1155 } 1156 1157 static int 1158 start_device(int vid) 1159 { 1160 struct spdk_vhost_dev *vdev; 1161 struct spdk_vhost_session *vsession; 1162 int rc = -1; 1163 uint16_t i; 1164 1165 pthread_mutex_lock(&g_spdk_vhost_mutex); 1166 1167 vsession = spdk_vhost_session_find_by_vid(vid); 1168 if (vsession == NULL) { 1169 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1170 goto out; 1171 } 1172 1173 vdev = vsession->vdev; 1174 if (vsession->started) { 1175 /* already started, nothing to do */ 1176 rc = 0; 1177 goto out; 1178 } 1179 1180 vsession->max_queues = 0; 1181 memset(vsession->virtqueue, 0, sizeof(vsession->virtqueue)); 1182 for (i = 0; i < SPDK_VHOST_MAX_VQUEUES; i++) { 1183 struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; 1184 1185 q->vring_idx = -1; 1186 if (rte_vhost_get_vhost_vring(vid, i, &q->vring)) { 1187 continue; 1188 } 1189 q->vring_idx = i; 1190 1191 if (q->vring.desc == NULL || q->vring.size == 0) { 1192 continue; 1193 } 1194 1195 if (rte_vhost_get_vring_base(vsession->vid, i, &q->last_avail_idx, &q->last_used_idx)) { 1196 q->vring.desc = NULL; 1197 continue; 1198 } 1199 1200 /* Disable notifications. */ 1201 if (rte_vhost_enable_guest_notification(vid, i, 0) != 0) { 1202 SPDK_ERRLOG("vhost device %d: Failed to disable guest notification on queue %"PRIu16"\n", vid, i); 1203 goto out; 1204 } 1205 1206 vsession->max_queues = i + 1; 1207 } 1208 1209 if (rte_vhost_get_negotiated_features(vid, &vsession->negotiated_features) != 0) { 1210 SPDK_ERRLOG("vhost device %d: Failed to get negotiated driver features\n", vid); 1211 goto out; 1212 } 1213 1214 if (rte_vhost_get_mem_table(vid, &vsession->mem) != 0) { 1215 SPDK_ERRLOG("vhost device %d: Failed to get guest memory table\n", vid); 1216 goto out; 1217 } 1218 1219 /* 1220 * Not sure right now but this look like some kind of QEMU bug and guest IO 1221 * might be frozed without kicking all queues after live-migration. This look like 1222 * the previous vhost instance failed to effectively deliver all interrupts before 1223 * the GET_VRING_BASE message. This shouldn't harm guest since spurious interrupts 1224 * should be ignored by guest virtio driver. 1225 * 1226 * Tested on QEMU 2.10.91 and 2.11.50. 1227 */ 1228 for (i = 0; i < vsession->max_queues; i++) { 1229 struct spdk_vhost_virtqueue *q = &vsession->virtqueue[i]; 1230 1231 if (q->vring.desc != NULL && q->vring.size > 0) { 1232 rte_vhost_vring_call(vsession->vid, q->vring_idx); 1233 } 1234 } 1235 1236 spdk_vhost_session_set_coalescing(vdev, vsession, NULL); 1237 spdk_vhost_session_mem_register(vsession); 1238 vsession->initialized = true; 1239 rc = vdev->backend->start_session(vsession); 1240 if (rc != 0) { 1241 spdk_vhost_session_mem_unregister(vsession); 1242 free(vsession->mem); 1243 goto out; 1244 } 1245 1246 out: 1247 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1248 return rc; 1249 } 1250 1251 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 1252 static int 1253 get_config(int vid, uint8_t *config, uint32_t len) 1254 { 1255 struct spdk_vhost_session *vsession; 1256 struct spdk_vhost_dev *vdev; 1257 int rc = -1; 1258 1259 pthread_mutex_lock(&g_spdk_vhost_mutex); 1260 vsession = spdk_vhost_session_find_by_vid(vid); 1261 if (vsession == NULL) { 1262 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1263 goto out; 1264 } 1265 1266 vdev = vsession->vdev; 1267 if (vdev->backend->vhost_get_config) { 1268 rc = vdev->backend->vhost_get_config(vdev, config, len); 1269 } 1270 1271 out: 1272 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1273 return rc; 1274 } 1275 1276 static int 1277 set_config(int vid, uint8_t *config, uint32_t offset, uint32_t size, uint32_t flags) 1278 { 1279 struct spdk_vhost_session *vsession; 1280 struct spdk_vhost_dev *vdev; 1281 int rc = -1; 1282 1283 pthread_mutex_lock(&g_spdk_vhost_mutex); 1284 vsession = spdk_vhost_session_find_by_vid(vid); 1285 if (vsession == NULL) { 1286 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1287 goto out; 1288 } 1289 1290 vdev = vsession->vdev; 1291 if (vdev->backend->vhost_set_config) { 1292 rc = vdev->backend->vhost_set_config(vdev, config, offset, size, flags); 1293 } 1294 1295 out: 1296 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1297 return rc; 1298 } 1299 #endif 1300 1301 int 1302 spdk_vhost_set_socket_path(const char *basename) 1303 { 1304 int ret; 1305 1306 if (basename && strlen(basename) > 0) { 1307 ret = snprintf(dev_dirname, sizeof(dev_dirname) - 2, "%s", basename); 1308 if (ret <= 0) { 1309 return -EINVAL; 1310 } 1311 if ((size_t)ret >= sizeof(dev_dirname) - 2) { 1312 SPDK_ERRLOG("Char dev dir path length %d is too long\n", ret); 1313 return -EINVAL; 1314 } 1315 1316 if (dev_dirname[ret - 1] != '/') { 1317 dev_dirname[ret] = '/'; 1318 dev_dirname[ret + 1] = '\0'; 1319 } 1320 } 1321 1322 return 0; 1323 } 1324 1325 void 1326 spdk_vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) 1327 { 1328 assert(vdev->backend->dump_info_json != NULL); 1329 vdev->backend->dump_info_json(vdev, w); 1330 } 1331 1332 int 1333 spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev) 1334 { 1335 if (vdev->pending_async_op_num) { 1336 return -EBUSY; 1337 } 1338 1339 return vdev->backend->remove_device(vdev); 1340 } 1341 1342 static int 1343 new_connection(int vid) 1344 { 1345 struct spdk_vhost_dev *vdev; 1346 struct spdk_vhost_session *vsession; 1347 char ifname[PATH_MAX]; 1348 1349 pthread_mutex_lock(&g_spdk_vhost_mutex); 1350 1351 if (rte_vhost_get_ifname(vid, ifname, PATH_MAX) < 0) { 1352 SPDK_ERRLOG("Couldn't get a valid ifname for device with vid %d\n", vid); 1353 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1354 return -1; 1355 } 1356 1357 vdev = spdk_vhost_dev_find(ifname); 1358 if (vdev == NULL) { 1359 SPDK_ERRLOG("Couldn't find device with vid %d to create connection for.\n", vid); 1360 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1361 return -1; 1362 } 1363 1364 /* We expect sessions inside vdev->vsessions to be sorted in ascending 1365 * order in regard of vsession->id. For now we always set id = vsessions_cnt++ 1366 * and append each session to the very end of the vsessions list. 1367 * This is required for spdk_vhost_dev_foreach_session() to work. 1368 */ 1369 if (vdev->vsessions_num == UINT_MAX) { 1370 assert(false); 1371 return -EINVAL; 1372 } 1373 1374 if (posix_memalign((void **)&vsession, SPDK_CACHE_LINE_SIZE, sizeof(*vsession) + 1375 vdev->backend->session_ctx_size)) { 1376 SPDK_ERRLOG("vsession alloc failed\n"); 1377 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1378 return -1; 1379 } 1380 memset(vsession, 0, sizeof(*vsession) + vdev->backend->session_ctx_size); 1381 1382 vsession->vdev = vdev; 1383 vsession->id = vdev->vsessions_num++; 1384 vsession->vid = vid; 1385 vsession->poll_group = NULL; 1386 vsession->started = false; 1387 vsession->initialized = false; 1388 vsession->next_stats_check_time = 0; 1389 vsession->stats_check_interval = SPDK_VHOST_STATS_CHECK_INTERVAL_MS * 1390 spdk_get_ticks_hz() / 1000UL; 1391 TAILQ_INSERT_TAIL(&vdev->vsessions, vsession, tailq); 1392 1393 spdk_vhost_session_install_rte_compat_hooks(vsession); 1394 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1395 return 0; 1396 } 1397 1398 static void 1399 destroy_connection(int vid) 1400 { 1401 struct spdk_vhost_session *vsession; 1402 1403 pthread_mutex_lock(&g_spdk_vhost_mutex); 1404 vsession = spdk_vhost_session_find_by_vid(vid); 1405 if (vsession == NULL) { 1406 SPDK_ERRLOG("Couldn't find session with vid %d.\n", vid); 1407 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1408 return; 1409 } 1410 1411 if (vsession->started) { 1412 _stop_session(vsession); 1413 } 1414 1415 TAILQ_REMOVE(&vsession->vdev->vsessions, vsession, tailq); 1416 free(vsession); 1417 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1418 } 1419 1420 void 1421 spdk_vhost_lock(void) 1422 { 1423 pthread_mutex_lock(&g_spdk_vhost_mutex); 1424 } 1425 1426 int 1427 spdk_vhost_trylock(void) 1428 { 1429 return -pthread_mutex_trylock(&g_spdk_vhost_mutex); 1430 } 1431 1432 void 1433 spdk_vhost_unlock(void) 1434 { 1435 pthread_mutex_unlock(&g_spdk_vhost_mutex); 1436 } 1437 1438 static void 1439 vhost_create_poll_group_done(void *ctx) 1440 { 1441 spdk_vhost_init_cb init_cb = ctx; 1442 int ret; 1443 1444 if (TAILQ_EMPTY(&g_poll_groups)) { 1445 /* No threads? Iteration failed? */ 1446 init_cb(-ECHILD); 1447 return; 1448 } 1449 1450 ret = spdk_vhost_scsi_controller_construct(); 1451 if (ret != 0) { 1452 SPDK_ERRLOG("Cannot construct vhost controllers\n"); 1453 goto out; 1454 } 1455 1456 ret = spdk_vhost_blk_controller_construct(); 1457 if (ret != 0) { 1458 SPDK_ERRLOG("Cannot construct vhost block controllers\n"); 1459 goto out; 1460 } 1461 1462 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB 1463 ret = spdk_vhost_nvme_controller_construct(); 1464 if (ret != 0) { 1465 SPDK_ERRLOG("Cannot construct vhost NVMe controllers\n"); 1466 goto out; 1467 } 1468 #endif 1469 1470 out: 1471 init_cb(ret); 1472 } 1473 1474 static void 1475 vhost_create_poll_group(void *ctx) 1476 { 1477 struct vhost_poll_group *pg; 1478 1479 pg = calloc(1, sizeof(*pg)); 1480 if (!pg) { 1481 SPDK_ERRLOG("Not enough memory to allocate poll groups\n"); 1482 spdk_app_stop(-ENOMEM); 1483 return; 1484 } 1485 1486 pg->thread = spdk_get_thread(); 1487 TAILQ_INSERT_TAIL(&g_poll_groups, pg, tailq); 1488 } 1489 1490 void 1491 spdk_vhost_init(spdk_vhost_init_cb init_cb) 1492 { 1493 size_t len; 1494 int ret; 1495 1496 g_vhost_init_thread = spdk_get_thread(); 1497 assert(g_vhost_init_thread != NULL); 1498 1499 if (dev_dirname[0] == '\0') { 1500 if (getcwd(dev_dirname, sizeof(dev_dirname) - 1) == NULL) { 1501 SPDK_ERRLOG("getcwd failed (%d): %s\n", errno, spdk_strerror(errno)); 1502 ret = -1; 1503 goto err_out; 1504 } 1505 1506 len = strlen(dev_dirname); 1507 if (dev_dirname[len - 1] != '/') { 1508 dev_dirname[len] = '/'; 1509 dev_dirname[len + 1] = '\0'; 1510 } 1511 } 1512 1513 g_tmp_cpuset = spdk_cpuset_alloc(); 1514 if (g_tmp_cpuset == NULL) { 1515 ret = -1; 1516 goto err_out; 1517 } 1518 1519 ret = sem_init(&g_dpdk_sem, 0, 0); 1520 if (ret != 0) { 1521 SPDK_ERRLOG("Failed to initialize semaphore for rte_vhost pthread.\n"); 1522 spdk_cpuset_free(g_tmp_cpuset); 1523 ret = -1; 1524 goto err_out; 1525 } 1526 1527 spdk_for_each_thread(vhost_create_poll_group, 1528 init_cb, 1529 vhost_create_poll_group_done); 1530 return; 1531 err_out: 1532 init_cb(ret); 1533 } 1534 1535 static void 1536 _spdk_vhost_fini(void *arg1) 1537 { 1538 struct spdk_vhost_dev *vdev, *tmp; 1539 struct vhost_poll_group *pg, *tpg; 1540 1541 spdk_vhost_lock(); 1542 vdev = spdk_vhost_dev_next(NULL); 1543 while (vdev != NULL) { 1544 tmp = spdk_vhost_dev_next(vdev); 1545 spdk_vhost_dev_remove(vdev); 1546 /* don't care if it fails, there's nothing we can do for now */ 1547 vdev = tmp; 1548 } 1549 spdk_vhost_unlock(); 1550 1551 /* All devices are removed now. */ 1552 sem_destroy(&g_dpdk_sem); 1553 spdk_cpuset_free(g_tmp_cpuset); 1554 TAILQ_FOREACH_SAFE(pg, &g_poll_groups, tailq, tpg) { 1555 TAILQ_REMOVE(&g_poll_groups, pg, tailq); 1556 free(pg); 1557 } 1558 g_fini_cpl_cb(); 1559 } 1560 1561 static void * 1562 session_shutdown(void *arg) 1563 { 1564 struct spdk_vhost_dev *vdev = NULL; 1565 1566 TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) { 1567 rte_vhost_driver_unregister(vdev->path); 1568 vdev->registered = false; 1569 } 1570 1571 SPDK_INFOLOG(SPDK_LOG_VHOST, "Exiting\n"); 1572 spdk_thread_send_msg(g_vhost_init_thread, _spdk_vhost_fini, NULL); 1573 return NULL; 1574 } 1575 1576 void 1577 spdk_vhost_fini(spdk_vhost_fini_cb fini_cb) 1578 { 1579 pthread_t tid; 1580 int rc; 1581 1582 assert(spdk_get_thread() == g_vhost_init_thread); 1583 g_fini_cpl_cb = fini_cb; 1584 1585 /* rte_vhost API for removing sockets is not asynchronous. Since it may call SPDK 1586 * ops for stopping a device or removing a connection, we need to call it from 1587 * a separate thread to avoid deadlock. 1588 */ 1589 rc = pthread_create(&tid, NULL, &session_shutdown, NULL); 1590 if (rc < 0) { 1591 SPDK_ERRLOG("Failed to start session shutdown thread (%d): %s\n", rc, spdk_strerror(rc)); 1592 abort(); 1593 } 1594 pthread_detach(tid); 1595 } 1596 1597 void 1598 spdk_vhost_config_json(struct spdk_json_write_ctx *w) 1599 { 1600 struct spdk_vhost_dev *vdev; 1601 uint32_t delay_base_us; 1602 uint32_t iops_threshold; 1603 1604 spdk_json_write_array_begin(w); 1605 1606 spdk_vhost_lock(); 1607 vdev = spdk_vhost_dev_next(NULL); 1608 while (vdev != NULL) { 1609 vdev->backend->write_config_json(vdev, w); 1610 1611 spdk_vhost_get_coalescing(vdev, &delay_base_us, &iops_threshold); 1612 if (delay_base_us) { 1613 spdk_json_write_object_begin(w); 1614 spdk_json_write_named_string(w, "method", "set_vhost_controller_coalescing"); 1615 1616 spdk_json_write_named_object_begin(w, "params"); 1617 spdk_json_write_named_string(w, "ctrlr", vdev->name); 1618 spdk_json_write_named_uint32(w, "delay_base_us", delay_base_us); 1619 spdk_json_write_named_uint32(w, "iops_threshold", iops_threshold); 1620 spdk_json_write_object_end(w); 1621 1622 spdk_json_write_object_end(w); 1623 } 1624 vdev = spdk_vhost_dev_next(vdev); 1625 } 1626 spdk_vhost_unlock(); 1627 1628 spdk_json_write_array_end(w); 1629 } 1630 1631 SPDK_LOG_REGISTER_COMPONENT("vhost", SPDK_LOG_VHOST) 1632 SPDK_LOG_REGISTER_COMPONENT("vhost_ring", SPDK_LOG_VHOST_RING) 1633