1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef SPDK_VHOST_INTERNAL_H 35 #define SPDK_VHOST_INTERNAL_H 36 37 #include "spdk/stdinc.h" 38 39 #include <rte_vhost.h> 40 41 #include "spdk_internal/log.h" 42 #include "spdk/event.h" 43 #include "spdk/rpc.h" 44 45 #define SPDK_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE 46 47 #ifndef VHOST_USER_F_PROTOCOL_FEATURES 48 #define VHOST_USER_F_PROTOCOL_FEATURES 30 49 #endif 50 51 #ifndef VIRTIO_F_VERSION_1 52 #define VIRTIO_F_VERSION_1 32 53 #endif 54 55 #ifndef VIRTIO_BLK_F_MQ 56 #define VIRTIO_BLK_F_MQ 12 /* support more than one vq */ 57 #endif 58 59 #ifndef VIRTIO_BLK_F_CONFIG_WCE 60 #define VIRTIO_BLK_F_CONFIG_WCE 11 61 #endif 62 63 #define SPDK_VHOST_MAX_VQUEUES 256 64 #define SPDK_VHOST_MAX_VQ_SIZE 1024 65 66 #define SPDK_VHOST_SCSI_CTRLR_MAX_DEVS 8 67 68 #define SPDK_VHOST_IOVS_MAX 129 69 70 /* 71 * Rate at which stats are checked for interrupt coalescing. 72 */ 73 #define SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS 10 74 /* 75 * Default threshold at which interrupts start to be coalesced. 76 */ 77 #define SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD 60000 78 79 /* 80 * Currently coalescing is not used by default. 81 * Setting this to value > 0 here or by RPC will enable coalescing. 82 */ 83 #define SPDK_VHOST_COALESCING_DELAY_BASE_US 0 84 85 86 #define SPDK_VHOST_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \ 87 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ 88 (1ULL << VIRTIO_F_VERSION_1) | \ 89 (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \ 90 (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ 91 (1ULL << VIRTIO_RING_F_INDIRECT_DESC)) 92 93 #define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \ 94 (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY)) 95 96 struct spdk_vhost_virtqueue { 97 struct rte_vhost_vring vring; 98 void *tasks; 99 100 /* Request count from last stats check */ 101 uint32_t req_cnt; 102 103 /* Request count from last event */ 104 uint16_t used_req_cnt; 105 106 /* How long interrupt is delayed */ 107 uint32_t irq_delay_time; 108 109 /* Next time when we need to send event */ 110 uint64_t next_event_time; 111 112 } __attribute((aligned(SPDK_CACHE_LINE_SIZE))); 113 114 struct spdk_vhost_dev_backend { 115 uint64_t virtio_features; 116 uint64_t disabled_features; 117 118 /** 119 * Callbacks for starting and pausing the device. 120 * The first param is struct spdk_vhost_dev *. 121 * The second one is event context that has to be 122 * passed to spdk_vhost_dev_backend_event_done(). 123 */ 124 spdk_vhost_event_fn start_device; 125 spdk_vhost_event_fn stop_device; 126 127 int (*vhost_get_config)(struct spdk_vhost_dev *vdev, uint8_t *config, uint32_t len); 128 int (*vhost_set_config)(struct spdk_vhost_dev *vdev, uint8_t *config, 129 uint32_t offset, uint32_t size, uint32_t flags); 130 131 void (*dump_info_json)(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); 132 void (*write_config_json)(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); 133 int (*remove_device)(struct spdk_vhost_dev *vdev); 134 }; 135 136 struct spdk_vhost_dev { 137 struct rte_vhost_memory *mem; 138 char *name; 139 char *path; 140 141 /* Unique device ID. */ 142 unsigned id; 143 144 /* rte_vhost device ID. */ 145 int vid; 146 int task_cnt; 147 int32_t lcore; 148 struct spdk_cpuset *cpumask; 149 bool registered; 150 151 const struct spdk_vhost_dev_backend *backend; 152 153 /* Saved orginal values used to setup coalescing to avoid integer 154 * rounding issues during save/load config. 155 */ 156 uint32_t coalescing_delay_us; 157 uint32_t coalescing_iops_threshold; 158 159 uint32_t coalescing_delay_time_base; 160 161 /* Threshold when event coalescing for virtqueue will be turned on. */ 162 uint32_t coalescing_io_rate_threshold; 163 164 /* Next time when stats for event coalescing will be checked. */ 165 uint64_t next_stats_check_time; 166 167 /* Interval used for event coalescing checking. */ 168 uint64_t stats_check_interval; 169 170 uint16_t max_queues; 171 172 uint64_t negotiated_features; 173 174 struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES]; 175 176 TAILQ_ENTRY(spdk_vhost_dev) tailq; 177 }; 178 179 struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name); 180 181 void *spdk_vhost_gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr, uint64_t len); 182 183 uint16_t spdk_vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *vq, uint16_t *reqs, 184 uint16_t reqs_len); 185 186 /** 187 * Get a virtio descriptor at given index in given virtqueue. 188 * The descriptor will provide access to the entire descriptor 189 * chain. The subsequent descriptors are accesible via 190 * \c spdk_vhost_vring_desc_get_next. 191 * \param vdev vhost device 192 * \param vq virtqueue 193 * \param req_idx descriptor index 194 * \param desc pointer to be set to the descriptor 195 * \param desc_table descriptor table to be used with 196 * \c spdk_vhost_vring_desc_get_next. This might be either 197 * default virtqueue descriptor table or per-chain indirect 198 * table. 199 * \param desc_table_size size of the *desc_table* 200 * \return 0 on success, -1 if given index is invalid. 201 * If -1 is returned, the content of params is undefined. 202 */ 203 int spdk_vhost_vq_get_desc(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *vq, 204 uint16_t req_idx, struct vring_desc **desc, struct vring_desc **desc_table, 205 uint32_t *desc_table_size); 206 207 /** 208 * Send IRQ/call client (if pending) for \c vq. 209 * \param vdev vhost device 210 * \param vq virtqueue 211 * \return 212 * 0 - if no interrupt was signalled 213 * 1 - if interrupt was signalled 214 */ 215 int spdk_vhost_vq_used_signal(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *vq); 216 217 218 /** 219 * Send IRQs for all queues that need to be signaled. 220 * \param vdev vhost device 221 * \param vq virtqueue 222 */ 223 void spdk_vhost_dev_used_signal(struct spdk_vhost_dev *vdev); 224 225 void spdk_vhost_vq_used_ring_enqueue(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *vq, 226 uint16_t id, uint32_t len); 227 228 /** 229 * Get subsequent descriptor from given table. 230 * \param desc current descriptor, will be set to the 231 * next descriptor (NULL in case this is the last 232 * descriptor in the chain or the next desc is invalid) 233 * \param desc_table descriptor table 234 * \param desc_table_size size of the *desc_table* 235 * \return 0 on success, -1 if given index is invalid 236 * The *desc* param will be set regardless of the 237 * return value. 238 */ 239 int spdk_vhost_vring_desc_get_next(struct vring_desc **desc, 240 struct vring_desc *desc_table, uint32_t desc_table_size); 241 bool spdk_vhost_vring_desc_is_wr(struct vring_desc *cur_desc); 242 243 int spdk_vhost_vring_desc_to_iov(struct spdk_vhost_dev *vdev, struct iovec *iov, 244 uint16_t *iov_index, const struct vring_desc *desc); 245 246 static inline bool __attribute__((always_inline)) 247 spdk_vhost_dev_has_feature(struct spdk_vhost_dev *vdev, unsigned feature_id) 248 { 249 return vdev->negotiated_features & (1ULL << feature_id); 250 } 251 252 int spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, 253 const struct spdk_vhost_dev_backend *backend); 254 int spdk_vhost_dev_unregister(struct spdk_vhost_dev *vdev); 255 256 int spdk_vhost_scsi_controller_construct(void); 257 int spdk_vhost_blk_controller_construct(void); 258 void spdk_vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w); 259 void spdk_vhost_dev_backend_event_done(void *event_ctx, int response); 260 void spdk_vhost_lock(void); 261 void spdk_vhost_unlock(void); 262 int spdk_remove_vhost_controller(struct spdk_vhost_dev *vdev); 263 int spdk_vhost_nvme_admin_passthrough(int vid, void *cmd, void *cqe, void *buf); 264 int spdk_vhost_nvme_set_cq_call(int vid, uint16_t qid, int fd); 265 int spdk_vhost_nvme_get_cap(int vid, uint64_t *cap); 266 int spdk_vhost_nvme_controller_construct(void); 267 int spdk_vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_t io_queues); 268 int spdk_vhost_nvme_dev_remove(struct spdk_vhost_dev *vdev); 269 int spdk_vhost_nvme_dev_add_ns(struct spdk_vhost_dev *vdev, 270 const char *bdev_name); 271 272 #endif /* SPDK_VHOST_INTERNAL_H */ 273