1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #ifndef _VFU_VIRTIO_INTERNAL_H 7 #define _VFU_VIRTIO_INTERNAL_H 8 9 #include <linux/virtio_config.h> 10 #include <linux/virtio_ring.h> 11 #include <linux/virtio_pci.h> 12 13 #include "spdk/vfu_target.h" 14 15 #define VIRTIO_HOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_F_VERSION_1) | \ 16 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \ 17 (1ULL << VIRTIO_F_RING_PACKED)) 18 19 /* virtio device layout: 20 * 21 * region 1: MSI-X Table 22 * region 2: MSI-X PBA 23 * region 4: virtio modern memory 64bits BAR 24 * Common configuration 0x0 - 0x1000 25 * ISR access 0x1000 - 0x2000 26 * Device specific configuration 0x2000 - 0x3000 27 * Notifications 0x3000 - 0x4000 28 */ 29 #define VIRTIO_PCI_COMMON_CFG_OFFSET (0x0) 30 #define VIRTIO_PCI_COMMON_CFG_LENGTH (0x1000) 31 #define VIRTIO_PCI_ISR_ACCESS_OFFSET (VIRTIO_PCI_COMMON_CFG_OFFSET + VIRTIO_PCI_COMMON_CFG_LENGTH) 32 #define VIRTIO_PCI_ISR_ACCESS_LENGTH (0x1000) 33 #define VIRTIO_PCI_SPECIFIC_CFG_OFFSET (VIRTIO_PCI_ISR_ACCESS_OFFSET + VIRTIO_PCI_ISR_ACCESS_LENGTH) 34 #define VIRTIO_PCI_SPECIFIC_CFG_LENGTH (0x1000) 35 #define VIRTIO_PCI_NOTIFICATIONS_OFFSET (VIRTIO_PCI_SPECIFIC_CFG_OFFSET + VIRTIO_PCI_SPECIFIC_CFG_LENGTH) 36 #define VIRTIO_PCI_NOTIFICATIONS_LENGTH (0x1000) 37 38 #define VIRTIO_PCI_BAR4_LENGTH (VIRTIO_PCI_NOTIFICATIONS_OFFSET + VIRTIO_PCI_NOTIFICATIONS_LENGTH) 39 40 #define VIRTIO_DEV_MAX_IOVS (129) 41 /* Maximum number of requests which can be processed one time */ 42 #define VIRTIO_DEV_VRING_MAX_REQS (32) 43 /* Maximum number of queues can be supported by virtio device */ 44 #define VIRTIO_DEV_MAX_VQS (64) 45 /* Default queue size */ 46 #define VIRTIO_VQ_DEFAULT_SIZE (128) 47 /* Maximum queue size */ 48 #define VIRTIO_VQ_MAX_SIZE (1024) 49 50 struct vfu_virtio_endpoint; 51 struct vfu_virtio_req; 52 53 struct virtio_pci_cfg { 54 /* Common PCI configuration */ 55 uint32_t guest_feat_lo; 56 uint32_t guest_feat_hi; 57 58 /* Negotiated feature bits */ 59 uint64_t guest_features; 60 61 uint32_t host_feature_select; 62 uint32_t guest_feature_select; 63 64 uint16_t msix_config; 65 uint8_t device_status; 66 uint8_t config_generation; 67 uint16_t queue_select; 68 69 /* ISR access */ 70 uint8_t isr; 71 }; 72 73 enum vfu_vq_state { 74 VFU_VQ_CREATED = 0, 75 VFU_VQ_ACTIVE, 76 VFU_VQ_INACTIVE, 77 }; 78 79 struct q_mapping { 80 /* iov of local process mapping. */ 81 struct iovec iov; 82 /* Stored sg, needed for unmap. */ 83 dma_sg_t *sg; 84 /* physical address */ 85 uint64_t phys_addr; 86 /* virtual address */ 87 union { 88 void *addr; 89 90 struct vring_desc *desc; 91 struct vring_packed_desc *desc_packed; 92 93 struct vring_avail *avail; 94 struct vring_packed_desc_event *driver_event; 95 96 struct vring_used *used; 97 struct vring_packed_desc_event *device_event; 98 }; 99 /* size in bytes */ 100 uint64_t len; 101 }; 102 103 struct vfu_virtio_vq { 104 /* Read Only */ 105 uint16_t id; 106 uint16_t qsize; 107 108 bool enabled; 109 uint16_t vector; 110 111 enum vfu_vq_state q_state; 112 STAILQ_HEAD(, vfu_virtio_req) free_reqs; 113 114 uint32_t desc_lo; 115 uint32_t desc_hi; 116 uint32_t avail_lo; 117 uint32_t avail_hi; 118 uint32_t used_lo; 119 uint32_t used_hi; 120 121 struct q_mapping avail; 122 struct q_mapping used; 123 struct q_mapping desc; 124 125 uint16_t last_avail_idx; 126 uint16_t last_used_idx; 127 128 struct { 129 /* To mark a descriptor as available in packed ring 130 * Equal to avail_wrap_counter in spec. 131 */ 132 uint8_t avail_phase : 1; 133 /* To mark a descriptor as used in packed ring 134 * Equal to used_wrap_counter in spec. 135 */ 136 uint8_t used_phase : 1; 137 uint8_t padding : 5; 138 bool packed_ring : 1; 139 } packed; 140 141 /* Request count from last event */ 142 uint16_t used_req_cnt; 143 /* Next time when we need to send event */ 144 uint64_t next_event_time; 145 }; 146 147 struct vfu_virtio_dev { 148 char name[SPDK_VFU_MAX_NAME_LEN]; 149 /* RO for Guest Driver */ 150 uint16_t num_queues; 151 /* Supported feature bits by host driver, RO for Guest Driver */ 152 uint64_t host_features; 153 154 struct virtio_pci_cfg cfg; 155 struct vfu_virtio_vq vqs[VIRTIO_DEV_MAX_VQS]; 156 157 struct vfu_virtio_endpoint *virtio_endpoint; 158 159 /* VIRTIO_DEV_MAX_VQS * 3 worth of dma_sg_size() */ 160 uint8_t sg[]; 161 }; 162 163 struct vfu_virtio_ops { 164 uint64_t (*get_device_features)(struct vfu_virtio_endpoint *virtio_endpoint); 165 struct vfu_virtio_req *(*alloc_req)(struct vfu_virtio_endpoint *virtio_endpoint, 166 struct vfu_virtio_vq *vq); 167 void (*free_req)(struct vfu_virtio_endpoint *virtio_endpoint, struct vfu_virtio_vq *vq, 168 struct vfu_virtio_req *req); 169 int (*exec_request)(struct vfu_virtio_endpoint *virtio_endpoint, struct vfu_virtio_vq *vq, 170 struct vfu_virtio_req *req); 171 int (*get_config)(struct vfu_virtio_endpoint *virtio_endpoint, char *buf, uint64_t offset, 172 uint64_t count); 173 int (*set_config)(struct vfu_virtio_endpoint *virtio_endpoint, char *buf, uint64_t offset, 174 uint64_t count); 175 int (*start_device)(struct vfu_virtio_endpoint *virtio_endpoint); 176 int (*stop_device)(struct vfu_virtio_endpoint *virtio_endpoint); 177 }; 178 179 struct vfu_virtio_endpoint { 180 struct vfu_virtio_dev *dev; 181 int devmem_fd; 182 volatile uint32_t *doorbells; 183 184 uint16_t num_queues; 185 uint16_t qsize; 186 bool packed_ring; 187 188 uint32_t coalescing_delay_us; 189 190 struct spdk_vfu_endpoint *endpoint; 191 struct spdk_thread *thread; 192 193 struct vfu_virtio_ops virtio_ops; 194 195 /* quiesce poller */ 196 uint32_t io_outstanding; 197 bool quiesce_in_progress; 198 struct spdk_poller *quiesce_poller; 199 }; 200 201 struct vfu_virtio_req { 202 struct vfu_virtio_dev *dev; 203 struct vfu_virtio_vq *vq; 204 205 STAILQ_ENTRY(vfu_virtio_req) link; 206 207 uint32_t payload_size; 208 uint32_t used_len; 209 210 /* split vring */ 211 uint16_t req_idx; 212 /* packed vring */ 213 uint16_t buffer_id; 214 uint16_t num_descs; 215 216 uint16_t iovcnt; 217 struct iovec iovs[VIRTIO_DEV_MAX_IOVS + 1]; 218 uint8_t desc_writeable[VIRTIO_DEV_MAX_IOVS + 1]; 219 220 struct iovec *indirect_iov; 221 dma_sg_t *indirect_sg; 222 223 /* VIRIO_DEV_MAX_IOVS + 1 worth of dma_sg_size() */ 224 uint8_t sg[]; 225 }; 226 227 static inline bool 228 virtio_guest_has_feature(struct vfu_virtio_dev *dev, uint32_t feature_bit) 229 { 230 assert(feature_bit <= 64); 231 232 return !!(dev->cfg.guest_features & (1ULL << feature_bit)); 233 } 234 235 static inline uint64_t 236 virtio_queue_desc_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq) 237 { 238 return sizeof(struct vring_desc) * vq->qsize; 239 } 240 241 static inline uint64_t 242 virtio_queue_avail_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq) 243 { 244 uint16_t event_size; 245 246 if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) { 247 return sizeof(struct vring_packed_desc_event); 248 } 249 250 event_size = virtio_guest_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 251 return (sizeof(struct vring_avail) + sizeof(uint16_t) * vq->qsize 252 + event_size); 253 } 254 255 static inline uint64_t 256 virtio_queue_used_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq) 257 { 258 uint16_t event_size; 259 260 if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) { 261 return sizeof(struct vring_packed_desc_event); 262 } 263 264 event_size = virtio_guest_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 265 return (sizeof(struct vring_used) + sizeof(struct vring_used_elem) * vq->qsize 266 + event_size); 267 } 268 269 static inline bool 270 virtio_queue_event_is_suppressed(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq) 271 { 272 bool is_suppressed = false; 273 274 if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) { 275 is_suppressed = vq->avail.driver_event->flags & VRING_PACKED_EVENT_FLAG_DISABLE; 276 } else { 277 is_suppressed = vq->avail.avail->flags & VRING_AVAIL_F_NO_INTERRUPT; 278 279 } 280 281 return is_suppressed; 282 } 283 284 static inline bool 285 virtio_dev_is_started(struct vfu_virtio_dev *dev) 286 { 287 return !!(dev->cfg.device_status & VIRTIO_CONFIG_S_DRIVER_OK); 288 } 289 290 static inline bool 291 virtio_vring_split_desc_is_indirect(struct vring_desc *desc) 292 { 293 return !!(desc->flags & VRING_DESC_F_INDIRECT); 294 } 295 296 static inline bool 297 virtio_vring_packed_desc_is_indirect(struct vring_packed_desc *desc) 298 { 299 return !!(desc->flags & VRING_DESC_F_INDIRECT); 300 } 301 302 static inline bool 303 virtio_vring_split_desc_is_wr(struct vring_desc *desc) 304 { 305 return !!(desc->flags & VRING_DESC_F_WRITE); 306 } 307 308 static inline bool 309 virtio_vring_packed_desc_is_wr(struct vring_packed_desc *desc) 310 { 311 return !!(desc->flags & VRING_DESC_F_WRITE); 312 } 313 314 static inline bool 315 virtio_vring_packed_is_avail(struct vring_packed_desc *desc, bool avail_phase) 316 { 317 bool avail_flag, used_flag; 318 uint16_t flags = desc->flags; 319 320 avail_flag = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 321 used_flag = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 322 323 /* To mark a desc as available, the driver sets the F_AVAIL bit in flags 324 * to match the internal avail wrap counter. It also sets the F_USED bit to 325 * match the inverse value but it's not mandatory. 326 */ 327 return (avail_flag != used_flag) && (avail_flag == avail_phase); 328 } 329 330 static inline bool 331 virtio_vring_packed_is_used(struct vring_packed_desc *desc, bool used_phase) 332 { 333 bool avail_flag, used_flag; 334 uint16_t flags = desc->flags; 335 336 avail_flag = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 337 used_flag = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 338 339 /* When the descriptor is used, two flags in descriptor 340 * avail flag and used flag are set to equal 341 * and used flag value == used_wrap_counter. 342 */ 343 return (used_flag == avail_flag) && (used_flag == used_phase); 344 } 345 346 static inline bool 347 virtio_req_iov_is_wr(struct vfu_virtio_req *req, uint32_t iov_num) 348 { 349 assert(iov_num <= VIRTIO_DEV_MAX_IOVS); 350 return req->desc_writeable[iov_num]; 351 } 352 353 static inline struct vfu_virtio_req * 354 vfu_virtio_vq_alloc_req(struct vfu_virtio_endpoint *endpoint, struct vfu_virtio_vq *vq) 355 { 356 assert(endpoint->virtio_ops.alloc_req != NULL); 357 return endpoint->virtio_ops.alloc_req(endpoint, vq); 358 } 359 360 static inline void 361 vfu_virtio_vq_free_req(struct vfu_virtio_endpoint *endpoint, struct vfu_virtio_vq *vq, 362 struct vfu_virtio_req *req) 363 { 364 assert(endpoint->virtio_ops.free_req); 365 endpoint->virtio_ops.free_req(endpoint, vq, req); 366 } 367 368 void virtio_vq_used_ring_split_enqueue(struct vfu_virtio_vq *vq, uint16_t req_idx, 369 uint32_t used_len); 370 void virtio_vq_used_ring_packed_enqueue(struct vfu_virtio_vq *vq, uint16_t buffer_id, 371 uint32_t num_descs, uint32_t used_len); 372 struct vfu_virtio_req *virito_dev_packed_ring_get_next_avail_req(struct vfu_virtio_dev *dev, 373 struct vfu_virtio_vq *vq); 374 struct vfu_virtio_req *virito_dev_split_ring_get_next_avail_req(struct vfu_virtio_dev *dev, 375 struct vfu_virtio_vq *vq); 376 377 int vfu_virtio_quiesce_cb(struct spdk_vfu_endpoint *endpoint); 378 379 void vfu_virtio_dev_put_req(struct vfu_virtio_req *req); 380 void vfu_virtio_finish_req(struct vfu_virtio_req *req); 381 void vfu_virtio_vq_flush_irq(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq); 382 int vfu_virito_dev_process_packed_ring(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq); 383 int vfu_virito_dev_process_split_ring(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq); 384 void vfu_virtio_notify_config(struct vfu_virtio_endpoint *virtio_endpoint); 385 int vfu_virtio_endpoint_setup(struct vfu_virtio_endpoint *virtio_endpoint, 386 struct spdk_vfu_endpoint *endpoint, 387 char *basename, const char *endpoint_name, 388 struct vfu_virtio_ops *ops); 389 int vfu_virtio_endpoint_destruct(struct vfu_virtio_endpoint *virtio_endpoint); 390 void vfu_virtio_get_device_info(struct vfu_virtio_endpoint *virtio_endpoint, 391 struct spdk_vfu_pci_device *device_info); 392 int vfu_virtio_attach_device(struct spdk_vfu_endpoint *endpoint); 393 int vfu_virtio_detach_device(struct spdk_vfu_endpoint *endpoint); 394 uint16_t vfu_virtio_get_vendor_capability(struct spdk_vfu_endpoint *endpoint, char *buf, 395 uint16_t buf_len, uint16_t idx); 396 int vfu_virtio_post_memory_add(struct spdk_vfu_endpoint *endpoint, void *map_start, void *map_end); 397 int vfu_virtio_pre_memory_remove(struct spdk_vfu_endpoint *endpoint, void *map_start, 398 void *map_end); 399 int vfu_virtio_pci_reset_cb(struct spdk_vfu_endpoint *endpoint); 400 int vfu_virtio_blk_add_bdev(const char *name, const char *bdev_name, 401 uint16_t num_queues, uint16_t qsize, bool packed_ring); 402 /* virtio_scsi */ 403 int vfu_virtio_scsi_add_target(const char *name, uint8_t scsi_target_num, 404 const char *bdev_name); 405 int vfu_virtio_scsi_remove_target(const char *name, uint8_t scsi_target_num); 406 int vfu_virtio_scsi_set_options(const char *name, uint16_t num_io_queues, uint16_t qsize, 407 bool packed_ring); 408 #endif 409