1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #ifndef __IDXD_H__ 7 #define __IDXD_H__ 8 9 #include "spdk/stdinc.h" 10 11 #include "spdk/idxd.h" 12 #include "spdk/queue.h" 13 #include "spdk/mmio.h" 14 #include "spdk/idxd_spec.h" 15 16 #ifdef __cplusplus 17 extern "C" { 18 #endif 19 20 /* TODO: get the gcc intrinsic to work. */ 21 #define nop() asm volatile ("nop") 22 static inline void movdir64b(void *dst, const void *src) 23 { 24 asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" 25 : "=m"(*(char *)dst) 26 : "d"(src), "a"(dst)); 27 } 28 29 #define IDXD_REGISTER_TIMEOUT_US 50 30 #define IDXD_DRAIN_TIMEOUT_US 500000 31 32 #define WQ_MODE_DEDICATED 1 33 34 /* TODO: consider setting the max per batch limit via RPC. */ 35 36 #define LOG2_WQ_MAX_XFER 30 /* 2^30 = 1073741824 */ 37 #define WQ_PRIORITY_1 1 38 #define IDXD_MAX_QUEUES 64 39 40 enum idxd_dev { 41 IDXD_DEV_TYPE_DSA = 0, 42 IDXD_DEV_TYPE_IAA = 1, 43 }; 44 45 /* Each pre-allocated batch structure goes on a per channel list and 46 * contains the memory for both user descriptors. 47 */ 48 struct idxd_batch { 49 struct idxd_hw_desc *user_desc; 50 struct idxd_ops *user_ops; 51 uint64_t user_desc_addr; 52 uint16_t index; 53 uint16_t refcnt; 54 uint16_t size; 55 struct spdk_idxd_io_channel *chan; 56 TAILQ_ENTRY(idxd_batch) link; 57 }; 58 59 struct device_config { 60 uint8_t config_num; 61 uint8_t num_groups; 62 uint16_t total_wqs; 63 uint16_t total_engines; 64 }; 65 66 struct idxd_ops; 67 68 struct spdk_idxd_io_channel { 69 struct spdk_idxd_device *idxd; 70 /* The portal is the address that we write descriptors to for submission. */ 71 void *portal; 72 uint32_t portal_offset; 73 74 bool pasid_enabled; 75 76 /* The currently open batch */ 77 struct idxd_batch *batch; 78 79 /* 80 * User descriptors (those included in a batch) are managed independently from 81 * data descriptors and are located in the batch structure. 82 */ 83 void *desc_base; 84 STAILQ_HEAD(, idxd_ops) ops_pool; 85 /* Current list of outstanding operations to poll. */ 86 STAILQ_HEAD(op_head, idxd_ops) ops_outstanding; 87 void *ops_base; 88 89 TAILQ_HEAD(, idxd_batch) batch_pool; 90 void *batch_base; 91 }; 92 93 struct pci_dev_id { 94 int vendor_id; 95 int device_id; 96 }; 97 98 /* 99 * This struct wraps the hardware completion record which is 32 bytes in 100 * size and must be 32 byte aligned. 101 */ 102 struct idxd_ops { 103 union { 104 struct dsa_hw_comp_record hw; 105 struct iaa_hw_comp_record iaa_hw; 106 }; 107 void *cb_arg; 108 spdk_idxd_req_cb cb_fn; 109 struct idxd_batch *batch; 110 struct idxd_hw_desc *desc; 111 union { 112 uint32_t *crc_dst; 113 uint32_t *output_size; 114 }; 115 struct idxd_ops *parent; 116 uint32_t count; 117 STAILQ_ENTRY(idxd_ops) link; 118 }; 119 SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 128, "size mismatch"); 120 121 struct spdk_idxd_impl { 122 const char *name; 123 int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb, 124 spdk_idxd_probe_cb probe_cb); 125 void (*destruct)(struct spdk_idxd_device *idxd); 126 void (*dump_sw_error)(struct spdk_idxd_device *idxd, void *portal); 127 char *(*portal_get_addr)(struct spdk_idxd_device *idxd); 128 129 STAILQ_ENTRY(spdk_idxd_impl) link; 130 }; 131 132 struct spdk_idxd_device { 133 struct spdk_idxd_impl *impl; 134 void *portal; 135 uint32_t socket_id; 136 uint32_t num_channels; 137 uint32_t total_wq_size; 138 uint32_t chan_per_device; 139 uint16_t batch_size; 140 pthread_mutex_t num_channels_lock; 141 bool pasid_enabled; 142 enum idxd_dev type; 143 struct iaa_aecs *aecs; 144 uint64_t aecs_addr; 145 uint32_t version; 146 }; 147 148 void idxd_impl_register(struct spdk_idxd_impl *impl); 149 150 #define SPDK_IDXD_IMPL_REGISTER(name, impl) \ 151 static void __attribute__((constructor)) idxd_impl_register_##name(void) \ 152 { \ 153 idxd_impl_register(impl); \ 154 } 155 156 #ifdef __cplusplus 157 } 158 #endif 159 160 #endif /* __IDXD_H__ */ 161