1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2020 Intel Corporation.
3 * All rights reserved.
4 */
5
6 #ifndef __IDXD_H__
7 #define __IDXD_H__
8
9 #include "spdk/stdinc.h"
10
11 #include "spdk/idxd.h"
12 #include "spdk/queue.h"
13 #include "spdk/mmio.h"
14 #include "spdk/idxd_spec.h"
15
16 #ifdef __cplusplus
17 extern "C" {
18 #endif
19
20 /* TODO: get the gcc intrinsic to work. */
21 #define nop() asm volatile ("nop")
movdir64b(void * dst,const void * src)22 static inline void movdir64b(void *dst, const void *src)
23 {
24 asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
25 : "=m"(*(char *)dst)
26 : "d"(src), "a"(dst));
27 }
28
29 #define IDXD_REGISTER_TIMEOUT_US 50
30 #define IDXD_DRAIN_TIMEOUT_US 500000
31
32 #define WQ_MODE_DEDICATED 1
33
34 /* TODO: consider setting the max per batch limit via RPC. */
35
36 #define LOG2_WQ_MAX_XFER 30 /* 2^30 = 1073741824 */
37 #define WQ_PRIORITY_1 1
38 #define IDXD_MAX_QUEUES 64
39
40 enum idxd_dev {
41 IDXD_DEV_TYPE_DSA = 0,
42 IDXD_DEV_TYPE_IAA = 1,
43 };
44
45 /* Each pre-allocated batch structure goes on a per channel list and
46 * contains the memory for both user descriptors.
47 */
48 struct idxd_batch {
49 struct idxd_hw_desc *user_desc;
50 struct idxd_ops *user_ops;
51 uint64_t user_desc_addr;
52 uint16_t index;
53 uint16_t refcnt;
54 uint16_t size;
55 struct spdk_idxd_io_channel *chan;
56 TAILQ_ENTRY(idxd_batch) link;
57 };
58
59 struct device_config {
60 uint8_t config_num;
61 uint8_t num_groups;
62 uint16_t total_wqs;
63 uint16_t total_engines;
64 };
65
66 struct idxd_ops;
67
68 struct spdk_idxd_io_channel {
69 struct spdk_idxd_device *idxd;
70 /* The portal is the address that we write descriptors to for submission. */
71 void *portal;
72 uint32_t portal_offset;
73
74 bool pasid_enabled;
75
76 /* The currently open batch */
77 struct idxd_batch *batch;
78
79 /*
80 * User descriptors (those included in a batch) are managed independently from
81 * data descriptors and are located in the batch structure.
82 */
83 void *desc_base;
84 STAILQ_HEAD(, idxd_ops) ops_pool;
85 /* Current list of outstanding operations to poll. */
86 STAILQ_HEAD(op_head, idxd_ops) ops_outstanding;
87 void *ops_base;
88
89 TAILQ_HEAD(, idxd_batch) batch_pool;
90 void *batch_base;
91 };
92
93 struct pci_dev_id {
94 int vendor_id;
95 int device_id;
96 };
97
98 /*
99 * This struct wraps the hardware completion record which is 32 bytes in
100 * size and must be 32 byte aligned.
101 */
102 struct idxd_ops {
103 union {
104 struct dsa_hw_comp_record hw;
105 struct iaa_hw_comp_record iaa_hw;
106 };
107 void *cb_arg;
108 spdk_idxd_req_cb cb_fn;
109 struct idxd_batch *batch;
110 struct idxd_hw_desc *desc;
111 union {
112 uint32_t *crc_dst;
113 uint32_t *output_size;
114 };
115 struct idxd_ops *parent;
116 uint32_t count;
117 STAILQ_ENTRY(idxd_ops) link;
118 };
119 SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 128, "size mismatch");
120
121 struct spdk_idxd_impl {
122 const char *name;
123 int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb,
124 spdk_idxd_probe_cb probe_cb);
125 void (*destruct)(struct spdk_idxd_device *idxd);
126 void (*dump_sw_error)(struct spdk_idxd_device *idxd, void *portal);
127 char *(*portal_get_addr)(struct spdk_idxd_device *idxd);
128
129 STAILQ_ENTRY(spdk_idxd_impl) link;
130 };
131
132 struct spdk_idxd_device {
133 struct spdk_idxd_impl *impl;
134 void *portal;
135 uint32_t socket_id;
136 uint32_t num_channels;
137 uint32_t total_wq_size;
138 uint32_t chan_per_device;
139 uint16_t batch_size;
140 pthread_mutex_t num_channels_lock;
141 bool pasid_enabled;
142 enum idxd_dev type;
143 struct iaa_aecs *aecs;
144 uint64_t aecs_addr;
145 uint32_t version;
146 };
147
148 void idxd_impl_register(struct spdk_idxd_impl *impl);
149
150 #define SPDK_IDXD_IMPL_REGISTER(name, impl) \
151 static void __attribute__((constructor)) idxd_impl_register_##name(void) \
152 { \
153 idxd_impl_register(impl); \
154 }
155
156 #ifdef __cplusplus
157 }
158 #endif
159
160 #endif /* __IDXD_H__ */
161