1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2020 Intel Corporation.
3 * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
4 * All rights reserved.
5 */
6
7 #include "accel_ioat.h"
8
9 #include "spdk/stdinc.h"
10
11 #include "spdk/accel_module.h"
12 #include "spdk/log.h"
13 #include "spdk/likely.h"
14
15 #include "spdk/env.h"
16 #include "spdk/event.h"
17 #include "spdk/thread.h"
18 #include "spdk/ioat.h"
19 #include "spdk/json.h"
20
21 static bool g_ioat_enable = false;
22 static bool g_ioat_initialized = false;
23
24 struct ioat_device {
25 struct spdk_ioat_chan *ioat;
26 bool is_allocated;
27 /** linked list pointer for device list */
28 TAILQ_ENTRY(ioat_device) tailq;
29 };
30
31 struct pci_device {
32 struct spdk_pci_device *pci_dev;
33 TAILQ_ENTRY(pci_device) tailq;
34 };
35
36 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices);
37 static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER;
38
39 static TAILQ_HEAD(, pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices);
40
41 struct ioat_io_channel {
42 struct spdk_ioat_chan *ioat_ch;
43 struct ioat_device *ioat_dev;
44 struct spdk_poller *poller;
45 };
46
47 static struct ioat_device *
ioat_allocate_device(void)48 ioat_allocate_device(void)
49 {
50 struct ioat_device *dev;
51
52 pthread_mutex_lock(&g_ioat_mutex);
53 TAILQ_FOREACH(dev, &g_devices, tailq) {
54 if (!dev->is_allocated) {
55 dev->is_allocated = true;
56 pthread_mutex_unlock(&g_ioat_mutex);
57 return dev;
58 }
59 }
60 pthread_mutex_unlock(&g_ioat_mutex);
61
62 return NULL;
63 }
64
65 static void
ioat_free_device(struct ioat_device * dev)66 ioat_free_device(struct ioat_device *dev)
67 {
68 pthread_mutex_lock(&g_ioat_mutex);
69 dev->is_allocated = false;
70 pthread_mutex_unlock(&g_ioat_mutex);
71 }
72
73 static int accel_ioat_init(void);
74 static void accel_ioat_exit(void *ctx);
75 static void accel_ioat_write_config_json(struct spdk_json_write_ctx *w);
76 static bool ioat_supports_opcode(enum spdk_accel_opcode opc);
77 static struct spdk_io_channel *ioat_get_io_channel(void);
78 static int ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task);
79
80 static size_t
accel_ioat_get_ctx_size(void)81 accel_ioat_get_ctx_size(void)
82 {
83 return sizeof(struct spdk_accel_task);
84 }
85
86 static struct spdk_accel_module_if g_ioat_module = {
87 .module_init = accel_ioat_init,
88 .module_fini = accel_ioat_exit,
89 .write_config_json = accel_ioat_write_config_json,
90 .get_ctx_size = accel_ioat_get_ctx_size,
91 .name = "ioat",
92 .supports_opcode = ioat_supports_opcode,
93 .get_io_channel = ioat_get_io_channel,
94 .submit_tasks = ioat_submit_tasks
95 };
96
97 static void
ioat_done(void * cb_arg)98 ioat_done(void *cb_arg)
99 {
100 struct spdk_accel_task *accel_task = cb_arg;
101
102 spdk_accel_task_complete(accel_task, 0);
103 }
104
105 static int
ioat_poll(void * arg)106 ioat_poll(void *arg)
107 {
108 struct spdk_ioat_chan *chan = arg;
109
110 return spdk_ioat_process_events(chan) != 0 ? SPDK_POLLER_BUSY :
111 SPDK_POLLER_IDLE;
112 }
113
114 static struct spdk_io_channel *ioat_get_io_channel(void);
115
116 static bool
ioat_supports_opcode(enum spdk_accel_opcode opc)117 ioat_supports_opcode(enum spdk_accel_opcode opc)
118 {
119 if (!g_ioat_initialized) {
120 assert(0);
121 return false;
122 }
123
124 switch (opc) {
125 case SPDK_ACCEL_OPC_COPY:
126 case SPDK_ACCEL_OPC_FILL:
127 return true;
128 default:
129 return false;
130 }
131 }
132
133 static int
ioat_submit_fill(struct ioat_io_channel * ioat_ch,struct spdk_accel_task * task)134 ioat_submit_fill(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task)
135 {
136 if (spdk_unlikely(task->d.iovcnt != 1)) {
137 return -EINVAL;
138 }
139
140 return spdk_ioat_build_fill(ioat_ch->ioat_ch, task, ioat_done,
141 task->d.iovs[0].iov_base, task->fill_pattern,
142 task->d.iovs[0].iov_len);
143 }
144
145 static int
ioat_submit_copy(struct ioat_io_channel * ioat_ch,struct spdk_accel_task * task)146 ioat_submit_copy(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task)
147 {
148 if (spdk_unlikely(task->d.iovcnt != 1 || task->s.iovcnt != 1)) {
149 return -EINVAL;
150 }
151
152 if (spdk_unlikely(task->d.iovs[0].iov_len != task->s.iovs[0].iov_len)) {
153 return -EINVAL;
154 }
155
156 return spdk_ioat_build_copy(ioat_ch->ioat_ch, task, ioat_done,
157 task->d.iovs[0].iov_base, task->s.iovs[0].iov_base,
158 task->d.iovs[0].iov_len);
159 }
160
161 static int
ioat_submit_tasks(struct spdk_io_channel * ch,struct spdk_accel_task * accel_task)162 ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task)
163 {
164 struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
165 struct spdk_accel_task *tmp;
166 int rc = 0;
167
168 do {
169 switch (accel_task->op_code) {
170 case SPDK_ACCEL_OPC_FILL:
171 rc = ioat_submit_fill(ioat_ch, accel_task);
172 break;
173 case SPDK_ACCEL_OPC_COPY:
174 rc = ioat_submit_copy(ioat_ch, accel_task);
175 break;
176 default:
177 assert(false);
178 break;
179 }
180
181 tmp = STAILQ_NEXT(accel_task, link);
182
183 /* Report any build errors via the callback now. */
184 if (rc) {
185 spdk_accel_task_complete(accel_task, rc);
186 }
187
188 accel_task = tmp;
189 } while (accel_task);
190
191 spdk_ioat_flush(ioat_ch->ioat_ch);
192
193 return 0;
194 }
195
196 static int
ioat_create_cb(void * io_device,void * ctx_buf)197 ioat_create_cb(void *io_device, void *ctx_buf)
198 {
199 struct ioat_io_channel *ch = ctx_buf;
200 struct ioat_device *ioat_dev;
201
202 ioat_dev = ioat_allocate_device();
203 if (ioat_dev == NULL) {
204 return -1;
205 }
206
207 ch->ioat_dev = ioat_dev;
208 ch->ioat_ch = ioat_dev->ioat;
209 ch->poller = SPDK_POLLER_REGISTER(ioat_poll, ch->ioat_ch, 0);
210
211 return 0;
212 }
213
214 static void
ioat_destroy_cb(void * io_device,void * ctx_buf)215 ioat_destroy_cb(void *io_device, void *ctx_buf)
216 {
217 struct ioat_io_channel *ch = ctx_buf;
218
219 ioat_free_device(ch->ioat_dev);
220 spdk_poller_unregister(&ch->poller);
221 }
222
223 static struct spdk_io_channel *
ioat_get_io_channel(void)224 ioat_get_io_channel(void)
225 {
226 return spdk_get_io_channel(&g_ioat_module);
227 }
228
229 static bool
probe_cb(void * cb_ctx,struct spdk_pci_device * pci_dev)230 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
231 {
232 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(pci_dev);
233 struct pci_device *pdev;
234
235 SPDK_INFOLOG(accel_ioat,
236 " Found matching device at %04x:%02x:%02x.%x vendor:0x%04x device:0x%04x\n",
237 pci_addr.domain,
238 pci_addr.bus,
239 pci_addr.dev,
240 pci_addr.func,
241 spdk_pci_device_get_vendor_id(pci_dev),
242 spdk_pci_device_get_device_id(pci_dev));
243
244 pdev = calloc(1, sizeof(*pdev));
245 if (pdev == NULL) {
246 return false;
247 }
248 pdev->pci_dev = pci_dev;
249 TAILQ_INSERT_TAIL(&g_pci_devices, pdev, tailq);
250
251 /* Claim the device in case conflict with other process */
252 if (spdk_pci_device_claim(pci_dev) < 0) {
253 return false;
254 }
255
256 return true;
257 }
258
259 static void
attach_cb(void * cb_ctx,struct spdk_pci_device * pci_dev,struct spdk_ioat_chan * ioat)260 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
261 {
262 struct ioat_device *dev;
263
264 dev = calloc(1, sizeof(*dev));
265 if (dev == NULL) {
266 SPDK_ERRLOG("Failed to allocate device struct\n");
267 return;
268 }
269
270 dev->ioat = ioat;
271 TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
272 }
273
274 void
accel_ioat_enable_probe(void)275 accel_ioat_enable_probe(void)
276 {
277 g_ioat_enable = true;
278 spdk_accel_module_list_add(&g_ioat_module);
279 }
280
281 static int
accel_ioat_init(void)282 accel_ioat_init(void)
283 {
284 if (!g_ioat_enable) {
285 assert(0);
286 return 0;
287 }
288
289 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
290 SPDK_ERRLOG("spdk_ioat_probe() failed\n");
291 return -1;
292 }
293
294 if (TAILQ_EMPTY(&g_devices)) {
295 return -ENODEV;
296 }
297
298 g_ioat_initialized = true;
299 spdk_io_device_register(&g_ioat_module, ioat_create_cb, ioat_destroy_cb,
300 sizeof(struct ioat_io_channel), "ioat_accel_module");
301 return 0;
302 }
303
304 static void
_device_unregister_cb(void * io_device)305 _device_unregister_cb(void *io_device)
306 {
307 struct ioat_device *dev = io_device;
308 struct pci_device *pci_dev;
309
310 while (!TAILQ_EMPTY(&g_devices)) {
311 dev = TAILQ_FIRST(&g_devices);
312 TAILQ_REMOVE(&g_devices, dev, tailq);
313 spdk_ioat_detach(dev->ioat);
314 free(dev);
315 }
316
317 while (!TAILQ_EMPTY(&g_pci_devices)) {
318 pci_dev = TAILQ_FIRST(&g_pci_devices);
319 TAILQ_REMOVE(&g_pci_devices, pci_dev, tailq);
320 spdk_pci_device_detach(pci_dev->pci_dev);
321 free(pci_dev);
322 }
323
324 g_ioat_initialized = false;
325
326 spdk_accel_module_finish();
327 }
328
329 static void
accel_ioat_exit(void * ctx)330 accel_ioat_exit(void *ctx)
331 {
332 if (g_ioat_initialized) {
333 spdk_io_device_unregister(&g_ioat_module, _device_unregister_cb);
334 } else {
335 spdk_accel_module_finish();
336 }
337 }
338
339 static void
accel_ioat_write_config_json(struct spdk_json_write_ctx * w)340 accel_ioat_write_config_json(struct spdk_json_write_ctx *w)
341 {
342 if (g_ioat_enable) {
343 spdk_json_write_object_begin(w);
344 spdk_json_write_named_string(w, "method", "ioat_scan_accel_module");
345 spdk_json_write_object_end(w);
346 }
347 }
348
349 SPDK_LOG_REGISTER_COMPONENT(accel_ioat)
350