xref: /spdk/module/accel/ioat/accel_ioat.c (revision 12fbe739a31b09aff0d05f354d4f3bbef99afc55) 
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2020 Intel Corporation.
3  *   Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
4  *   All rights reserved.
5  */
6 
7 #include "accel_ioat.h"
8 
9 #include "spdk/stdinc.h"
10 
11 #include "spdk/accel_module.h"
12 #include "spdk/log.h"
13 #include "spdk/likely.h"
14 
15 #include "spdk/env.h"
16 #include "spdk/event.h"
17 #include "spdk/thread.h"
18 #include "spdk/ioat.h"
19 
20 static bool g_ioat_enable = false;
21 static bool g_ioat_initialized = false;
22 
23 struct ioat_device {
24 	struct spdk_ioat_chan *ioat;
25 	bool is_allocated;
26 	/** linked list pointer for device list */
27 	TAILQ_ENTRY(ioat_device) tailq;
28 };
29 
30 struct pci_device {
31 	struct spdk_pci_device *pci_dev;
32 	TAILQ_ENTRY(pci_device) tailq;
33 };
34 
35 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices);
36 static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER;
37 
38 static TAILQ_HEAD(, pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices);
39 
40 struct ioat_io_channel {
41 	struct spdk_ioat_chan		*ioat_ch;
42 	struct ioat_device		*ioat_dev;
43 	struct spdk_poller		*poller;
44 };
45 
46 static struct ioat_device *
47 ioat_allocate_device(void)
48 {
49 	struct ioat_device *dev;
50 
51 	pthread_mutex_lock(&g_ioat_mutex);
52 	TAILQ_FOREACH(dev, &g_devices, tailq) {
53 		if (!dev->is_allocated) {
54 			dev->is_allocated = true;
55 			pthread_mutex_unlock(&g_ioat_mutex);
56 			return dev;
57 		}
58 	}
59 	pthread_mutex_unlock(&g_ioat_mutex);
60 
61 	return NULL;
62 }
63 
64 static void
65 ioat_free_device(struct ioat_device *dev)
66 {
67 	pthread_mutex_lock(&g_ioat_mutex);
68 	dev->is_allocated = false;
69 	pthread_mutex_unlock(&g_ioat_mutex);
70 }
71 
72 static int accel_ioat_init(void);
73 static void accel_ioat_exit(void *ctx);
74 static bool ioat_supports_opcode(enum spdk_accel_opcode opc);
75 static struct spdk_io_channel *ioat_get_io_channel(void);
76 static int ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task);
77 
78 static size_t
79 accel_ioat_get_ctx_size(void)
80 {
81 	return sizeof(struct spdk_accel_task);
82 }
83 
84 static struct spdk_accel_module_if g_ioat_module = {
85 	.module_init = accel_ioat_init,
86 	.module_fini = accel_ioat_exit,
87 	.write_config_json = NULL,
88 	.get_ctx_size = accel_ioat_get_ctx_size,
89 	.name			= "ioat",
90 	.supports_opcode	= ioat_supports_opcode,
91 	.get_io_channel		= ioat_get_io_channel,
92 	.submit_tasks		= ioat_submit_tasks
93 };
94 
95 static void
96 ioat_done(void *cb_arg)
97 {
98 	struct spdk_accel_task *accel_task = cb_arg;
99 
100 	spdk_accel_task_complete(accel_task, 0);
101 }
102 
103 static int
104 ioat_poll(void *arg)
105 {
106 	struct spdk_ioat_chan *chan = arg;
107 
108 	return spdk_ioat_process_events(chan) != 0 ? SPDK_POLLER_BUSY :
109 	       SPDK_POLLER_IDLE;
110 }
111 
112 static struct spdk_io_channel *ioat_get_io_channel(void);
113 
114 static bool
115 ioat_supports_opcode(enum spdk_accel_opcode opc)
116 {
117 	if (!g_ioat_initialized) {
118 		assert(0);
119 		return false;
120 	}
121 
122 	switch (opc) {
123 	case SPDK_ACCEL_OPC_COPY:
124 	case SPDK_ACCEL_OPC_FILL:
125 		return true;
126 	default:
127 		return false;
128 	}
129 }
130 
131 static int
132 ioat_submit_fill(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task)
133 {
134 	if (spdk_unlikely(task->d.iovcnt != 1)) {
135 		return -EINVAL;
136 	}
137 
138 	return spdk_ioat_build_fill(ioat_ch->ioat_ch, task, ioat_done,
139 				    task->d.iovs[0].iov_base, task->fill_pattern,
140 				    task->d.iovs[0].iov_len);
141 }
142 
143 static int
144 ioat_submit_copy(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task)
145 {
146 	if (spdk_unlikely(task->d.iovcnt != 1 || task->s.iovcnt != 1)) {
147 		return -EINVAL;
148 	}
149 
150 	if (spdk_unlikely(task->d.iovs[0].iov_len != task->s.iovs[0].iov_len)) {
151 		return -EINVAL;
152 	}
153 
154 	return spdk_ioat_build_copy(ioat_ch->ioat_ch, task, ioat_done,
155 				    task->d.iovs[0].iov_base, task->s.iovs[0].iov_base,
156 				    task->d.iovs[0].iov_len);
157 }
158 
159 static int
160 ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task)
161 {
162 	struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
163 	struct spdk_accel_task *tmp;
164 	int rc = 0;
165 
166 	do {
167 		switch (accel_task->op_code) {
168 		case SPDK_ACCEL_OPC_FILL:
169 			rc = ioat_submit_fill(ioat_ch,  accel_task);
170 			break;
171 		case SPDK_ACCEL_OPC_COPY:
172 			rc = ioat_submit_copy(ioat_ch, accel_task);
173 			break;
174 		default:
175 			assert(false);
176 			break;
177 		}
178 
179 		tmp = TAILQ_NEXT(accel_task, link);
180 
181 		/* Report any build errors via the callback now. */
182 		if (rc) {
183 			spdk_accel_task_complete(accel_task, rc);
184 		}
185 
186 		accel_task = tmp;
187 	} while (accel_task);
188 
189 	spdk_ioat_flush(ioat_ch->ioat_ch);
190 
191 	return 0;
192 }
193 
194 static int
195 ioat_create_cb(void *io_device, void *ctx_buf)
196 {
197 	struct ioat_io_channel *ch = ctx_buf;
198 	struct ioat_device *ioat_dev;
199 
200 	ioat_dev = ioat_allocate_device();
201 	if (ioat_dev == NULL) {
202 		return -1;
203 	}
204 
205 	ch->ioat_dev = ioat_dev;
206 	ch->ioat_ch = ioat_dev->ioat;
207 	ch->poller = SPDK_POLLER_REGISTER(ioat_poll, ch->ioat_ch, 0);
208 
209 	return 0;
210 }
211 
212 static void
213 ioat_destroy_cb(void *io_device, void *ctx_buf)
214 {
215 	struct ioat_io_channel *ch = ctx_buf;
216 
217 	ioat_free_device(ch->ioat_dev);
218 	spdk_poller_unregister(&ch->poller);
219 }
220 
221 static struct spdk_io_channel *
222 ioat_get_io_channel(void)
223 {
224 	return spdk_get_io_channel(&g_ioat_module);
225 }
226 
227 static bool
228 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
229 {
230 	struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(pci_dev);
231 	struct pci_device *pdev;
232 
233 	SPDK_INFOLOG(accel_ioat,
234 		     " Found matching device at %04x:%02x:%02x.%x vendor:0x%04x device:0x%04x\n",
235 		     pci_addr.domain,
236 		     pci_addr.bus,
237 		     pci_addr.dev,
238 		     pci_addr.func,
239 		     spdk_pci_device_get_vendor_id(pci_dev),
240 		     spdk_pci_device_get_device_id(pci_dev));
241 
242 	pdev = calloc(1, sizeof(*pdev));
243 	if (pdev == NULL) {
244 		return false;
245 	}
246 	pdev->pci_dev = pci_dev;
247 	TAILQ_INSERT_TAIL(&g_pci_devices, pdev, tailq);
248 
249 	/* Claim the device in case conflict with other process */
250 	if (spdk_pci_device_claim(pci_dev) < 0) {
251 		return false;
252 	}
253 
254 	return true;
255 }
256 
257 static void
258 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
259 {
260 	struct ioat_device *dev;
261 
262 	dev = calloc(1, sizeof(*dev));
263 	if (dev == NULL) {
264 		SPDK_ERRLOG("Failed to allocate device struct\n");
265 		return;
266 	}
267 
268 	dev->ioat = ioat;
269 	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
270 }
271 
272 void
273 accel_ioat_enable_probe(void)
274 {
275 	g_ioat_enable = true;
276 	spdk_accel_module_list_add(&g_ioat_module);
277 }
278 
279 static int
280 accel_ioat_init(void)
281 {
282 	if (!g_ioat_enable) {
283 		assert(0);
284 		return 0;
285 	}
286 
287 	if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
288 		SPDK_ERRLOG("spdk_ioat_probe() failed\n");
289 		return -1;
290 	}
291 
292 	if (TAILQ_EMPTY(&g_devices)) {
293 		SPDK_NOTICELOG("No available ioat devices\n");
294 		return -1;
295 	}
296 
297 	g_ioat_initialized = true;
298 	spdk_io_device_register(&g_ioat_module, ioat_create_cb, ioat_destroy_cb,
299 				sizeof(struct ioat_io_channel), "ioat_accel_module");
300 	return 0;
301 }
302 
303 static void
304 _device_unregister_cb(void *io_device)
305 {
306 	struct ioat_device *dev = io_device;
307 	struct pci_device *pci_dev;
308 
309 	while (!TAILQ_EMPTY(&g_devices)) {
310 		dev = TAILQ_FIRST(&g_devices);
311 		TAILQ_REMOVE(&g_devices, dev, tailq);
312 		spdk_ioat_detach(dev->ioat);
313 		free(dev);
314 	}
315 
316 	while (!TAILQ_EMPTY(&g_pci_devices)) {
317 		pci_dev = TAILQ_FIRST(&g_pci_devices);
318 		TAILQ_REMOVE(&g_pci_devices, pci_dev, tailq);
319 		spdk_pci_device_detach(pci_dev->pci_dev);
320 		free(pci_dev);
321 	}
322 
323 	g_ioat_initialized = false;
324 
325 	spdk_accel_module_finish();
326 }
327 
328 static void
329 accel_ioat_exit(void *ctx)
330 {
331 	if (g_ioat_initialized) {
332 		spdk_io_device_unregister(&g_ioat_module, _device_unregister_cb);
333 	} else {
334 		spdk_accel_module_finish();
335 	}
336 }
337 
338 SPDK_LOG_REGISTER_COMPONENT(accel_ioat)
339