xref: /spdk/lib/ioat/ioat.c (revision 1fc4165fe9bf8512483356ad8e6d27f793f2e3db) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 
36 #include "ioat_internal.h"
37 
38 #include "spdk/env.h"
39 #include "spdk/util.h"
40 
41 #include "spdk_internal/log.h"
42 #include "spdk_internal/memory.h"
43 
44 struct ioat_driver {
45 	pthread_mutex_t			lock;
46 	TAILQ_HEAD(, spdk_ioat_chan)	attached_chans;
47 };
48 
49 static struct ioat_driver g_ioat_driver = {
50 	.lock = PTHREAD_MUTEX_INITIALIZER,
51 	.attached_chans = TAILQ_HEAD_INITIALIZER(g_ioat_driver.attached_chans),
52 };
53 
54 static uint64_t
55 ioat_get_chansts(struct spdk_ioat_chan *ioat)
56 {
57 	return spdk_mmio_read_8(&ioat->regs->chansts);
58 }
59 
60 static void
61 ioat_write_chancmp(struct spdk_ioat_chan *ioat, uint64_t addr)
62 {
63 	spdk_mmio_write_8(&ioat->regs->chancmp, addr);
64 }
65 
66 static void
67 ioat_write_chainaddr(struct spdk_ioat_chan *ioat, uint64_t addr)
68 {
69 	spdk_mmio_write_8(&ioat->regs->chainaddr, addr);
70 }
71 
72 static inline void
73 ioat_suspend(struct spdk_ioat_chan *ioat)
74 {
75 	ioat->regs->chancmd = SPDK_IOAT_CHANCMD_SUSPEND;
76 }
77 
78 static inline void
79 ioat_reset(struct spdk_ioat_chan *ioat)
80 {
81 	ioat->regs->chancmd = SPDK_IOAT_CHANCMD_RESET;
82 }
83 
84 static inline uint32_t
85 ioat_reset_pending(struct spdk_ioat_chan *ioat)
86 {
87 	uint8_t cmd;
88 
89 	cmd = ioat->regs->chancmd;
90 	return (cmd & SPDK_IOAT_CHANCMD_RESET) == SPDK_IOAT_CHANCMD_RESET;
91 }
92 
93 static int
94 ioat_map_pci_bar(struct spdk_ioat_chan *ioat)
95 {
96 	int regs_bar, rc;
97 	void *addr;
98 	uint64_t phys_addr, size;
99 
100 	regs_bar = 0;
101 	rc = spdk_pci_device_map_bar(ioat->device, regs_bar, &addr, &phys_addr, &size);
102 	if (rc != 0 || addr == NULL) {
103 		SPDK_ERRLOG("pci_device_map_range failed with error code %d\n",
104 			    rc);
105 		return -1;
106 	}
107 
108 	ioat->regs = (volatile struct spdk_ioat_registers *)addr;
109 
110 	return 0;
111 }
112 
113 static int
114 ioat_unmap_pci_bar(struct spdk_ioat_chan *ioat)
115 {
116 	int rc = 0;
117 	void *addr = (void *)ioat->regs;
118 
119 	if (addr) {
120 		rc = spdk_pci_device_unmap_bar(ioat->device, 0, addr);
121 	}
122 	return rc;
123 }
124 
125 
126 static inline uint32_t
127 ioat_get_active(struct spdk_ioat_chan *ioat)
128 {
129 	return (ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1);
130 }
131 
132 static inline uint32_t
133 ioat_get_ring_space(struct spdk_ioat_chan *ioat)
134 {
135 	return (1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1;
136 }
137 
138 static uint32_t
139 ioat_get_ring_index(struct spdk_ioat_chan *ioat, uint32_t index)
140 {
141 	return index & ((1 << ioat->ring_size_order) - 1);
142 }
143 
144 static void
145 ioat_get_ring_entry(struct spdk_ioat_chan *ioat, uint32_t index,
146 		    struct ioat_descriptor **desc,
147 		    union spdk_ioat_hw_desc **hw_desc)
148 {
149 	uint32_t i = ioat_get_ring_index(ioat, index);
150 
151 	*desc = &ioat->ring[i];
152 	*hw_desc = &ioat->hw_ring[i];
153 }
154 
155 static void
156 ioat_submit_single(struct spdk_ioat_chan *ioat)
157 {
158 	ioat->head++;
159 }
160 
161 void
162 spdk_ioat_flush(struct spdk_ioat_chan *ioat)
163 {
164 	uint32_t index = ioat_get_ring_index(ioat, ioat->head - 1);
165 	union spdk_ioat_hw_desc *hw_desc;
166 
167 	hw_desc = &ioat->hw_ring[index];
168 	hw_desc->dma.u.control.completion_update = 1;
169 	ioat->regs->dmacount = (uint16_t)ioat->head;
170 }
171 
172 static struct ioat_descriptor *
173 ioat_prep_null(struct spdk_ioat_chan *ioat)
174 {
175 	struct ioat_descriptor *desc;
176 	union spdk_ioat_hw_desc *hw_desc;
177 
178 	if (ioat_get_ring_space(ioat) < 1) {
179 		return NULL;
180 	}
181 
182 	ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc);
183 
184 	hw_desc->dma.u.control_raw = 0;
185 	hw_desc->dma.u.control.op = SPDK_IOAT_OP_COPY;
186 	hw_desc->dma.u.control.null = 1;
187 
188 	hw_desc->dma.size = 8;
189 	hw_desc->dma.src_addr = 0;
190 	hw_desc->dma.dest_addr = 0;
191 
192 	desc->callback_fn = NULL;
193 	desc->callback_arg = NULL;
194 
195 	ioat_submit_single(ioat);
196 
197 	return desc;
198 }
199 
200 static struct ioat_descriptor *
201 ioat_prep_copy(struct spdk_ioat_chan *ioat, uint64_t dst,
202 	       uint64_t src, uint32_t len)
203 {
204 	struct ioat_descriptor *desc;
205 	union spdk_ioat_hw_desc *hw_desc;
206 
207 	assert(len <= ioat->max_xfer_size);
208 
209 	if (ioat_get_ring_space(ioat) < 1) {
210 		return NULL;
211 	}
212 
213 	ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc);
214 
215 	hw_desc->dma.u.control_raw = 0;
216 	hw_desc->dma.u.control.op = SPDK_IOAT_OP_COPY;
217 
218 	hw_desc->dma.size = len;
219 	hw_desc->dma.src_addr = src;
220 	hw_desc->dma.dest_addr = dst;
221 
222 	desc->callback_fn = NULL;
223 	desc->callback_arg = NULL;
224 
225 	ioat_submit_single(ioat);
226 
227 	return desc;
228 }
229 
230 static struct ioat_descriptor *
231 ioat_prep_fill(struct spdk_ioat_chan *ioat, uint64_t dst,
232 	       uint64_t fill_pattern, uint32_t len)
233 {
234 	struct ioat_descriptor *desc;
235 	union spdk_ioat_hw_desc *hw_desc;
236 
237 	assert(len <= ioat->max_xfer_size);
238 
239 	if (ioat_get_ring_space(ioat) < 1) {
240 		return NULL;
241 	}
242 
243 	ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc);
244 
245 	hw_desc->fill.u.control_raw = 0;
246 	hw_desc->fill.u.control.op = SPDK_IOAT_OP_FILL;
247 
248 	hw_desc->fill.size = len;
249 	hw_desc->fill.src_data = fill_pattern;
250 	hw_desc->fill.dest_addr = dst;
251 
252 	desc->callback_fn = NULL;
253 	desc->callback_arg = NULL;
254 
255 	ioat_submit_single(ioat);
256 
257 	return desc;
258 }
259 
260 static int ioat_reset_hw(struct spdk_ioat_chan *ioat)
261 {
262 	int timeout;
263 	uint64_t status;
264 	uint32_t chanerr;
265 	int rc;
266 
267 	status = ioat_get_chansts(ioat);
268 	if (is_ioat_active(status) || is_ioat_idle(status)) {
269 		ioat_suspend(ioat);
270 	}
271 
272 	timeout = 20; /* in milliseconds */
273 	while (is_ioat_active(status) || is_ioat_idle(status)) {
274 		spdk_delay_us(1000);
275 		timeout--;
276 		if (timeout == 0) {
277 			SPDK_ERRLOG("timed out waiting for suspend\n");
278 			return -1;
279 		}
280 		status = ioat_get_chansts(ioat);
281 	}
282 
283 	/*
284 	 * Clear any outstanding errors.
285 	 * CHANERR is write-1-to-clear, so write the current CHANERR bits back to reset everything.
286 	 */
287 	chanerr = ioat->regs->chanerr;
288 	ioat->regs->chanerr = chanerr;
289 
290 	if (ioat->regs->cbver < SPDK_IOAT_VER_3_3) {
291 		rc = spdk_pci_device_cfg_read32(ioat->device, &chanerr,
292 						SPDK_IOAT_PCI_CHANERR_INT_OFFSET);
293 		if (rc) {
294 			SPDK_ERRLOG("failed to read the internal channel error register\n");
295 			return -1;
296 		}
297 
298 		spdk_pci_device_cfg_write32(ioat->device, chanerr,
299 					    SPDK_IOAT_PCI_CHANERR_INT_OFFSET);
300 	}
301 
302 	ioat_reset(ioat);
303 
304 	timeout = 20;
305 	while (ioat_reset_pending(ioat)) {
306 		spdk_delay_us(1000);
307 		timeout--;
308 		if (timeout == 0) {
309 			SPDK_ERRLOG("timed out waiting for reset\n");
310 			return -1;
311 		}
312 	}
313 
314 	return 0;
315 }
316 
317 static int
318 ioat_process_channel_events(struct spdk_ioat_chan *ioat)
319 {
320 	struct ioat_descriptor *desc;
321 	uint64_t status, completed_descriptor, hw_desc_phys_addr;
322 	uint32_t tail;
323 
324 	if (ioat->head == ioat->tail) {
325 		return 0;
326 	}
327 
328 	status = *ioat->comp_update;
329 	completed_descriptor = status & SPDK_IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
330 
331 	if (is_ioat_halted(status)) {
332 		SPDK_ERRLOG("Channel halted (%x)\n", ioat->regs->chanerr);
333 		return -1;
334 	}
335 
336 	if (completed_descriptor == ioat->last_seen) {
337 		return 0;
338 	}
339 
340 	do {
341 		tail = ioat_get_ring_index(ioat, ioat->tail);
342 		desc = &ioat->ring[tail];
343 
344 		if (desc->callback_fn) {
345 			desc->callback_fn(desc->callback_arg);
346 		}
347 
348 		hw_desc_phys_addr = desc->phys_addr;
349 		ioat->tail++;
350 	} while (hw_desc_phys_addr != completed_descriptor);
351 
352 	ioat->last_seen = hw_desc_phys_addr;
353 	return 0;
354 }
355 
356 static void
357 ioat_channel_destruct(struct spdk_ioat_chan *ioat)
358 {
359 	ioat_unmap_pci_bar(ioat);
360 
361 	if (ioat->ring) {
362 		free(ioat->ring);
363 	}
364 
365 	if (ioat->hw_ring) {
366 		spdk_dma_free(ioat->hw_ring);
367 	}
368 
369 	if (ioat->comp_update) {
370 		spdk_dma_free((void *)ioat->comp_update);
371 		ioat->comp_update = NULL;
372 	}
373 }
374 
375 static int
376 ioat_channel_start(struct spdk_ioat_chan *ioat)
377 {
378 	uint8_t xfercap, version;
379 	uint64_t status;
380 	int i, num_descriptors;
381 	uint64_t comp_update_bus_addr = 0;
382 	uint64_t phys_addr;
383 
384 	if (ioat_map_pci_bar(ioat) != 0) {
385 		SPDK_ERRLOG("ioat_map_pci_bar() failed\n");
386 		return -1;
387 	}
388 
389 	version = ioat->regs->cbver;
390 	if (version < SPDK_IOAT_VER_3_0) {
391 		SPDK_ERRLOG(" unsupported IOAT version %u.%u\n",
392 			    version >> 4, version & 0xF);
393 		return -1;
394 	}
395 
396 	/* Always support DMA copy */
397 	ioat->dma_capabilities = SPDK_IOAT_ENGINE_COPY_SUPPORTED;
398 	if (ioat->regs->dmacapability & SPDK_IOAT_DMACAP_BFILL) {
399 		ioat->dma_capabilities |= SPDK_IOAT_ENGINE_FILL_SUPPORTED;
400 	}
401 	xfercap = ioat->regs->xfercap;
402 
403 	/* Only bits [4:0] are valid. */
404 	xfercap &= 0x1f;
405 	if (xfercap == 0) {
406 		/* 0 means 4 GB max transfer size. */
407 		ioat->max_xfer_size = 1ULL << 32;
408 	} else if (xfercap < 12) {
409 		/* XFERCAP must be at least 12 (4 KB) according to the spec. */
410 		SPDK_ERRLOG("invalid XFERCAP value %u\n", xfercap);
411 		return -1;
412 	} else {
413 		ioat->max_xfer_size = 1U << xfercap;
414 	}
415 
416 	ioat->comp_update = spdk_dma_zmalloc(sizeof(*ioat->comp_update), SPDK_IOAT_CHANCMP_ALIGN,
417 					     &comp_update_bus_addr);
418 	if (ioat->comp_update == NULL) {
419 		return -1;
420 	}
421 
422 	ioat->ring_size_order = IOAT_DEFAULT_ORDER;
423 
424 	num_descriptors = 1 << ioat->ring_size_order;
425 
426 	ioat->ring = calloc(num_descriptors, sizeof(struct ioat_descriptor));
427 	if (!ioat->ring) {
428 		return -1;
429 	}
430 
431 	ioat->hw_ring = spdk_dma_zmalloc(num_descriptors * sizeof(union spdk_ioat_hw_desc), 64,
432 					 NULL);
433 	if (!ioat->hw_ring) {
434 		return -1;
435 	}
436 
437 	for (i = 0; i < num_descriptors; i++) {
438 		phys_addr = spdk_vtophys(&ioat->hw_ring[i], NULL);
439 		if (phys_addr == SPDK_VTOPHYS_ERROR) {
440 			SPDK_ERRLOG("Failed to translate descriptor %u to physical address\n", i);
441 			return -1;
442 		}
443 
444 		ioat->ring[i].phys_addr = phys_addr;
445 		ioat->hw_ring[ioat_get_ring_index(ioat, i - 1)].generic.next = phys_addr;
446 	}
447 
448 	ioat->head = 0;
449 	ioat->tail = 0;
450 	ioat->last_seen = 0;
451 
452 	ioat_reset_hw(ioat);
453 
454 	ioat->regs->chanctrl = SPDK_IOAT_CHANCTRL_ANY_ERR_ABORT_EN;
455 	ioat_write_chancmp(ioat, comp_update_bus_addr);
456 	ioat_write_chainaddr(ioat, ioat->ring[0].phys_addr);
457 
458 	ioat_prep_null(ioat);
459 	spdk_ioat_flush(ioat);
460 
461 	i = 100;
462 	while (i-- > 0) {
463 		spdk_delay_us(100);
464 		status = ioat_get_chansts(ioat);
465 		if (is_ioat_idle(status)) {
466 			break;
467 		}
468 	}
469 
470 	if (is_ioat_idle(status)) {
471 		ioat_process_channel_events(ioat);
472 	} else {
473 		SPDK_ERRLOG("could not start channel: status = %p\n error = %#x\n",
474 			    (void *)status, ioat->regs->chanerr);
475 		return -1;
476 	}
477 
478 	return 0;
479 }
480 
481 /* Caller must hold g_ioat_driver.lock */
482 static struct spdk_ioat_chan *
483 ioat_attach(struct spdk_pci_device *device)
484 {
485 	struct spdk_ioat_chan *ioat;
486 	uint32_t cmd_reg;
487 
488 	ioat = calloc(1, sizeof(struct spdk_ioat_chan));
489 	if (ioat == NULL) {
490 		return NULL;
491 	}
492 
493 	/* Enable PCI busmaster. */
494 	spdk_pci_device_cfg_read32(device, &cmd_reg, 4);
495 	cmd_reg |= 0x4;
496 	spdk_pci_device_cfg_write32(device, cmd_reg, 4);
497 
498 	ioat->device = device;
499 
500 	if (ioat_channel_start(ioat) != 0) {
501 		ioat_channel_destruct(ioat);
502 		free(ioat);
503 		return NULL;
504 	}
505 
506 	return ioat;
507 }
508 
509 struct ioat_enum_ctx {
510 	spdk_ioat_probe_cb probe_cb;
511 	spdk_ioat_attach_cb attach_cb;
512 	void *cb_ctx;
513 };
514 
515 /* This function must only be called while holding g_ioat_driver.lock */
516 static int
517 ioat_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
518 {
519 	struct ioat_enum_ctx *enum_ctx = ctx;
520 	struct spdk_ioat_chan *ioat;
521 
522 	/* Verify that this device is not already attached */
523 	TAILQ_FOREACH(ioat, &g_ioat_driver.attached_chans, tailq) {
524 		/*
525 		 * NOTE: This assumes that the PCI abstraction layer will use the same device handle
526 		 *  across enumerations; we could compare by BDF instead if this is not true.
527 		 */
528 		if (pci_dev == ioat->device) {
529 			return 0;
530 		}
531 	}
532 
533 	if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) {
534 		/*
535 		 * Since I/OAT init is relatively quick, just perform the full init during probing.
536 		 *  If this turns out to be a bottleneck later, this can be changed to work like
537 		 *  NVMe with a list of devices to initialize in parallel.
538 		 */
539 		ioat = ioat_attach(pci_dev);
540 		if (ioat == NULL) {
541 			SPDK_ERRLOG("ioat_attach() failed\n");
542 			return -1;
543 		}
544 
545 		TAILQ_INSERT_TAIL(&g_ioat_driver.attached_chans, ioat, tailq);
546 
547 		enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, ioat);
548 	}
549 
550 	return 0;
551 }
552 
553 int
554 spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_cb attach_cb)
555 {
556 	int rc;
557 	struct ioat_enum_ctx enum_ctx;
558 
559 	pthread_mutex_lock(&g_ioat_driver.lock);
560 
561 	enum_ctx.probe_cb = probe_cb;
562 	enum_ctx.attach_cb = attach_cb;
563 	enum_ctx.cb_ctx = cb_ctx;
564 
565 	rc = spdk_pci_enumerate(spdk_pci_ioat_get_driver(), ioat_enum_cb, &enum_ctx);
566 
567 	pthread_mutex_unlock(&g_ioat_driver.lock);
568 
569 	return rc;
570 }
571 
572 void
573 spdk_ioat_detach(struct spdk_ioat_chan *ioat)
574 {
575 	struct ioat_driver	*driver = &g_ioat_driver;
576 
577 	/* ioat should be in the free list (not registered to a thread)
578 	 * when calling ioat_detach().
579 	 */
580 	pthread_mutex_lock(&driver->lock);
581 	TAILQ_REMOVE(&driver->attached_chans, ioat, tailq);
582 	pthread_mutex_unlock(&driver->lock);
583 
584 	ioat_channel_destruct(ioat);
585 	free(ioat);
586 }
587 
588 int
589 spdk_ioat_build_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
590 		     void *dst, const void *src, uint64_t nbytes)
591 {
592 	struct ioat_descriptor	*last_desc;
593 	uint64_t	remaining, op_size;
594 	uint64_t	vdst, vsrc;
595 	uint64_t	vdst_page, vsrc_page;
596 	uint64_t	pdst_page, psrc_page;
597 	uint32_t	orig_head;
598 
599 	if (!ioat) {
600 		return -EINVAL;
601 	}
602 
603 	orig_head = ioat->head;
604 
605 	vdst = (uint64_t)dst;
606 	vsrc = (uint64_t)src;
607 	vdst_page = vsrc_page = 0;
608 	pdst_page = psrc_page = SPDK_VTOPHYS_ERROR;
609 
610 	remaining = nbytes;
611 	while (remaining) {
612 		if (_2MB_PAGE(vsrc) != vsrc_page) {
613 			vsrc_page = _2MB_PAGE(vsrc);
614 			psrc_page = spdk_vtophys((void *)vsrc_page, NULL);
615 		}
616 
617 		if (_2MB_PAGE(vdst) != vdst_page) {
618 			vdst_page = _2MB_PAGE(vdst);
619 			pdst_page = spdk_vtophys((void *)vdst_page, NULL);
620 		}
621 		op_size = remaining;
622 		op_size = spdk_min(op_size, (VALUE_2MB - _2MB_OFFSET(vsrc)));
623 		op_size = spdk_min(op_size, (VALUE_2MB - _2MB_OFFSET(vdst)));
624 		op_size = spdk_min(op_size, ioat->max_xfer_size);
625 		remaining -= op_size;
626 
627 		last_desc = ioat_prep_copy(ioat,
628 					   pdst_page + _2MB_OFFSET(vdst),
629 					   psrc_page + _2MB_OFFSET(vsrc),
630 					   op_size);
631 
632 		if (remaining == 0 || last_desc == NULL) {
633 			break;
634 		}
635 
636 		vsrc += op_size;
637 		vdst += op_size;
638 
639 	}
640 	/* Issue null descriptor for null transfer */
641 	if (nbytes == 0) {
642 		last_desc = ioat_prep_null(ioat);
643 	}
644 
645 	if (last_desc) {
646 		last_desc->callback_fn = cb_fn;
647 		last_desc->callback_arg = cb_arg;
648 	} else {
649 		/*
650 		 * Ran out of descriptors in the ring - reset head to leave things as they were
651 		 * in case we managed to fill out any descriptors.
652 		 */
653 		ioat->head = orig_head;
654 		return -ENOMEM;
655 	}
656 
657 	return 0;
658 }
659 
660 int
661 spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
662 		      void *dst, const void *src, uint64_t nbytes)
663 {
664 	int rc;
665 
666 	rc = spdk_ioat_build_copy(ioat, cb_arg, cb_fn, dst, src, nbytes);
667 	if (rc != 0) {
668 		return rc;
669 	}
670 
671 	spdk_ioat_flush(ioat);
672 	return 0;
673 }
674 
675 int
676 spdk_ioat_build_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
677 		     void *dst, uint64_t fill_pattern, uint64_t nbytes)
678 {
679 	struct ioat_descriptor	*last_desc = NULL;
680 	uint64_t	remaining, op_size;
681 	uint64_t	vdst;
682 	uint32_t	orig_head;
683 
684 	if (!ioat) {
685 		return -EINVAL;
686 	}
687 
688 	if (!(ioat->dma_capabilities & SPDK_IOAT_ENGINE_FILL_SUPPORTED)) {
689 		SPDK_ERRLOG("Channel does not support memory fill\n");
690 		return -1;
691 	}
692 
693 	orig_head = ioat->head;
694 
695 	vdst = (uint64_t)dst;
696 	remaining = nbytes;
697 
698 	while (remaining) {
699 		op_size = remaining;
700 		op_size = spdk_min(op_size, (VALUE_2MB - _2MB_OFFSET(vdst)));
701 		op_size = spdk_min(op_size, ioat->max_xfer_size);
702 		remaining -= op_size;
703 
704 		last_desc = ioat_prep_fill(ioat,
705 					   spdk_vtophys((void *)vdst, NULL),
706 					   fill_pattern,
707 					   op_size);
708 
709 		if (remaining == 0 || last_desc == NULL) {
710 			break;
711 		}
712 
713 		vdst += op_size;
714 	}
715 
716 	if (last_desc) {
717 		last_desc->callback_fn = cb_fn;
718 		last_desc->callback_arg = cb_arg;
719 	} else {
720 		/*
721 		 * Ran out of descriptors in the ring - reset head to leave things as they were
722 		 * in case we managed to fill out any descriptors.
723 		 */
724 		ioat->head = orig_head;
725 		return -ENOMEM;
726 	}
727 
728 	return 0;
729 }
730 
731 int
732 spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
733 		      void *dst, uint64_t fill_pattern, uint64_t nbytes)
734 {
735 	int rc;
736 
737 	rc = spdk_ioat_build_fill(ioat, cb_arg, cb_fn, dst, fill_pattern, nbytes);
738 	if (rc != 0) {
739 		return rc;
740 	}
741 
742 	spdk_ioat_flush(ioat);
743 	return 0;
744 }
745 
746 uint32_t
747 spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat)
748 {
749 	if (!ioat) {
750 		return 0;
751 	}
752 	return ioat->dma_capabilities;
753 }
754 
755 int
756 spdk_ioat_process_events(struct spdk_ioat_chan *ioat)
757 {
758 	return ioat_process_channel_events(ioat);
759 }
760 
761 SPDK_LOG_REGISTER_COMPONENT("ioat", SPDK_LOG_IOAT)
762