xref: /spdk/app/spdk_dd/spdk_dd.c (revision bef7abee6bd20debd5d579116c12a5c6266f1daf)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (C) 2020 Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 
8 #include "spdk/bdev.h"
9 #include "spdk/event.h"
10 #include "spdk/fd.h"
11 #include "spdk/string.h"
12 #include "spdk/util.h"
13 #include "spdk/vmd.h"
14 
15 #include <libaio.h>
16 
17 #ifdef SPDK_CONFIG_URING
18 #include <liburing.h>
19 #endif
20 
21 #define TIMESPEC_TO_MS(time) ((time.tv_sec * 1000) + (time.tv_nsec / 1000000))
22 #define STATUS_POLLER_PERIOD_SEC 1
23 
24 struct spdk_dd_opts {
25 	char		*input_file;
26 	char		*output_file;
27 	char		*input_file_flags;
28 	char		*output_file_flags;
29 	char		*input_bdev;
30 	char		*output_bdev;
31 	uint64_t	input_offset;
32 	uint64_t	output_offset;
33 	int64_t		io_unit_size;
34 	int64_t		io_unit_count;
35 	uint32_t	queue_depth;
36 	bool		aio;
37 	bool		sparse;
38 };
39 
40 static struct spdk_dd_opts g_opts = {
41 	.io_unit_size = 4096,
42 	.queue_depth = 2,
43 };
44 
45 enum dd_submit_type {
46 	DD_POPULATE,
47 	DD_READ,
48 	DD_WRITE,
49 };
50 
51 struct dd_io {
52 	uint64_t		offset;
53 	uint64_t		length;
54 	struct iocb		iocb;
55 	enum dd_submit_type	type;
56 #ifdef SPDK_CONFIG_URING
57 	int			idx;
58 #endif
59 	void			*buf;
60 	STAILQ_ENTRY(dd_io)	link;
61 };
62 
63 enum dd_target_type {
64 	DD_TARGET_TYPE_FILE,
65 	DD_TARGET_TYPE_BDEV,
66 };
67 
68 struct dd_target {
69 	enum dd_target_type	type;
70 
71 	union {
72 		struct {
73 			struct spdk_bdev *bdev;
74 			struct spdk_bdev_desc *desc;
75 			struct spdk_io_channel *ch;
76 		} bdev;
77 
78 #ifdef SPDK_CONFIG_URING
79 		struct {
80 			int fd;
81 			int idx;
82 		} uring;
83 #endif
84 		struct {
85 			int fd;
86 		} aio;
87 	} u;
88 
89 	/* Block size of underlying device. */
90 	uint32_t	block_size;
91 
92 	/* Position of next I/O in bytes */
93 	uint64_t	pos;
94 
95 	/* Total size of target in bytes */
96 	uint64_t	total_size;
97 
98 	bool open;
99 };
100 
101 struct dd_job {
102 	struct dd_target	input;
103 	struct dd_target	output;
104 
105 	struct dd_io		*ios;
106 
107 	union {
108 #ifdef SPDK_CONFIG_URING
109 		struct {
110 			struct io_uring ring;
111 			bool active;
112 			struct spdk_poller *poller;
113 		} uring;
114 #endif
115 		struct {
116 			io_context_t io_ctx;
117 			struct spdk_poller *poller;
118 		} aio;
119 	} u;
120 
121 	uint32_t		outstanding;
122 	uint64_t		copy_size;
123 	STAILQ_HEAD(, dd_io)	seek_queue;
124 
125 	struct timespec		start_time;
126 	uint64_t		total_bytes;
127 	uint64_t		incremental_bytes;
128 	struct spdk_poller	*status_poller;
129 };
130 
131 struct dd_flags {
132 	char *name;
133 	int flag;
134 };
135 
136 static struct dd_flags g_flags[] = {
137 	{"append", O_APPEND},
138 	{"direct", O_DIRECT},
139 	{"directory", O_DIRECTORY},
140 	{"dsync", O_DSYNC},
141 	{"noatime", O_NOATIME},
142 	{"noctty", O_NOCTTY},
143 	{"nofollow", O_NOFOLLOW},
144 	{"nonblock", O_NONBLOCK},
145 	{"sync", O_SYNC},
146 	{NULL, 0}
147 };
148 
149 static struct dd_job g_job = {};
150 static int g_error = 0;
151 static bool g_interrupt;
152 
153 static void dd_target_seek(struct dd_io *io);
154 static void _dd_bdev_seek_hole_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
155 
156 static void
157 dd_cleanup_bdev(struct dd_target io)
158 {
159 	/* This can be only on the error path.
160 	 * To prevent the SEGV, need add checks here.
161 	 */
162 	if (io.u.bdev.ch) {
163 		spdk_put_io_channel(io.u.bdev.ch);
164 	}
165 
166 	spdk_bdev_close(io.u.bdev.desc);
167 }
168 
169 static void
170 dd_exit(int rc)
171 {
172 	if (g_job.input.type == DD_TARGET_TYPE_FILE) {
173 #ifdef SPDK_CONFIG_URING
174 		if (g_opts.aio == false) {
175 			close(g_job.input.u.uring.fd);
176 		} else
177 #endif
178 		{
179 			close(g_job.input.u.aio.fd);
180 		}
181 	} else if (g_job.input.type == DD_TARGET_TYPE_BDEV && g_job.input.open) {
182 		dd_cleanup_bdev(g_job.input);
183 	}
184 
185 	if (g_job.output.type == DD_TARGET_TYPE_FILE) {
186 #ifdef SPDK_CONFIG_URING
187 		if (g_opts.aio == false) {
188 			close(g_job.output.u.uring.fd);
189 		} else
190 #endif
191 		{
192 			close(g_job.output.u.aio.fd);
193 		}
194 	} else if (g_job.output.type == DD_TARGET_TYPE_BDEV && g_job.output.open) {
195 		dd_cleanup_bdev(g_job.output);
196 	}
197 
198 	if (g_job.input.type == DD_TARGET_TYPE_FILE || g_job.output.type == DD_TARGET_TYPE_FILE) {
199 #ifdef SPDK_CONFIG_URING
200 		if (g_opts.aio == false) {
201 			spdk_poller_unregister(&g_job.u.uring.poller);
202 		} else
203 #endif
204 		{
205 			spdk_poller_unregister(&g_job.u.aio.poller);
206 		}
207 	}
208 
209 	spdk_poller_unregister(&g_job.status_poller);
210 
211 	spdk_app_stop(rc);
212 }
213 
214 static void
215 dd_show_progress(bool finish)
216 {
217 	char *unit_str[5] = {"", "k", "M", "G", "T"};
218 	char *speed_type_str[2] = {"", "average "};
219 	char *size_unit_str = "";
220 	char *speed_unit_str = "";
221 	char *speed_type;
222 	uint64_t size_unit = 1;
223 	uint64_t speed_unit = 1;
224 	uint64_t speed, tmp_speed;
225 	int i = 0;
226 	uint64_t milliseconds;
227 	uint64_t size, tmp_size;
228 
229 	size = g_job.incremental_bytes;
230 
231 	g_job.incremental_bytes = 0;
232 	g_job.total_bytes += size;
233 
234 	if (finish) {
235 		struct timespec time_now;
236 
237 		clock_gettime(CLOCK_REALTIME, &time_now);
238 
239 		milliseconds = spdk_max(1, TIMESPEC_TO_MS(time_now) - TIMESPEC_TO_MS(g_job.start_time));
240 		size = g_job.total_bytes;
241 	} else {
242 		milliseconds = STATUS_POLLER_PERIOD_SEC * 1000;
243 	}
244 
245 	/* Find the right unit for size displaying (B vs kB vs MB vs GB vs TB) */
246 	tmp_size = size;
247 	while (tmp_size > 1024 * 10) {
248 		tmp_size >>= 10;
249 		size_unit <<= 10;
250 		size_unit_str = unit_str[++i];
251 		if (i == 4) {
252 			break;
253 		}
254 	}
255 
256 	speed_type = finish ? speed_type_str[1] : speed_type_str[0];
257 	speed = (size * 1000) / milliseconds;
258 
259 	i = 0;
260 
261 	/* Find the right unit for speed displaying (Bps vs kBps vs MBps vs GBps vs TBps) */
262 	tmp_speed = speed;
263 	while (tmp_speed > 1024 * 10) {
264 		tmp_speed >>= 10;
265 		speed_unit <<= 10;
266 		speed_unit_str = unit_str[++i];
267 		if (i == 4) {
268 			break;
269 		}
270 	}
271 
272 	printf("\33[2K\rCopying: %" PRIu64 "/%" PRIu64 " [%sB] (%s%" PRIu64 " %sBps)",
273 	       g_job.total_bytes / size_unit, g_job.copy_size / size_unit, size_unit_str, speed_type,
274 	       speed / speed_unit, speed_unit_str);
275 	fflush(stdout);
276 }
277 
278 static int
279 dd_status_poller(void *ctx)
280 {
281 	dd_show_progress(false);
282 	return SPDK_POLLER_BUSY;
283 }
284 
285 static void
286 dd_finalize_output(void)
287 {
288 	off_t curr_offset;
289 	int rc = 0;
290 
291 	if (g_job.outstanding > 0) {
292 		return;
293 	}
294 
295 	if (g_opts.output_file) {
296 		curr_offset = lseek(g_job.output.u.aio.fd, 0, SEEK_END);
297 		if (curr_offset == (off_t) -1) {
298 			SPDK_ERRLOG("Could not seek output file for finalize: %s\n", strerror(errno));
299 			g_error = errno;
300 		} else if ((uint64_t)curr_offset < g_job.copy_size + g_job.output.pos) {
301 			rc = ftruncate(g_job.output.u.aio.fd, g_job.copy_size + g_job.output.pos);
302 			if (rc != 0) {
303 				SPDK_ERRLOG("Could not truncate output file for finalize: %s\n", strerror(errno));
304 				g_error = errno;
305 			}
306 		}
307 	}
308 
309 	if (g_error == 0) {
310 		dd_show_progress(true);
311 		printf("\n\n");
312 	}
313 	dd_exit(g_error);
314 }
315 
316 #ifdef SPDK_CONFIG_URING
317 static void
318 dd_uring_submit(struct dd_io *io, struct dd_target *target, uint64_t length, uint64_t offset)
319 {
320 	struct io_uring_sqe *sqe;
321 
322 	sqe = io_uring_get_sqe(&g_job.u.uring.ring);
323 	if (io->type == DD_READ || io->type == DD_POPULATE) {
324 		io_uring_prep_read_fixed(sqe, target->u.uring.idx, io->buf, length, offset, io->idx);
325 	} else {
326 		io_uring_prep_write_fixed(sqe, target->u.uring.idx, io->buf, length, offset, io->idx);
327 	}
328 	sqe->flags |= IOSQE_FIXED_FILE;
329 	io_uring_sqe_set_data(sqe, io);
330 	io_uring_submit(&g_job.u.uring.ring);
331 }
332 #endif
333 
334 static void
335 _dd_write_bdev_done(struct spdk_bdev_io *bdev_io,
336 		    bool success,
337 		    void *cb_arg)
338 {
339 	struct dd_io *io = cb_arg;
340 
341 	assert(g_job.outstanding > 0);
342 	g_job.outstanding--;
343 	spdk_bdev_free_io(bdev_io);
344 	dd_target_seek(io);
345 }
346 
347 static void
348 dd_target_write(struct dd_io *io)
349 {
350 	struct dd_target *target = &g_job.output;
351 	uint64_t length = SPDK_CEIL_DIV(io->length, target->block_size) * target->block_size;
352 	uint64_t read_region_start = g_opts.input_offset * g_opts.io_unit_size;
353 	uint64_t read_offset = io->offset - read_region_start;
354 	uint64_t write_region_start = g_opts.output_offset * g_opts.io_unit_size;
355 	uint64_t write_offset = write_region_start + read_offset;
356 	int rc = 0;
357 
358 	if (g_error != 0 || g_interrupt == true) {
359 		if (g_job.outstanding == 0) {
360 			if (g_error == 0) {
361 				dd_show_progress(true);
362 				printf("\n\n");
363 			}
364 			dd_exit(g_error);
365 		}
366 		return;
367 	}
368 
369 	g_job.incremental_bytes += io->length;
370 	g_job.outstanding++;
371 	io->type = DD_WRITE;
372 
373 	if (target->type == DD_TARGET_TYPE_FILE) {
374 #ifdef SPDK_CONFIG_URING
375 		if (g_opts.aio == false) {
376 			dd_uring_submit(io, target, length, write_offset);
377 		} else
378 #endif
379 		{
380 			struct iocb *iocb = &io->iocb;
381 
382 			io_prep_pwrite(iocb, target->u.aio.fd, io->buf, length, write_offset);
383 			iocb->data = io;
384 			if (io_submit(g_job.u.aio.io_ctx, 1, &iocb) < 0) {
385 				rc = -errno;
386 			}
387 		}
388 	} else if (target->type == DD_TARGET_TYPE_BDEV) {
389 		rc = spdk_bdev_write(target->u.bdev.desc, target->u.bdev.ch, io->buf, write_offset, length,
390 				     _dd_write_bdev_done, io);
391 	}
392 
393 	if (rc != 0) {
394 		SPDK_ERRLOG("%s\n", strerror(-rc));
395 		assert(g_job.outstanding > 0);
396 		g_job.outstanding--;
397 		g_error = rc;
398 		if (g_job.outstanding == 0) {
399 			dd_exit(rc);
400 		}
401 		return;
402 	}
403 }
404 
405 static void
406 _dd_read_bdev_done(struct spdk_bdev_io *bdev_io,
407 		   bool success,
408 		   void *cb_arg)
409 {
410 	struct dd_io *io = cb_arg;
411 
412 	spdk_bdev_free_io(bdev_io);
413 
414 	assert(g_job.outstanding > 0);
415 	g_job.outstanding--;
416 	dd_target_write(io);
417 }
418 
419 static void
420 dd_target_read(struct dd_io *io)
421 {
422 	struct dd_target *target = &g_job.input;
423 	int rc = 0;
424 
425 	if (g_error != 0 || g_interrupt == true) {
426 		if (g_job.outstanding == 0) {
427 			dd_exit(g_error);
428 		}
429 		return;
430 	}
431 
432 	g_job.outstanding++;
433 	io->type = DD_READ;
434 
435 	if (target->type == DD_TARGET_TYPE_FILE) {
436 #ifdef SPDK_CONFIG_URING
437 		if (g_opts.aio == false) {
438 			dd_uring_submit(io, target, io->length, io->offset);
439 		} else
440 #endif
441 		{
442 			struct iocb *iocb = &io->iocb;
443 
444 			io_prep_pread(iocb, target->u.aio.fd, io->buf, io->length, io->offset);
445 			iocb->data = io;
446 			if (io_submit(g_job.u.aio.io_ctx, 1, &iocb) < 0) {
447 				rc = -errno;
448 			}
449 		}
450 	} else if (target->type == DD_TARGET_TYPE_BDEV) {
451 		rc = spdk_bdev_read(target->u.bdev.desc, target->u.bdev.ch, io->buf, io->offset, io->length,
452 				    _dd_read_bdev_done, io);
453 	}
454 
455 	if (rc != 0) {
456 		SPDK_ERRLOG("%s\n", strerror(-rc));
457 		assert(g_job.outstanding > 0);
458 		g_job.outstanding--;
459 		g_error = rc;
460 		if (g_job.outstanding == 0) {
461 			dd_exit(rc);
462 		}
463 		return;
464 	}
465 }
466 
467 static void
468 _dd_target_populate_buffer_done(struct spdk_bdev_io *bdev_io,
469 				bool success,
470 				void *cb_arg)
471 {
472 	struct dd_io *io = cb_arg;
473 
474 	assert(g_job.outstanding > 0);
475 	g_job.outstanding--;
476 	spdk_bdev_free_io(bdev_io);
477 	dd_target_read(io);
478 }
479 
480 static void
481 dd_target_populate_buffer(struct dd_io *io)
482 {
483 	struct dd_target *target = &g_job.output;
484 	uint64_t read_region_start = g_opts.input_offset * g_opts.io_unit_size;
485 	uint64_t read_offset = g_job.input.pos - read_region_start;
486 	uint64_t write_region_start = g_opts.output_offset * g_opts.io_unit_size;
487 	uint64_t write_offset = write_region_start + read_offset;
488 	uint64_t length;
489 	int rc = 0;
490 
491 	io->offset = g_job.input.pos;
492 	io->length = spdk_min(io->length, g_job.copy_size - read_offset);
493 
494 	if (io->length == 0 || g_error != 0 || g_interrupt == true) {
495 		if (g_job.outstanding == 0) {
496 			if (g_error == 0) {
497 				dd_show_progress(true);
498 				printf("\n\n");
499 			}
500 			dd_exit(g_error);
501 		}
502 		return;
503 	}
504 
505 	g_job.input.pos += io->length;
506 
507 	if ((io->length % target->block_size) == 0) {
508 		dd_target_read(io);
509 		return;
510 	}
511 
512 	/* Read whole blocks from output to combine buffers later */
513 	g_job.outstanding++;
514 	io->type = DD_POPULATE;
515 
516 	length = SPDK_CEIL_DIV(io->length, target->block_size) * target->block_size;
517 
518 	if (target->type == DD_TARGET_TYPE_FILE) {
519 #ifdef SPDK_CONFIG_URING
520 		if (g_opts.aio == false) {
521 			dd_uring_submit(io, target, length, write_offset);
522 		} else
523 #endif
524 		{
525 			struct iocb *iocb = &io->iocb;
526 
527 			io_prep_pread(iocb, target->u.aio.fd, io->buf, length, write_offset);
528 			iocb->data = io;
529 			if (io_submit(g_job.u.aio.io_ctx, 1, &iocb) < 0) {
530 				rc = -errno;
531 			}
532 		}
533 	} else if (target->type == DD_TARGET_TYPE_BDEV) {
534 		rc = spdk_bdev_read(target->u.bdev.desc, target->u.bdev.ch, io->buf, write_offset, length,
535 				    _dd_target_populate_buffer_done, io);
536 	}
537 
538 	if (rc != 0) {
539 		SPDK_ERRLOG("%s\n", strerror(-rc));
540 		assert(g_job.outstanding > 0);
541 		g_job.outstanding--;
542 		g_error = rc;
543 		if (g_job.outstanding == 0) {
544 			dd_exit(rc);
545 		}
546 		return;
547 	}
548 }
549 
550 static off_t
551 dd_file_seek_data(void)
552 {
553 	off_t next_data_offset = (off_t) -1;
554 
555 	next_data_offset = lseek(g_job.input.u.aio.fd, g_job.input.pos, SEEK_DATA);
556 
557 	if (next_data_offset == (off_t) -1) {
558 		/* NXIO with SEEK_DATA means there are no more data to read.
559 		 * But in case of input and output files, we may have to finalize output file
560 		 * inserting a hole to the end of the file.
561 		 */
562 		if (errno == ENXIO) {
563 			dd_finalize_output();
564 		} else if (g_job.outstanding == 0) {
565 			SPDK_ERRLOG("Could not seek input file for data: %s\n", strerror(errno));
566 			g_error = errno;
567 			dd_exit(g_error);
568 		}
569 	}
570 
571 	return next_data_offset;
572 }
573 
574 static off_t
575 dd_file_seek_hole(void)
576 {
577 	off_t next_hole_offset = (off_t) -1;
578 
579 	next_hole_offset = lseek(g_job.input.u.aio.fd, g_job.input.pos, SEEK_HOLE);
580 
581 	if (next_hole_offset == (off_t) -1 && g_job.outstanding == 0) {
582 		SPDK_ERRLOG("Could not seek input file for hole: %s\n", strerror(errno));
583 		g_error = errno;
584 		dd_exit(g_error);
585 	}
586 
587 	return next_hole_offset;
588 }
589 
590 static void
591 _dd_bdev_seek_data_done(struct spdk_bdev_io *bdev_io,
592 			bool success,
593 			void *cb_arg)
594 {
595 	struct dd_io *io = cb_arg;
596 	uint64_t next_data_offset_blocks = UINT64_MAX;
597 	struct dd_target *target = &g_job.input;
598 	int rc = 0;
599 
600 	if (g_error != 0 || g_interrupt == true) {
601 		STAILQ_REMOVE_HEAD(&g_job.seek_queue, link);
602 		if (g_job.outstanding == 0) {
603 			if (g_error == 0) {
604 				dd_show_progress(true);
605 				printf("\n\n");
606 			}
607 			dd_exit(g_error);
608 		}
609 		return;
610 	}
611 
612 	assert(g_job.outstanding > 0);
613 	g_job.outstanding--;
614 
615 	next_data_offset_blocks = spdk_bdev_io_get_seek_offset(bdev_io);
616 	spdk_bdev_free_io(bdev_io);
617 
618 	/* UINT64_MAX means there are no more data to read.
619 	 * But in case of input and output files, we may have to finalize output file
620 	 * inserting a hole to the end of the file.
621 	 */
622 	if (next_data_offset_blocks == UINT64_MAX) {
623 		STAILQ_REMOVE_HEAD(&g_job.seek_queue, link);
624 		dd_finalize_output();
625 		return;
626 	}
627 
628 	g_job.input.pos = next_data_offset_blocks * g_job.input.block_size;
629 
630 	g_job.outstanding++;
631 	rc = spdk_bdev_seek_hole(target->u.bdev.desc, target->u.bdev.ch,
632 				 g_job.input.pos / g_job.input.block_size,
633 				 _dd_bdev_seek_hole_done, io);
634 
635 	if (rc != 0) {
636 		SPDK_ERRLOG("%s\n", strerror(-rc));
637 		STAILQ_REMOVE_HEAD(&g_job.seek_queue, link);
638 		assert(g_job.outstanding > 0);
639 		g_job.outstanding--;
640 		g_error = rc;
641 		if (g_job.outstanding == 0) {
642 			dd_exit(rc);
643 		}
644 	}
645 }
646 
647 static void
648 _dd_bdev_seek_hole_done(struct spdk_bdev_io *bdev_io,
649 			bool success,
650 			void *cb_arg)
651 {
652 	struct dd_io *io = cb_arg;
653 	struct dd_target *target = &g_job.input;
654 	uint64_t next_hole_offset_blocks = UINT64_MAX;
655 	struct dd_io *seek_io;
656 	int rc = 0;
657 
658 	/* First seek operation is the one in progress, i.e. this one just ended */
659 	STAILQ_REMOVE_HEAD(&g_job.seek_queue, link);
660 
661 	if (g_error != 0 || g_interrupt == true) {
662 		if (g_job.outstanding == 0) {
663 			if (g_error == 0) {
664 				dd_show_progress(true);
665 				printf("\n\n");
666 			}
667 			dd_exit(g_error);
668 		}
669 		return;
670 	}
671 
672 	assert(g_job.outstanding > 0);
673 	g_job.outstanding--;
674 
675 	next_hole_offset_blocks = spdk_bdev_io_get_seek_offset(bdev_io);
676 	spdk_bdev_free_io(bdev_io);
677 
678 	/* UINT64_MAX means there are no more holes. */
679 	if (next_hole_offset_blocks == UINT64_MAX) {
680 		io->length = g_opts.io_unit_size;
681 	} else {
682 		io->length = spdk_min((uint64_t)g_opts.io_unit_size,
683 				      next_hole_offset_blocks * g_job.input.block_size - g_job.input.pos);
684 	}
685 
686 	dd_target_populate_buffer(io);
687 
688 	/* If input reading is not at the end, start following seek operation in the queue */
689 	if (!STAILQ_EMPTY(&g_job.seek_queue) && g_job.input.pos < g_job.input.total_size) {
690 		seek_io = STAILQ_FIRST(&g_job.seek_queue);
691 		assert(seek_io != NULL);
692 		g_job.outstanding++;
693 		rc = spdk_bdev_seek_data(target->u.bdev.desc, target->u.bdev.ch,
694 					 g_job.input.pos / g_job.input.block_size,
695 					 _dd_bdev_seek_data_done, seek_io);
696 
697 		if (rc != 0) {
698 			SPDK_ERRLOG("%s\n", strerror(-rc));
699 			assert(g_job.outstanding > 0);
700 			g_job.outstanding--;
701 			g_error = rc;
702 			if (g_job.outstanding == 0) {
703 				dd_exit(rc);
704 			}
705 		}
706 	}
707 }
708 
709 static void
710 dd_target_seek(struct dd_io *io)
711 {
712 	struct dd_target *target = &g_job.input;
713 	uint64_t read_region_start = g_opts.input_offset * g_opts.io_unit_size;
714 	uint64_t read_offset = g_job.input.pos - read_region_start;
715 	off_t next_data_offset = (off_t) -1;
716 	off_t next_hole_offset = (off_t) -1;
717 	int rc = 0;
718 
719 	if (!g_opts.sparse) {
720 		dd_target_populate_buffer(io);
721 		return;
722 	}
723 
724 	if (g_job.copy_size - read_offset == 0 || g_error != 0 || g_interrupt == true) {
725 		if (g_job.outstanding == 0) {
726 			if (g_error == 0) {
727 				dd_show_progress(true);
728 				printf("\n\n");
729 			}
730 			dd_exit(g_error);
731 		}
732 		return;
733 	}
734 
735 	if (target->type == DD_TARGET_TYPE_FILE) {
736 		next_data_offset = dd_file_seek_data();
737 		if (next_data_offset < 0) {
738 			return;
739 		} else if ((uint64_t)next_data_offset > g_job.input.pos) {
740 			g_job.input.pos = next_data_offset;
741 		}
742 
743 		next_hole_offset = dd_file_seek_hole();
744 		if (next_hole_offset < 0) {
745 			return;
746 		} else if ((uint64_t)next_hole_offset > g_job.input.pos) {
747 			io->length = spdk_min((uint64_t)g_opts.io_unit_size,
748 					      (uint64_t)(next_hole_offset - g_job.input.pos));
749 		} else {
750 			io->length = g_opts.io_unit_size;
751 		}
752 
753 		dd_target_populate_buffer(io);
754 	} else if (target->type == DD_TARGET_TYPE_BDEV) {
755 		/* Check if other seek operation is in progress */
756 		if (STAILQ_EMPTY(&g_job.seek_queue)) {
757 			g_job.outstanding++;
758 			rc = spdk_bdev_seek_data(target->u.bdev.desc, target->u.bdev.ch,
759 						 g_job.input.pos / g_job.input.block_size,
760 						 _dd_bdev_seek_data_done, io);
761 
762 		}
763 
764 		STAILQ_INSERT_TAIL(&g_job.seek_queue, io, link);
765 	}
766 
767 	if (rc != 0) {
768 		SPDK_ERRLOG("%s\n", strerror(-rc));
769 		assert(g_job.outstanding > 0);
770 		g_job.outstanding--;
771 		g_error = rc;
772 		if (g_job.outstanding == 0) {
773 			dd_exit(rc);
774 		}
775 		return;
776 	}
777 }
778 
779 static void
780 dd_complete_poll(struct dd_io *io)
781 {
782 	assert(g_job.outstanding > 0);
783 	g_job.outstanding--;
784 
785 	switch (io->type) {
786 	case DD_POPULATE:
787 		dd_target_read(io);
788 		break;
789 	case DD_READ:
790 		dd_target_write(io);
791 		break;
792 	case DD_WRITE:
793 		dd_target_seek(io);
794 		break;
795 	default:
796 		assert(false);
797 		break;
798 	}
799 }
800 
801 #ifdef SPDK_CONFIG_URING
802 static int
803 dd_uring_poll(void *ctx)
804 {
805 	struct io_uring_cqe *cqe;
806 	struct dd_io *io;
807 	int rc = 0;
808 	int i;
809 
810 	for (i = 0; i < (int)g_opts.queue_depth; i++) {
811 		rc = io_uring_peek_cqe(&g_job.u.uring.ring, &cqe);
812 		if (rc == 0) {
813 			if (cqe->res == -EAGAIN) {
814 				continue;
815 			} else if (cqe->res < 0) {
816 				SPDK_ERRLOG("%s\n", strerror(-cqe->res));
817 				g_error = cqe->res;
818 			}
819 
820 			io = io_uring_cqe_get_data(cqe);
821 			io_uring_cqe_seen(&g_job.u.uring.ring, cqe);
822 
823 			dd_complete_poll(io);
824 		} else if (rc != - EAGAIN) {
825 			SPDK_ERRLOG("%s\n", strerror(-rc));
826 			g_error = rc;
827 		}
828 	}
829 
830 	return rc;
831 }
832 #endif
833 
834 static int
835 dd_aio_poll(void *ctx)
836 {
837 	struct io_event events[32];
838 	int rc = 0;
839 	int i;
840 	struct timespec timeout;
841 	struct dd_io *io;
842 
843 	timeout.tv_sec = 0;
844 	timeout.tv_nsec = 0;
845 
846 	rc = io_getevents(g_job.u.aio.io_ctx, 0, 32, events, &timeout);
847 
848 	if (rc < 0) {
849 		SPDK_ERRLOG("%s\n", strerror(-rc));
850 		dd_exit(rc);
851 	}
852 
853 	for (i = 0; i < rc; i++) {
854 		io = events[i].data;
855 		if (events[i].res != io->length) {
856 			g_error = rc = -ENOSPC;
857 		}
858 
859 		dd_complete_poll(io);
860 	}
861 
862 	return rc;
863 }
864 
865 static int
866 dd_open_file(struct dd_target *target, const char *fname, int flags, uint64_t skip_blocks,
867 	     bool input)
868 {
869 	int *fd;
870 
871 #ifdef SPDK_CONFIG_URING
872 	if (g_opts.aio == false) {
873 		fd = &target->u.uring.fd;
874 	} else
875 #endif
876 	{
877 		fd = &target->u.aio.fd;
878 	}
879 
880 	flags |= O_RDWR;
881 
882 	if (input == false && ((flags & O_DIRECTORY) == 0)) {
883 		flags |= O_CREAT;
884 	}
885 
886 	if (input == false && ((flags & O_APPEND) == 0)) {
887 		flags |= O_TRUNC;
888 	}
889 
890 	target->type = DD_TARGET_TYPE_FILE;
891 	*fd = open(fname, flags, 0600);
892 	if (*fd < 0) {
893 		SPDK_ERRLOG("Could not open file %s: %s\n", fname, strerror(errno));
894 		return *fd;
895 	}
896 
897 	target->block_size = spdk_max(spdk_fd_get_blocklen(*fd), 1);
898 
899 	target->total_size = spdk_fd_get_size(*fd);
900 	if (target->total_size == 0) {
901 		target->total_size = g_opts.io_unit_size * g_opts.io_unit_count;
902 	}
903 
904 	if (input == true) {
905 		g_opts.queue_depth = spdk_min(g_opts.queue_depth,
906 					      (target->total_size / g_opts.io_unit_size) - skip_blocks + 1);
907 	}
908 
909 	if (g_opts.io_unit_count != 0) {
910 		g_opts.queue_depth = spdk_min(g_opts.queue_depth, g_opts.io_unit_count);
911 	}
912 
913 	return 0;
914 }
915 
916 static void
917 dd_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
918 		 void *event_ctx)
919 {
920 	SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
921 }
922 
923 static int
924 dd_open_bdev(struct dd_target *target, const char *bdev_name, uint64_t skip_blocks)
925 {
926 	int rc;
927 
928 	target->type = DD_TARGET_TYPE_BDEV;
929 
930 	rc = spdk_bdev_open_ext(bdev_name, true, dd_bdev_event_cb, NULL, &target->u.bdev.desc);
931 	if (rc < 0) {
932 		SPDK_ERRLOG("Could not open bdev %s: %s\n", bdev_name, strerror(-rc));
933 		return rc;
934 	}
935 
936 	target->u.bdev.bdev = spdk_bdev_desc_get_bdev(target->u.bdev.desc);
937 	target->open = true;
938 
939 	target->u.bdev.ch = spdk_bdev_get_io_channel(target->u.bdev.desc);
940 	if (target->u.bdev.ch == NULL) {
941 		spdk_bdev_close(target->u.bdev.desc);
942 		SPDK_ERRLOG("Could not get I/O channel: %s\n", strerror(ENOMEM));
943 		return -ENOMEM;
944 	}
945 
946 	target->block_size = spdk_bdev_get_block_size(target->u.bdev.bdev);
947 	target->total_size = spdk_bdev_get_num_blocks(target->u.bdev.bdev) * target->block_size;
948 
949 	g_opts.queue_depth = spdk_min(g_opts.queue_depth,
950 				      (target->total_size / g_opts.io_unit_size) - skip_blocks + 1);
951 
952 	if (g_opts.io_unit_count != 0) {
953 		g_opts.queue_depth = spdk_min(g_opts.queue_depth, g_opts.io_unit_count);
954 	}
955 
956 	return 0;
957 }
958 
959 static void
960 dd_finish(void)
961 {
962 	/* Interrupt operation */
963 	g_interrupt = true;
964 }
965 
966 static int
967 parse_flags(char *file_flags)
968 {
969 	char *input_flag;
970 	int flags = 0;
971 	int i;
972 	bool found = false;
973 
974 	/* Translate input flags to file open flags */
975 	while ((input_flag = strsep(&file_flags, ","))) {
976 		for (i = 0; g_flags[i].name != NULL; i++) {
977 			if (!strcmp(input_flag, g_flags[i].name)) {
978 				flags |= g_flags[i].flag;
979 				found = true;
980 				break;
981 			}
982 		}
983 
984 		if (found == false) {
985 			SPDK_ERRLOG("Unknown file flag: %s\n", input_flag);
986 			return -EINVAL;
987 		}
988 
989 		found = false;
990 	}
991 
992 	return flags;
993 }
994 
995 #ifdef SPDK_CONFIG_URING
996 static bool
997 dd_is_blk(int fd)
998 {
999 	struct stat st;
1000 
1001 	if (fstat(fd, &st) != 0) {
1002 		return false;
1003 	}
1004 
1005 	return S_ISBLK(st.st_mode);
1006 }
1007 
1008 static int
1009 dd_register_files(void)
1010 {
1011 	int fds[2];
1012 	unsigned count = 0;
1013 
1014 	if (g_opts.input_file) {
1015 		fds[count] = g_job.input.u.uring.fd;
1016 		g_job.input.u.uring.idx = count;
1017 		count++;
1018 	}
1019 
1020 	if (g_opts.output_file) {
1021 		fds[count] = g_job.output.u.uring.fd;
1022 		g_job.output.u.uring.idx = count;
1023 		count++;
1024 	}
1025 
1026 	return io_uring_register_files(&g_job.u.uring.ring, fds, count);
1027 
1028 }
1029 
1030 static int
1031 dd_register_buffers(void)
1032 {
1033 	struct iovec *iovs;
1034 	int i, rc;
1035 
1036 	iovs = calloc(g_opts.queue_depth, sizeof(struct iovec));
1037 	if (iovs == NULL) {
1038 		return -ENOMEM;
1039 	}
1040 
1041 	for (i = 0; i < (int)g_opts.queue_depth; i++) {
1042 		iovs[i].iov_base = g_job.ios[i].buf;
1043 		iovs[i].iov_len = g_opts.io_unit_size;
1044 		g_job.ios[i].idx = i;
1045 	}
1046 
1047 	rc = io_uring_register_buffers(&g_job.u.uring.ring, iovs, g_opts.queue_depth);
1048 
1049 	free(iovs);
1050 	return rc;
1051 }
1052 #endif
1053 
1054 static void
1055 dd_run(void *arg1)
1056 {
1057 	uint64_t write_size;
1058 	uint32_t i;
1059 	int rc, flags = 0;
1060 
1061 	if (g_opts.input_file) {
1062 		if (g_opts.input_file_flags) {
1063 			flags = parse_flags(g_opts.input_file_flags);
1064 		}
1065 
1066 		if (dd_open_file(&g_job.input, g_opts.input_file, flags, g_opts.input_offset, true) < 0) {
1067 			SPDK_ERRLOG("%s: %s\n", g_opts.input_file, strerror(errno));
1068 			dd_exit(-errno);
1069 			return;
1070 		}
1071 	} else if (g_opts.input_bdev) {
1072 		rc = dd_open_bdev(&g_job.input, g_opts.input_bdev, g_opts.input_offset);
1073 		if (rc < 0) {
1074 			SPDK_ERRLOG("%s: %s\n", g_opts.input_bdev, strerror(-rc));
1075 			dd_exit(rc);
1076 			return;
1077 		}
1078 	}
1079 
1080 	write_size = g_opts.io_unit_count * g_opts.io_unit_size;
1081 	g_job.input.pos = g_opts.input_offset * g_opts.io_unit_size;
1082 
1083 	/* We cannot check write size for input files because /dev/zeros, /dev/random, etc would not work.
1084 	 * We will handle that during copying */
1085 	if (g_opts.input_bdev && g_job.input.pos > g_job.input.total_size) {
1086 		SPDK_ERRLOG("--skip value too big (%" PRIu64 ") - only %" PRIu64 " blocks available in input\n",
1087 			    g_opts.input_offset, g_job.input.total_size / g_opts.io_unit_size);
1088 		dd_exit(-ENOSPC);
1089 		return;
1090 	}
1091 
1092 	if (g_opts.io_unit_count != 0 && g_opts.input_bdev &&
1093 	    write_size + g_job.input.pos > g_job.input.total_size) {
1094 		SPDK_ERRLOG("--count value too big (%" PRIu64 ") - only %" PRIu64 " blocks available from input\n",
1095 			    g_opts.io_unit_count, (g_job.input.total_size - g_job.input.pos) / g_opts.io_unit_size);
1096 		dd_exit(-ENOSPC);
1097 		return;
1098 	}
1099 
1100 	if (g_opts.io_unit_count != 0) {
1101 		g_job.copy_size = write_size;
1102 	} else {
1103 		g_job.copy_size = g_job.input.total_size - g_job.input.pos;
1104 	}
1105 
1106 	g_job.output.pos = g_opts.output_offset * g_opts.io_unit_size;
1107 
1108 	if (g_opts.output_file) {
1109 		flags = 0;
1110 
1111 		if (g_opts.output_file_flags) {
1112 			flags = parse_flags(g_opts.output_file_flags);
1113 		}
1114 
1115 		if (dd_open_file(&g_job.output, g_opts.output_file, flags, g_opts.output_offset, false) < 0) {
1116 			SPDK_ERRLOG("%s: %s\n", g_opts.output_file, strerror(errno));
1117 			dd_exit(-errno);
1118 			return;
1119 		}
1120 	} else if (g_opts.output_bdev) {
1121 		rc = dd_open_bdev(&g_job.output, g_opts.output_bdev, g_opts.output_offset);
1122 		if (rc < 0) {
1123 			SPDK_ERRLOG("%s: %s\n", g_opts.output_bdev, strerror(-rc));
1124 			dd_exit(rc);
1125 			return;
1126 		}
1127 
1128 		if (g_job.output.pos > g_job.output.total_size) {
1129 			SPDK_ERRLOG("--seek value too big (%" PRIu64 ") - only %" PRIu64 " blocks available in output\n",
1130 				    g_opts.output_offset, g_job.output.total_size / g_opts.io_unit_size);
1131 			dd_exit(-ENOSPC);
1132 			return;
1133 		}
1134 
1135 		if (g_opts.io_unit_count != 0 && write_size + g_job.output.pos > g_job.output.total_size) {
1136 			SPDK_ERRLOG("--count value too big (%" PRIu64 ") - only %" PRIu64 " blocks available in output\n",
1137 				    g_opts.io_unit_count, (g_job.output.total_size - g_job.output.pos) / g_opts.io_unit_size);
1138 			dd_exit(-ENOSPC);
1139 			return;
1140 		}
1141 	}
1142 
1143 	if ((g_job.output.block_size > g_opts.io_unit_size) ||
1144 	    (g_job.input.block_size > g_opts.io_unit_size)) {
1145 		SPDK_ERRLOG("--bs value cannot be less than input (%d) neither output (%d) native block size\n",
1146 			    g_job.input.block_size, g_job.output.block_size);
1147 		dd_exit(-EINVAL);
1148 		return;
1149 	}
1150 
1151 	if (g_opts.input_bdev && g_opts.io_unit_size % g_job.input.block_size != 0) {
1152 		SPDK_ERRLOG("--bs value must be a multiple of input native block size (%d)\n",
1153 			    g_job.input.block_size);
1154 		dd_exit(-EINVAL);
1155 		return;
1156 	}
1157 
1158 	g_job.ios = calloc(g_opts.queue_depth, sizeof(struct dd_io));
1159 	if (g_job.ios == NULL) {
1160 		SPDK_ERRLOG("%s\n", strerror(ENOMEM));
1161 		dd_exit(-ENOMEM);
1162 		return;
1163 	}
1164 
1165 	for (i = 0; i < g_opts.queue_depth; i++) {
1166 		g_job.ios[i].buf = spdk_malloc(g_opts.io_unit_size, 0x1000, NULL, 0, SPDK_MALLOC_DMA);
1167 		if (g_job.ios[i].buf == NULL) {
1168 			SPDK_ERRLOG("%s - try smaller block size value\n", strerror(ENOMEM));
1169 			dd_exit(-ENOMEM);
1170 			return;
1171 		}
1172 		g_job.ios[i].length = (uint64_t)g_opts.io_unit_size;
1173 	}
1174 
1175 	if (g_opts.input_file || g_opts.output_file) {
1176 #ifdef SPDK_CONFIG_URING
1177 		if (g_opts.aio == false) {
1178 			unsigned int io_uring_flags = IORING_SETUP_SQPOLL;
1179 			int flags = parse_flags(g_opts.input_file_flags) & parse_flags(g_opts.output_file_flags);
1180 
1181 			if ((flags & O_DIRECT) != 0 &&
1182 			    dd_is_blk(g_job.input.u.uring.fd) &&
1183 			    dd_is_blk(g_job.output.u.uring.fd)) {
1184 				io_uring_flags = IORING_SETUP_IOPOLL;
1185 			}
1186 
1187 			g_job.u.uring.poller = SPDK_POLLER_REGISTER(dd_uring_poll, NULL, 0);
1188 			rc = io_uring_queue_init(g_opts.queue_depth * 2, &g_job.u.uring.ring, io_uring_flags);
1189 			if (rc) {
1190 				SPDK_ERRLOG("Failed to create io_uring: %d (%s)\n", rc, spdk_strerror(-rc));
1191 				dd_exit(rc);
1192 				return;
1193 			}
1194 			g_job.u.uring.active = true;
1195 
1196 			/* Register the files */
1197 			rc = dd_register_files();
1198 			if (rc) {
1199 				SPDK_ERRLOG("Failed to register files with io_uring: %d (%s)\n", rc, spdk_strerror(-rc));
1200 				dd_exit(rc);
1201 				return;
1202 			}
1203 
1204 			/* Register the buffers */
1205 			rc = dd_register_buffers();
1206 			if (rc) {
1207 				SPDK_ERRLOG("Failed to register buffers with io_uring: %d (%s)\n", rc, spdk_strerror(-rc));
1208 				dd_exit(rc);
1209 				return;
1210 			}
1211 
1212 		} else
1213 #endif
1214 		{
1215 			g_job.u.aio.poller = SPDK_POLLER_REGISTER(dd_aio_poll, NULL, 0);
1216 			io_setup(g_opts.queue_depth, &g_job.u.aio.io_ctx);
1217 		}
1218 	}
1219 
1220 	clock_gettime(CLOCK_REALTIME, &g_job.start_time);
1221 
1222 	g_job.status_poller = SPDK_POLLER_REGISTER(dd_status_poller, NULL,
1223 			      STATUS_POLLER_PERIOD_SEC * SPDK_SEC_TO_USEC);
1224 
1225 	STAILQ_INIT(&g_job.seek_queue);
1226 
1227 	for (i = 0; i < g_opts.queue_depth; i++) {
1228 		dd_target_seek(&g_job.ios[i]);
1229 	}
1230 
1231 }
1232 
1233 enum dd_cmdline_opts {
1234 	DD_OPTION_IF = 0x1000,
1235 	DD_OPTION_OF,
1236 	DD_OPTION_IFLAGS,
1237 	DD_OPTION_OFLAGS,
1238 	DD_OPTION_IB,
1239 	DD_OPTION_OB,
1240 	DD_OPTION_SKIP,
1241 	DD_OPTION_SEEK,
1242 	DD_OPTION_BS,
1243 	DD_OPTION_QD,
1244 	DD_OPTION_COUNT,
1245 	DD_OPTION_AIO,
1246 	DD_OPTION_SPARSE,
1247 };
1248 
1249 static struct option g_cmdline_opts[] = {
1250 	{
1251 		.name = "if",
1252 		.has_arg = 1,
1253 		.flag = NULL,
1254 		.val = DD_OPTION_IF,
1255 	},
1256 	{
1257 		.name = "of",
1258 		.has_arg = 1,
1259 		.flag = NULL,
1260 		.val = DD_OPTION_OF,
1261 	},
1262 	{
1263 		.name = "iflag",
1264 		.has_arg = 1,
1265 		.flag = NULL,
1266 		.val = DD_OPTION_IFLAGS,
1267 	},
1268 	{
1269 		.name = "oflag",
1270 		.has_arg = 1,
1271 		.flag = NULL,
1272 		.val = DD_OPTION_OFLAGS,
1273 	},
1274 	{
1275 		.name = "ib",
1276 		.has_arg = 1,
1277 		.flag = NULL,
1278 		.val = DD_OPTION_IB,
1279 	},
1280 	{
1281 		.name = "ob",
1282 		.has_arg = 1,
1283 		.flag = NULL,
1284 		.val = DD_OPTION_OB,
1285 	},
1286 	{
1287 		.name = "skip",
1288 		.has_arg = 1,
1289 		.flag = NULL,
1290 		.val = DD_OPTION_SKIP,
1291 	},
1292 	{
1293 		.name = "seek",
1294 		.has_arg = 1,
1295 		.flag = NULL,
1296 		.val = DD_OPTION_SEEK,
1297 	},
1298 	{
1299 		.name = "bs",
1300 		.has_arg = 1,
1301 		.flag = NULL,
1302 		.val = DD_OPTION_BS,
1303 	},
1304 	{
1305 		.name = "qd",
1306 		.has_arg = 1,
1307 		.flag = NULL,
1308 		.val = DD_OPTION_QD,
1309 	},
1310 	{
1311 		.name = "count",
1312 		.has_arg = 1,
1313 		.flag = NULL,
1314 		.val = DD_OPTION_COUNT,
1315 	},
1316 	{
1317 		.name = "aio",
1318 		.has_arg = 0,
1319 		.flag = NULL,
1320 		.val = DD_OPTION_AIO,
1321 	},
1322 	{
1323 		.name = "sparse",
1324 		.has_arg = 0,
1325 		.flag = NULL,
1326 		.val = DD_OPTION_SPARSE,
1327 	},
1328 	{
1329 		.name = NULL
1330 	}
1331 };
1332 
1333 static void
1334 usage(void)
1335 {
1336 	printf("[--------- DD Options ---------]\n");
1337 	printf(" --if Input file. Must specify either --if or --ib.\n");
1338 	printf(" --ib Input bdev. Must specifier either --if or --ib\n");
1339 	printf(" --of Output file. Must specify either --of or --ob.\n");
1340 	printf(" --ob Output bdev. Must specify either --of or --ob.\n");
1341 	printf(" --iflag Input file flags.\n");
1342 	printf(" --oflag Output file flags.\n");
1343 	printf(" --bs I/O unit size (default: %" PRId64 ")\n", g_opts.io_unit_size);
1344 	printf(" --qd Queue depth (default: %d)\n", g_opts.queue_depth);
1345 	printf(" --count I/O unit count. The number of I/O units to copy. (default: all)\n");
1346 	printf(" --skip Skip this many I/O units at start of input. (default: 0)\n");
1347 	printf(" --seek Skip this many I/O units at start of output. (default: 0)\n");
1348 	printf(" --aio Force usage of AIO. (by default io_uring is used if available)\n");
1349 	printf(" --sparse Enable hole skipping in input target\n");
1350 	printf(" Available iflag and oflag values:\n");
1351 	printf("  append - append mode\n");
1352 	printf("  direct - use direct I/O for data\n");
1353 	printf("  directory - fail unless a directory\n");
1354 	printf("  dsync - use synchronized I/O for data\n");
1355 	printf("  noatime - do not update access time\n");
1356 	printf("  noctty - do not assign controlling terminal from file\n");
1357 	printf("  nofollow - do not follow symlinks\n");
1358 	printf("  nonblock - use non-blocking I/O\n");
1359 	printf("  sync - use synchronized I/O for data and metadata\n");
1360 }
1361 
1362 static int
1363 parse_args(int argc, char *argv)
1364 {
1365 	switch (argc) {
1366 	case DD_OPTION_IF:
1367 		g_opts.input_file = strdup(argv);
1368 		break;
1369 	case DD_OPTION_OF:
1370 		g_opts.output_file = strdup(argv);
1371 		break;
1372 	case DD_OPTION_IFLAGS:
1373 		g_opts.input_file_flags = strdup(argv);
1374 		break;
1375 	case DD_OPTION_OFLAGS:
1376 		g_opts.output_file_flags = strdup(argv);
1377 		break;
1378 	case DD_OPTION_IB:
1379 		g_opts.input_bdev = strdup(argv);
1380 		break;
1381 	case DD_OPTION_OB:
1382 		g_opts.output_bdev = strdup(argv);
1383 		break;
1384 	case DD_OPTION_SKIP:
1385 		g_opts.input_offset = spdk_strtol(optarg, 10);
1386 		break;
1387 	case DD_OPTION_SEEK:
1388 		g_opts.output_offset = spdk_strtol(optarg, 10);
1389 		break;
1390 	case DD_OPTION_BS:
1391 		g_opts.io_unit_size = spdk_strtol(optarg, 10);
1392 		break;
1393 	case DD_OPTION_QD:
1394 		g_opts.queue_depth = spdk_strtol(optarg, 10);
1395 		break;
1396 	case DD_OPTION_COUNT:
1397 		g_opts.io_unit_count = spdk_strtol(optarg, 10);
1398 		break;
1399 	case DD_OPTION_AIO:
1400 		g_opts.aio = true;
1401 		break;
1402 	case DD_OPTION_SPARSE:
1403 		g_opts.sparse = true;
1404 		break;
1405 	default:
1406 		usage();
1407 		return 1;
1408 	}
1409 	return 0;
1410 }
1411 
1412 static void
1413 dd_free(void)
1414 {
1415 	uint32_t i;
1416 
1417 	free(g_opts.input_file);
1418 	free(g_opts.output_file);
1419 	free(g_opts.input_bdev);
1420 	free(g_opts.output_bdev);
1421 	free(g_opts.input_file_flags);
1422 	free(g_opts.output_file_flags);
1423 
1424 
1425 	if (g_job.input.type == DD_TARGET_TYPE_FILE || g_job.output.type == DD_TARGET_TYPE_FILE) {
1426 #ifdef SPDK_CONFIG_URING
1427 		if (g_opts.aio == false) {
1428 			if (g_job.u.uring.active) {
1429 				io_uring_unregister_files(&g_job.u.uring.ring);
1430 				io_uring_queue_exit(&g_job.u.uring.ring);
1431 			}
1432 		} else
1433 #endif
1434 		{
1435 			io_destroy(g_job.u.aio.io_ctx);
1436 		}
1437 	}
1438 
1439 	if (g_job.ios) {
1440 		for (i = 0; i < g_opts.queue_depth; i++) {
1441 			spdk_free(g_job.ios[i].buf);
1442 		}
1443 
1444 		free(g_job.ios);
1445 	}
1446 }
1447 
1448 int
1449 main(int argc, char **argv)
1450 {
1451 	struct spdk_app_opts opts = {};
1452 	int rc = 1;
1453 
1454 	spdk_app_opts_init(&opts, sizeof(opts));
1455 	opts.name = "spdk_dd";
1456 	opts.reactor_mask = "0x1";
1457 	opts.shutdown_cb = dd_finish;
1458 	rc = spdk_app_parse_args(argc, argv, &opts, "", g_cmdline_opts, parse_args, usage);
1459 	if (rc == SPDK_APP_PARSE_ARGS_FAIL) {
1460 		SPDK_ERRLOG("Invalid arguments\n");
1461 		goto end;
1462 	} else if (rc == SPDK_APP_PARSE_ARGS_HELP) {
1463 		goto end;
1464 	}
1465 
1466 	if (g_opts.input_file != NULL && g_opts.input_bdev != NULL) {
1467 		SPDK_ERRLOG("You may specify either --if or --ib, but not both.\n");
1468 		rc = EINVAL;
1469 		goto end;
1470 	}
1471 
1472 	if (g_opts.output_file != NULL && g_opts.output_bdev != NULL) {
1473 		SPDK_ERRLOG("You may specify either --of or --ob, but not both.\n");
1474 		rc = EINVAL;
1475 		goto end;
1476 	}
1477 
1478 	if (g_opts.input_file == NULL && g_opts.input_bdev == NULL) {
1479 		SPDK_ERRLOG("You must specify either --if or --ib\n");
1480 		rc = EINVAL;
1481 		goto end;
1482 	}
1483 
1484 	if (g_opts.output_file == NULL && g_opts.output_bdev == NULL) {
1485 		SPDK_ERRLOG("You must specify either --of or --ob\n");
1486 		rc = EINVAL;
1487 		goto end;
1488 	}
1489 
1490 	if (g_opts.io_unit_size <= 0) {
1491 		SPDK_ERRLOG("Invalid --bs value\n");
1492 		rc = EINVAL;
1493 		goto end;
1494 	}
1495 
1496 	if (g_opts.io_unit_count < 0) {
1497 		SPDK_ERRLOG("Invalid --count value\n");
1498 		rc = EINVAL;
1499 		goto end;
1500 	}
1501 
1502 	if (g_opts.output_file == NULL && g_opts.output_file_flags != NULL) {
1503 		SPDK_ERRLOG("--oflags may be used only with --of\n");
1504 		rc = EINVAL;
1505 		goto end;
1506 	}
1507 
1508 	if (g_opts.input_file == NULL && g_opts.input_file_flags != NULL) {
1509 		SPDK_ERRLOG("--iflags may be used only with --if\n");
1510 		rc = EINVAL;
1511 		goto end;
1512 	}
1513 
1514 	rc = spdk_app_start(&opts, dd_run, NULL);
1515 	if (rc) {
1516 		SPDK_ERRLOG("Error occurred while performing copy\n");
1517 	}
1518 
1519 	dd_free();
1520 	spdk_app_fini();
1521 
1522 end:
1523 	return rc;
1524 }
1525