48 	struct raid_bdev_io *raid_io;  member
255 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_xor_stripe_continue()  local
256 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_xor_stripe_continue()
284 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_xor_stripe()  local
285 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_xor_stripe()
297 		num_blocks = raid_io->num_blocks;  in raid5f_xor_stripe()
324 	if (raid_io->md_buf != NULL) {  in raid5f_xor_stripe()
370 	if (raid_bdev_io_complete_part(stripe_req->raid_io, 1, status)) {  in raid5f_stripe_request_chunk_write_complete()
379 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_stripe_request_chunk_read_complete()  local
381 	raid_bdev_io_complete_part(raid_io, 1, status);  in raid5f_stripe_request_chunk_read_complete()
408 	struct raid_bdev_io *raid_io = _raid_io;  in raid5f_chunk_submit_retry()  local
409 	struct stripe_request *stripe_req = raid_io->module_private;  in raid5f_chunk_submit_retry()
415 raid5f_init_ext_io_opts(struct spdk_bdev_ext_io_opts *opts, struct raid_bdev_io *raid_io)  in raid5f_init_ext_io_opts()  argument
419 	opts->memory_domain = raid_io->memory_domain;  in raid5f_init_ext_io_opts()
420 	opts->memory_domain_ctx = raid_io->memory_domain_ctx;  in raid5f_init_ext_io_opts()
421 	opts->metadata = raid_io->md_buf;  in raid5f_init_ext_io_opts()
428 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_chunk_submit()  local
429 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_chunk_submit()
431 	struct spdk_io_channel *base_ch = raid_bdev_channel_get_base_channel(raid_io->raid_ch,  in raid5f_chunk_submit()
437 	raid5f_init_ext_io_opts(&io_opts, raid_io);  in raid5f_chunk_submit()
440 	raid_io->base_bdev_io_submitted++;  in raid5f_chunk_submit()
445 			raid_bdev_io_complete_part(raid_io, 1, SPDK_BDEV_IO_STATUS_SUCCESS);  in raid5f_chunk_submit()
455 			raid_bdev_io_complete_part(raid_io, 1, SPDK_BDEV_IO_STATUS_SUCCESS);  in raid5f_chunk_submit()
462 						 base_offset_blocks, raid_io->num_blocks,  in raid5f_chunk_submit()
472 		raid_io->base_bdev_io_submitted--;  in raid5f_chunk_submit()
474 			raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),  in raid5f_chunk_submit()
486 							     raid_io->base_bdev_io_submitted;  in raid5f_chunk_submit()
489 							     raid_io->base_bdev_io_submitted;  in raid5f_chunk_submit()
492 			if (raid_bdev_io_complete_part(raid_io, base_bdev_io_not_submitted,  in raid5f_chunk_submit()
523 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_stripe_request_map_iovecs()  local
524 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_stripe_request_map_iovecs()
538 		for (i = raid_io_iov_idx; i < raid_io->iovcnt; i++) {  in raid5f_stripe_request_map_iovecs()
540 			off += raid_io->iovs[i].iov_len;  in raid5f_stripe_request_map_iovecs()
546 		assert(raid_io_iov_idx + chunk_iovcnt <= raid_io->iovcnt);  in raid5f_stripe_request_map_iovecs()
553 		if (raid_io->md_buf != NULL) {  in raid5f_stripe_request_map_iovecs()
554 			chunk->md_buf = raid_io->md_buf +  in raid5f_stripe_request_map_iovecs()
560 			const struct iovec *raid_io_iov = &raid_io->iovs[raid_io_iov_idx];  in raid5f_stripe_request_map_iovecs()
590 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_stripe_request_submit_chunks()  local
591 	struct chunk *start = &stripe_req->chunks[raid_io->base_bdev_io_submitted];  in raid5f_stripe_request_submit_chunks()
602 raid5f_stripe_request_init(struct stripe_request *stripe_req, struct raid_bdev_io *raid_io,  in raid5f_stripe_request_init()  argument
605 	stripe_req->raid_io = raid_io;  in raid5f_stripe_request_init()
607 	stripe_req->parity_chunk = &stripe_req->chunks[raid5f_stripe_parity_chunk_index(raid_io->raid_bdev,  in raid5f_stripe_request_init()
614 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_stripe_write_request_xor_done()  local
618 		raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);  in raid5f_stripe_write_request_xor_done()
625 raid5f_submit_write_request(struct raid_bdev_io *raid_io, uint64_t stripe_index)  in raid5f_submit_write_request()  argument
627 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_submit_write_request()
628 	struct raid5f_io_channel *r5ch = raid_bdev_channel_get_module_ctx(raid_io->raid_ch);  in raid5f_submit_write_request()
637 	raid5f_stripe_request_init(stripe_req, raid_io, stripe_index);  in raid5f_submit_write_request()
646 	raid_io->module_private = stripe_req;  in raid5f_submit_write_request()
647 	raid_io->base_bdev_io_remaining = raid_bdev->num_base_bdevs;  in raid5f_submit_write_request()
649 …if (raid_bdev_channel_get_base_channel(raid_io->raid_ch, stripe_req->parity_chunk->index) != NULL)…  in raid5f_submit_write_request()
661 	struct raid_bdev_io *raid_io = cb_arg;  in raid5f_chunk_read_complete()  local
665 	raid_bdev_io_complete(raid_io, success ? SPDK_BDEV_IO_STATUS_SUCCESS :  in raid5f_chunk_read_complete()
669 static void raid5f_submit_rw_request(struct raid_bdev_io *raid_io);
674 	struct raid_bdev_io *raid_io = _raid_io;  in _raid5f_submit_rw_request()  local
676 	raid5f_submit_rw_request(raid_io);  in _raid5f_submit_rw_request()
682 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_stripe_request_reconstruct_xor_done()  local
686 	raid_bdev_io_complete(raid_io,  in raid5f_stripe_request_reconstruct_xor_done()
691 raid5f_reconstruct_reads_completed_cb(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status)  in raid5f_reconstruct_reads_completed_cb()  argument
693 	struct stripe_request *stripe_req = raid_io->module_private;  in raid5f_reconstruct_reads_completed_cb()
695 	raid_io->completion_cb = NULL;  in raid5f_reconstruct_reads_completed_cb()
706 raid5f_submit_reconstruct_read(struct raid_bdev_io *raid_io, uint64_t stripe_index,  in raid5f_submit_reconstruct_read()  argument
709 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_submit_reconstruct_read()
710 	struct raid5f_io_channel *r5ch = raid_bdev_channel_get_module_ctx(raid_io->raid_ch);  in raid5f_submit_reconstruct_read()
711 	void *raid_io_md = raid_io->md_buf;  in raid5f_submit_reconstruct_read()
723 	raid5f_stripe_request_init(stripe_req, raid_io, stripe_index);  in raid5f_submit_reconstruct_read()
735 			ret = raid5f_chunk_set_iovcnt(chunk, raid_io->iovcnt);  in raid5f_submit_reconstruct_read()
740 			for (i = 0; i < raid_io->iovcnt; i++) {  in raid5f_submit_reconstruct_read()
741 				chunk->iovs[i] = raid_io->iovs[i];  in raid5f_submit_reconstruct_read()
749 			iov->iov_len = raid_io->num_blocks * raid_bdev->bdev.blocklen;  in raid5f_submit_reconstruct_read()
760 	raid_io->module_private = stripe_req;  in raid5f_submit_reconstruct_read()
761 	raid_io->base_bdev_io_remaining = raid_bdev->num_base_bdevs;  in raid5f_submit_reconstruct_read()
762 	raid_io->completion_cb = raid5f_reconstruct_reads_completed_cb;  in raid5f_submit_reconstruct_read()
772 raid5f_submit_read_request(struct raid_bdev_io *raid_io, uint64_t stripe_index,  in raid5f_submit_read_request()  argument
775 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_submit_read_request()
780 	struct spdk_io_channel *base_ch = raid_bdev_channel_get_base_channel(raid_io->raid_ch, chunk_idx);  in raid5f_submit_read_request()
786 	raid5f_init_ext_io_opts(&io_opts, raid_io);  in raid5f_submit_read_request()
788 		return raid5f_submit_reconstruct_read(raid_io, stripe_index, chunk_idx, chunk_offset,  in raid5f_submit_read_request()
792 	ret = raid_bdev_readv_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,  in raid5f_submit_read_request()
793 					 base_offset_blocks, raid_io->num_blocks,  in raid5f_submit_read_request()
794 					 raid5f_chunk_read_complete, raid_io, &io_opts);  in raid5f_submit_read_request()
796 		raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),  in raid5f_submit_read_request()
805 raid5f_submit_rw_request(struct raid_bdev_io *raid_io)  in raid5f_submit_rw_request()  argument
807 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_submit_rw_request()
809 	uint64_t stripe_index = raid_io->offset_blocks / r5f_info->stripe_blocks;  in raid5f_submit_rw_request()
810 	uint64_t stripe_offset = raid_io->offset_blocks % r5f_info->stripe_blocks;  in raid5f_submit_rw_request()
813 	switch (raid_io->type) {  in raid5f_submit_rw_request()
815 		assert(raid_io->num_blocks <= raid_bdev->strip_size);  in raid5f_submit_rw_request()
816 		ret = raid5f_submit_read_request(raid_io, stripe_index, stripe_offset);  in raid5f_submit_rw_request()
820 		assert(raid_io->num_blocks == r5f_info->stripe_blocks);  in raid5f_submit_rw_request()
821 		ret = raid5f_submit_write_request(raid_io, stripe_index);  in raid5f_submit_rw_request()
829 		raid_bdev_io_complete(raid_io, ret == -ENOMEM ? SPDK_BDEV_IO_STATUS_NOMEM :  in raid5f_submit_rw_request()
1166 	struct raid_bdev_io *raid_io = &process_req->raid_io;  in raid5f_process_submit_write()  local
1167 	struct raid_bdev *raid_bdev = raid_io->raid_bdev;  in raid5f_process_submit_write()
1173 	raid5f_init_ext_io_opts(&io_opts, raid_io);  in raid5f_process_submit_write()
1175 					  raid_io->iovs, raid_io->iovcnt,  in raid5f_process_submit_write()
1180 			raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(process_req->target->desc),  in raid5f_process_submit_write()
1191 	struct raid_bdev_io *raid_io = stripe_req->raid_io;  in raid5f_process_stripe_request_reconstruct_xor_done()  local
1192 	struct raid_bdev_process_request *process_req = SPDK_CONTAINEROF(raid_io,  in raid5f_process_stripe_request_reconstruct_xor_done()
1193 			struct raid_bdev_process_request, raid_io);  in raid5f_process_stripe_request_reconstruct_xor_done()
1212 	struct raid_bdev_io *raid_io = &process_req->raid_io;  in raid5f_submit_process_request()  local
1223 	raid_bdev_io_init(raid_io, raid_ch, SPDK_BDEV_IO_TYPE_READ,  in raid5f_submit_process_request()
1227 	ret = raid5f_submit_reconstruct_read(raid_io, stripe_index, chunk_idx, 0,  in raid5f_submit_process_request()