1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/ftl.h" 8 #include "spdk/likely.h" 9 #include "spdk/util.h" 10 11 #include "ftl_io.h" 12 #include "ftl_core.h" 13 #include "ftl_band.h" 14 #include "ftl_debug.h" 15 16 void 17 ftl_io_inc_req(struct ftl_io *io) 18 { 19 io->dev->num_inflight++; 20 io->req_cnt++; 21 } 22 23 void 24 ftl_io_dec_req(struct ftl_io *io) 25 { 26 assert(io->dev->num_inflight > 0); 27 assert(io->req_cnt > 0); 28 29 io->dev->num_inflight--; 30 io->req_cnt--; 31 } 32 33 struct iovec * 34 ftl_io_iovec(struct ftl_io *io) 35 { 36 return &io->iov[0]; 37 } 38 39 uint64_t 40 ftl_io_get_lba(const struct ftl_io *io, size_t offset) 41 { 42 assert(offset < io->num_blocks); 43 return io->lba + offset; 44 } 45 46 uint64_t 47 ftl_io_current_lba(const struct ftl_io *io) 48 { 49 return ftl_io_get_lba(io, io->pos); 50 } 51 52 void 53 ftl_io_advance(struct ftl_io *io, size_t num_blocks) 54 { 55 struct iovec *iov = ftl_io_iovec(io); 56 size_t iov_blocks, block_left = num_blocks; 57 58 io->pos += num_blocks; 59 60 if (io->iov_cnt == 0) { 61 return; 62 } 63 64 while (block_left > 0) { 65 assert(io->iov_pos < io->iov_cnt); 66 iov_blocks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE; 67 68 if (io->iov_off + block_left < iov_blocks) { 69 io->iov_off += block_left; 70 break; 71 } 72 73 assert(iov_blocks > io->iov_off); 74 block_left -= (iov_blocks - io->iov_off); 75 io->iov_off = 0; 76 io->iov_pos++; 77 } 78 } 79 80 size_t 81 ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt) 82 { 83 size_t num_blocks = 0, i = 0; 84 85 for (; i < iov_cnt; ++i) { 86 if (iov[i].iov_len & (FTL_BLOCK_SIZE - 1)) { 87 return 0; 88 } 89 90 num_blocks += iov[i].iov_len / FTL_BLOCK_SIZE; 91 } 92 93 return num_blocks; 94 } 95 96 void * 97 ftl_io_iovec_addr(struct ftl_io *io) 98 { 99 assert(io->iov_pos < io->iov_cnt); 100 assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len); 101 102 return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base + 103 io->iov_off * FTL_BLOCK_SIZE; 104 } 105 106 size_t 107 ftl_io_iovec_len_left(struct ftl_io *io) 108 { 109 if (io->iov_pos < io->iov_cnt) { 110 struct iovec *iov = ftl_io_iovec(io); 111 return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off; 112 } else { 113 return 0; 114 } 115 } 116 117 static void 118 ftl_io_cb(struct ftl_io *io, void *arg, int status) 119 { 120 struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); 121 size_t result __attribute__((unused)); 122 123 if (spdk_unlikely(status)) { 124 io->status = status; 125 126 if (-EAGAIN == status) { 127 /* IO has to be rescheduled again */ 128 switch (io->type) { 129 case FTL_IO_READ: 130 ftl_io_clear(io); 131 TAILQ_INSERT_HEAD(&io->dev->rd_sq, io, queue_entry); 132 break; 133 case FTL_IO_WRITE: 134 ftl_io_clear(io); 135 TAILQ_INSERT_HEAD(&io->dev->wr_sq, io, queue_entry); 136 break; 137 case FTL_IO_TRIM: 138 ftl_io_clear(io); 139 TAILQ_INSERT_HEAD(&io->dev->trim_sq, io, queue_entry); 140 break; 141 default: 142 /* Unknown IO type, complete to the user */ 143 assert(0); 144 break; 145 } 146 147 } 148 149 if (!io->status) { 150 /* IO rescheduled, return from the function */ 151 return; 152 } 153 } 154 155 if (io->map) { 156 ftl_mempool_put(ioch->map_pool, io->map); 157 } 158 159 result = spdk_ring_enqueue(ioch->cq, (void **)&io, 1, NULL); 160 assert(result != 0); 161 } 162 163 int 164 ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks, 165 struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type) 166 { 167 struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(_ioch); 168 struct spdk_ftl_dev *dev = ioch->dev; 169 170 memset(io, 0, sizeof(struct ftl_io)); 171 io->ioch = _ioch; 172 173 io->flags |= FTL_IO_INITIALIZED; 174 io->type = type; 175 io->dev = dev; 176 io->addr = FTL_ADDR_INVALID; 177 io->cb_ctx = cb_ctx; 178 io->lba = lba; 179 io->user_fn = cb_fn; 180 io->iov = iov; 181 io->iov_cnt = iov_cnt; 182 io->num_blocks = num_blocks; 183 io->trace = ftl_trace_alloc_id(dev); 184 185 ftl_trace_lba_io_init(io->dev, io); 186 return 0; 187 } 188 189 static void 190 ftl_io_complete_verify(struct ftl_io *io) 191 { 192 struct spdk_ftl_dev *dev = io->dev; 193 uint64_t i; 194 uint64_t lba = io->lba; 195 196 assert(io->num_blocks <= dev->xfer_size); 197 198 if (FTL_IO_WRITE == io->type) { 199 return; 200 } 201 202 if (spdk_unlikely(io->status)) { 203 return; 204 } 205 206 for (i = 0; i < io->num_blocks; i++, lba++) { 207 ftl_addr current_addr = ftl_l2p_get(dev, lba); 208 209 /* If user read request gets stuck for whatever reason, then it's possible the LBA 210 * has been relocated by GC or compaction and it may no longer be safe to return data 211 * from that address */ 212 if (spdk_unlikely(current_addr != io->map[i])) { 213 io->status = -EAGAIN; 214 break; 215 } 216 } 217 } 218 219 void 220 ftl_io_complete(struct ftl_io *io) 221 { 222 io->flags &= ~FTL_IO_INITIALIZED; 223 io->done = true; 224 225 if (io->flags & FTL_IO_PINNED) { 226 ftl_io_complete_verify(io); 227 ftl_l2p_unpin(io->dev, io->lba, io->num_blocks); 228 } 229 230 ftl_io_cb(io, io->cb_ctx, io->status); 231 } 232 233 void 234 ftl_io_fail(struct ftl_io *io, int status) 235 { 236 io->status = status; 237 ftl_io_advance(io, io->num_blocks - io->pos); 238 } 239 240 void 241 ftl_io_clear(struct ftl_io *io) 242 { 243 io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0; 244 io->done = false; 245 io->status = 0; 246 io->flags = 0; 247 io->band = NULL; 248 } 249