1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/blob_bdev.h" 9 #include "spdk/blob.h" 10 #include "spdk/thread.h" 11 #include "spdk/log.h" 12 #include "spdk/endian.h" 13 #define __SPDK_BDEV_MODULE_ONLY 14 #include "spdk/bdev_module.h" 15 16 struct blob_bdev { 17 struct spdk_bs_dev bs_dev; 18 struct spdk_bdev *bdev; 19 struct spdk_bdev_desc *desc; 20 bool claimed; 21 }; 22 23 struct blob_resubmit { 24 struct spdk_bdev_io_wait_entry bdev_io_wait; 25 enum spdk_bdev_io_type io_type; 26 struct spdk_bs_dev *dev; 27 struct spdk_io_channel *channel; 28 void *payload; 29 int iovcnt; 30 uint64_t lba; 31 uint32_t lba_count; 32 struct spdk_bs_dev_cb_args *cb_args; 33 struct spdk_blob_ext_io_opts *ext_io_opts; 34 }; 35 static void bdev_blob_resubmit(void *); 36 37 static inline struct spdk_bdev_desc * 38 __get_desc(struct spdk_bs_dev *dev) 39 { 40 return ((struct blob_bdev *)dev)->desc; 41 } 42 43 static inline struct spdk_bdev * 44 __get_bdev(struct spdk_bs_dev *dev) 45 { 46 return ((struct blob_bdev *)dev)->bdev; 47 } 48 49 static void 50 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 51 { 52 struct spdk_bs_dev_cb_args *cb_args = arg; 53 int bserrno; 54 55 if (success) { 56 bserrno = 0; 57 } else { 58 bserrno = -EIO; 59 } 60 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 61 spdk_bdev_free_io(bdev_io); 62 } 63 64 static void 65 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 66 int iovcnt, uint64_t lba, uint32_t lba_count, enum spdk_bdev_io_type io_type, 67 struct spdk_bs_dev_cb_args *cb_args, struct spdk_blob_ext_io_opts *ext_io_opts) 68 { 69 int rc; 70 struct spdk_bdev *bdev = __get_bdev(dev); 71 struct blob_resubmit *ctx; 72 73 ctx = calloc(1, sizeof(struct blob_resubmit)); 74 75 if (ctx == NULL) { 76 SPDK_ERRLOG("Not enough memory to queue io\n"); 77 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 78 return; 79 } 80 81 ctx->io_type = io_type; 82 ctx->dev = dev; 83 ctx->channel = channel; 84 ctx->payload = payload; 85 ctx->iovcnt = iovcnt; 86 ctx->lba = lba; 87 ctx->lba_count = lba_count; 88 ctx->cb_args = cb_args; 89 ctx->bdev_io_wait.bdev = bdev; 90 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 91 ctx->bdev_io_wait.cb_arg = ctx; 92 ctx->ext_io_opts = ext_io_opts; 93 94 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 95 if (rc != 0) { 96 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 97 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 98 free(ctx); 99 assert(false); 100 } 101 } 102 103 static void 104 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 105 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 106 { 107 int rc; 108 109 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 110 lba_count, bdev_blob_io_complete, cb_args); 111 if (rc == -ENOMEM) { 112 bdev_blob_queue_io(dev, channel, payload, 0, lba, 113 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 114 } else if (rc != 0) { 115 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 116 } 117 } 118 119 static void 120 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 121 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 122 { 123 int rc; 124 125 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 126 lba_count, bdev_blob_io_complete, cb_args); 127 if (rc == -ENOMEM) { 128 bdev_blob_queue_io(dev, channel, payload, 0, lba, 129 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 130 } else if (rc != 0) { 131 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 132 } 133 } 134 135 static void 136 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 137 struct iovec *iov, int iovcnt, 138 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 139 { 140 int rc; 141 142 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 143 lba_count, bdev_blob_io_complete, cb_args); 144 if (rc == -ENOMEM) { 145 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 146 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 147 } else if (rc != 0) { 148 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 149 } 150 } 151 152 static void 153 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 154 struct iovec *iov, int iovcnt, 155 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 156 { 157 int rc; 158 159 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 160 lba_count, bdev_blob_io_complete, cb_args); 161 if (rc == -ENOMEM) { 162 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 163 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 164 } else if (rc != 0) { 165 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 166 } 167 } 168 169 static void 170 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 171 struct iovec *iov, int iovcnt, 172 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 173 struct spdk_blob_ext_io_opts *io_opts) 174 { 175 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 176 int rc; 177 178 if (io_opts) { 179 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 180 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 181 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 182 bdev_io_opts = io_opts->user_ctx; 183 assert(bdev_io_opts); 184 } 185 186 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 187 bdev_blob_io_complete, cb_args, bdev_io_opts); 188 if (rc == -ENOMEM) { 189 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 190 io_opts); 191 } else if (rc != 0) { 192 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 193 } 194 } 195 196 static void 197 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 198 struct iovec *iov, int iovcnt, 199 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 200 struct spdk_blob_ext_io_opts *io_opts) 201 { 202 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 203 int rc; 204 205 if (io_opts) { 206 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 207 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 208 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 209 bdev_io_opts = io_opts->user_ctx; 210 assert(bdev_io_opts); 211 } 212 213 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 214 bdev_blob_io_complete, cb_args, bdev_io_opts); 215 if (rc == -ENOMEM) { 216 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 217 io_opts); 218 } else if (rc != 0) { 219 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 220 } 221 } 222 223 static void 224 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 225 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 226 { 227 int rc; 228 229 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 230 lba_count, bdev_blob_io_complete, cb_args); 231 if (rc == -ENOMEM) { 232 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 233 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 234 } else if (rc != 0) { 235 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 236 } 237 } 238 239 static void 240 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 241 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 242 { 243 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 244 int rc; 245 246 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 247 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 248 bdev_blob_io_complete, cb_args); 249 if (rc == -ENOMEM) { 250 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 251 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 252 } else if (rc != 0) { 253 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 254 } 255 } else { 256 /* 257 * If the device doesn't support unmap, immediately complete 258 * the request. Blobstore does not rely on unmap zeroing 259 * data. 260 */ 261 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 262 } 263 } 264 265 static void 266 bdev_blob_resubmit(void *arg) 267 { 268 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 269 270 switch (ctx->io_type) { 271 case SPDK_BDEV_IO_TYPE_READ: 272 if (ctx->iovcnt > 0) { 273 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 274 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 275 } else { 276 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 277 ctx->lba, ctx->lba_count, ctx->cb_args); 278 } 279 break; 280 case SPDK_BDEV_IO_TYPE_WRITE: 281 if (ctx->iovcnt > 0) { 282 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 283 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 284 } else { 285 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 286 ctx->lba, ctx->lba_count, ctx->cb_args); 287 } 288 break; 289 case SPDK_BDEV_IO_TYPE_UNMAP: 290 bdev_blob_unmap(ctx->dev, ctx->channel, 291 ctx->lba, ctx->lba_count, ctx->cb_args); 292 break; 293 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 294 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 295 ctx->lba, ctx->lba_count, ctx->cb_args); 296 break; 297 default: 298 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 299 assert(false); 300 break; 301 } 302 free(ctx); 303 } 304 305 int 306 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 307 { 308 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 309 int rc; 310 311 rc = spdk_bdev_module_claim_bdev(blob_bdev->bdev, NULL, module); 312 if (rc != 0) { 313 SPDK_ERRLOG("could not claim bs dev\n"); 314 return rc; 315 } 316 317 blob_bdev->claimed = true; 318 319 return rc; 320 } 321 322 static struct spdk_io_channel * 323 bdev_blob_create_channel(struct spdk_bs_dev *dev) 324 { 325 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 326 327 return spdk_bdev_get_io_channel(blob_bdev->desc); 328 } 329 330 static void 331 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 332 { 333 spdk_put_io_channel(channel); 334 } 335 336 static void 337 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 338 { 339 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 340 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 341 342 if (blob_bdev->claimed) { 343 spdk_bdev_module_release_bdev(blob_bdev->bdev); 344 } 345 346 spdk_bdev_close(desc); 347 free(bs_dev); 348 } 349 350 static struct spdk_bdev * 351 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 352 { 353 return __get_bdev(bs_dev); 354 } 355 356 static void 357 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 358 { 359 struct spdk_bdev *bdev; 360 361 bdev = spdk_bdev_desc_get_bdev(desc); 362 assert(bdev != NULL); 363 364 b->bdev = bdev; 365 b->desc = desc; 366 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 367 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 368 b->bs_dev.create_channel = bdev_blob_create_channel; 369 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 370 b->bs_dev.destroy = bdev_blob_destroy; 371 b->bs_dev.read = bdev_blob_read; 372 b->bs_dev.write = bdev_blob_write; 373 b->bs_dev.readv = bdev_blob_readv; 374 b->bs_dev.writev = bdev_blob_writev; 375 b->bs_dev.readv_ext = bdev_blob_readv_ext; 376 b->bs_dev.writev_ext = bdev_blob_writev_ext; 377 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 378 b->bs_dev.unmap = bdev_blob_unmap; 379 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 380 } 381 382 int 383 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 384 void *event_ctx, struct spdk_bs_dev **_bs_dev) 385 { 386 struct blob_bdev *b; 387 struct spdk_bdev_desc *desc; 388 int rc; 389 390 b = calloc(1, sizeof(*b)); 391 392 if (b == NULL) { 393 SPDK_ERRLOG("could not allocate blob_bdev\n"); 394 return -ENOMEM; 395 } 396 397 rc = spdk_bdev_open_ext(bdev_name, true, event_cb, event_ctx, &desc); 398 if (rc != 0) { 399 free(b); 400 return rc; 401 } 402 403 blob_bdev_init(b, desc); 404 405 *_bs_dev = &b->bs_dev; 406 407 return 0; 408 } 409