1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob_bdev.h" 10 #include "spdk/blob.h" 11 #include "spdk/thread.h" 12 #include "spdk/log.h" 13 #include "spdk/endian.h" 14 #define __SPDK_BDEV_MODULE_ONLY 15 #include "spdk/bdev_module.h" 16 17 struct blob_bdev { 18 struct spdk_bs_dev bs_dev; 19 struct spdk_bdev *bdev; 20 struct spdk_bdev_desc *desc; 21 bool claimed; 22 }; 23 24 struct blob_resubmit { 25 struct spdk_bdev_io_wait_entry bdev_io_wait; 26 enum spdk_bdev_io_type io_type; 27 struct spdk_bs_dev *dev; 28 struct spdk_io_channel *channel; 29 void *payload; 30 int iovcnt; 31 uint64_t lba; 32 uint64_t src_lba; 33 uint32_t lba_count; 34 struct spdk_bs_dev_cb_args *cb_args; 35 struct spdk_blob_ext_io_opts *ext_io_opts; 36 }; 37 static void bdev_blob_resubmit(void *); 38 39 static inline struct spdk_bdev_desc * 40 __get_desc(struct spdk_bs_dev *dev) 41 { 42 return ((struct blob_bdev *)dev)->desc; 43 } 44 45 static inline struct spdk_bdev * 46 __get_bdev(struct spdk_bs_dev *dev) 47 { 48 return ((struct blob_bdev *)dev)->bdev; 49 } 50 51 static void 52 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 53 { 54 struct spdk_bs_dev_cb_args *cb_args = arg; 55 int bserrno; 56 57 if (success) { 58 bserrno = 0; 59 } else { 60 bserrno = -EIO; 61 } 62 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 63 spdk_bdev_free_io(bdev_io); 64 } 65 66 static void 67 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 68 int iovcnt, uint64_t lba, uint64_t src_lba, uint32_t lba_count, 69 enum spdk_bdev_io_type io_type, struct spdk_bs_dev_cb_args *cb_args, 70 struct spdk_blob_ext_io_opts *ext_io_opts) 71 { 72 int rc; 73 struct spdk_bdev *bdev = __get_bdev(dev); 74 struct blob_resubmit *ctx; 75 76 ctx = calloc(1, sizeof(struct blob_resubmit)); 77 78 if (ctx == NULL) { 79 SPDK_ERRLOG("Not enough memory to queue io\n"); 80 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 81 return; 82 } 83 84 ctx->io_type = io_type; 85 ctx->dev = dev; 86 ctx->channel = channel; 87 ctx->payload = payload; 88 ctx->iovcnt = iovcnt; 89 ctx->lba = lba; 90 ctx->src_lba = src_lba; 91 ctx->lba_count = lba_count; 92 ctx->cb_args = cb_args; 93 ctx->bdev_io_wait.bdev = bdev; 94 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 95 ctx->bdev_io_wait.cb_arg = ctx; 96 ctx->ext_io_opts = ext_io_opts; 97 98 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 99 if (rc != 0) { 100 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 101 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 102 free(ctx); 103 assert(false); 104 } 105 } 106 107 static void 108 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 109 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 110 { 111 int rc; 112 113 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 114 lba_count, bdev_blob_io_complete, cb_args); 115 if (rc == -ENOMEM) { 116 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 117 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 118 } else if (rc != 0) { 119 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 120 } 121 } 122 123 static void 124 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 125 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 126 { 127 int rc; 128 129 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 130 lba_count, bdev_blob_io_complete, cb_args); 131 if (rc == -ENOMEM) { 132 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 133 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 134 } else if (rc != 0) { 135 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 136 } 137 } 138 139 static void 140 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 141 struct iovec *iov, int iovcnt, 142 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 143 { 144 int rc; 145 146 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 147 lba_count, bdev_blob_io_complete, cb_args); 148 if (rc == -ENOMEM) { 149 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 150 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 151 } else if (rc != 0) { 152 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 153 } 154 } 155 156 static void 157 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 158 struct iovec *iov, int iovcnt, 159 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 160 { 161 int rc; 162 163 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 164 lba_count, bdev_blob_io_complete, cb_args); 165 if (rc == -ENOMEM) { 166 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 167 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 168 } else if (rc != 0) { 169 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 170 } 171 } 172 173 static void 174 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 175 struct iovec *iov, int iovcnt, 176 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 177 struct spdk_blob_ext_io_opts *io_opts) 178 { 179 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 180 int rc; 181 182 if (io_opts) { 183 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 184 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 185 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 186 bdev_io_opts = io_opts->user_ctx; 187 assert(bdev_io_opts); 188 } 189 190 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 191 bdev_blob_io_complete, cb_args, bdev_io_opts); 192 if (rc == -ENOMEM) { 193 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 194 io_opts); 195 } else if (rc != 0) { 196 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 197 } 198 } 199 200 static void 201 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 202 struct iovec *iov, int iovcnt, 203 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 204 struct spdk_blob_ext_io_opts *io_opts) 205 { 206 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 207 int rc; 208 209 if (io_opts) { 210 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 211 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 212 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 213 bdev_io_opts = io_opts->user_ctx; 214 assert(bdev_io_opts); 215 } 216 217 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 218 bdev_blob_io_complete, cb_args, bdev_io_opts); 219 if (rc == -ENOMEM) { 220 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 221 io_opts); 222 } else if (rc != 0) { 223 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 224 } 225 } 226 227 static void 228 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 229 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 230 { 231 int rc; 232 233 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 234 lba_count, bdev_blob_io_complete, cb_args); 235 if (rc == -ENOMEM) { 236 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 237 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 238 } else if (rc != 0) { 239 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 240 } 241 } 242 243 static void 244 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 245 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 246 { 247 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 248 int rc; 249 250 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 251 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 252 bdev_blob_io_complete, cb_args); 253 if (rc == -ENOMEM) { 254 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 255 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 256 } else if (rc != 0) { 257 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 258 } 259 } else { 260 /* 261 * If the device doesn't support unmap, immediately complete 262 * the request. Blobstore does not rely on unmap zeroing 263 * data. 264 */ 265 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 266 } 267 } 268 269 static void 270 bdev_blob_copy(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 271 uint64_t dst_lba, uint64_t src_lba, uint64_t lba_count, 272 struct spdk_bs_dev_cb_args *cb_args) 273 { 274 int rc; 275 276 rc = spdk_bdev_copy_blocks(__get_desc(dev), channel, 277 dst_lba, src_lba, lba_count, 278 bdev_blob_io_complete, cb_args); 279 if (rc == -ENOMEM) { 280 bdev_blob_queue_io(dev, channel, NULL, 0, dst_lba, src_lba, 281 lba_count, SPDK_BDEV_IO_TYPE_COPY, cb_args, NULL); 282 } else if (rc != 0) { 283 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 284 } 285 } 286 287 static void 288 bdev_blob_resubmit(void *arg) 289 { 290 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 291 292 switch (ctx->io_type) { 293 case SPDK_BDEV_IO_TYPE_READ: 294 if (ctx->iovcnt > 0) { 295 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 296 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 297 } else { 298 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 299 ctx->lba, ctx->lba_count, ctx->cb_args); 300 } 301 break; 302 case SPDK_BDEV_IO_TYPE_WRITE: 303 if (ctx->iovcnt > 0) { 304 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 305 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 306 } else { 307 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 308 ctx->lba, ctx->lba_count, ctx->cb_args); 309 } 310 break; 311 case SPDK_BDEV_IO_TYPE_UNMAP: 312 bdev_blob_unmap(ctx->dev, ctx->channel, 313 ctx->lba, ctx->lba_count, ctx->cb_args); 314 break; 315 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 316 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 317 ctx->lba, ctx->lba_count, ctx->cb_args); 318 break; 319 case SPDK_BDEV_IO_TYPE_COPY: 320 bdev_blob_copy(ctx->dev, ctx->channel, 321 ctx->lba, ctx->src_lba, ctx->lba_count, ctx->cb_args); 322 break; 323 default: 324 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 325 assert(false); 326 break; 327 } 328 free(ctx); 329 } 330 331 int 332 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 333 { 334 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 335 int rc; 336 337 rc = spdk_bdev_module_claim_bdev(blob_bdev->bdev, NULL, module); 338 if (rc != 0) { 339 SPDK_ERRLOG("could not claim bs dev\n"); 340 return rc; 341 } 342 343 blob_bdev->claimed = true; 344 345 return rc; 346 } 347 348 static struct spdk_io_channel * 349 bdev_blob_create_channel(struct spdk_bs_dev *dev) 350 { 351 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 352 353 return spdk_bdev_get_io_channel(blob_bdev->desc); 354 } 355 356 static void 357 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 358 { 359 spdk_put_io_channel(channel); 360 } 361 362 static void 363 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 364 { 365 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 366 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 367 368 if (blob_bdev->claimed) { 369 spdk_bdev_module_release_bdev(blob_bdev->bdev); 370 } 371 372 spdk_bdev_close(desc); 373 free(bs_dev); 374 } 375 376 static struct spdk_bdev * 377 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 378 { 379 return __get_bdev(bs_dev); 380 } 381 382 static bool 383 bdev_blob_is_zeroes(struct spdk_bs_dev *dev, uint64_t lba, uint64_t lba_count) 384 { 385 return false; 386 } 387 388 static bool 389 bdev_blob_translate_lba(struct spdk_bs_dev *dev, uint64_t lba, uint64_t *base_lba) 390 { 391 *base_lba = lba; 392 return true; 393 } 394 395 static void 396 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 397 { 398 struct spdk_bdev *bdev; 399 400 bdev = spdk_bdev_desc_get_bdev(desc); 401 assert(bdev != NULL); 402 403 b->bdev = bdev; 404 b->desc = desc; 405 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 406 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 407 b->bs_dev.create_channel = bdev_blob_create_channel; 408 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 409 b->bs_dev.destroy = bdev_blob_destroy; 410 b->bs_dev.read = bdev_blob_read; 411 b->bs_dev.write = bdev_blob_write; 412 b->bs_dev.readv = bdev_blob_readv; 413 b->bs_dev.writev = bdev_blob_writev; 414 b->bs_dev.readv_ext = bdev_blob_readv_ext; 415 b->bs_dev.writev_ext = bdev_blob_writev_ext; 416 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 417 b->bs_dev.unmap = bdev_blob_unmap; 418 if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 419 b->bs_dev.copy = bdev_blob_copy; 420 } 421 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 422 b->bs_dev.is_zeroes = bdev_blob_is_zeroes; 423 b->bs_dev.translate_lba = bdev_blob_translate_lba; 424 } 425 426 int 427 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 428 void *event_ctx, struct spdk_bs_dev **_bs_dev) 429 { 430 struct blob_bdev *b; 431 struct spdk_bdev_desc *desc; 432 int rc; 433 434 b = calloc(1, sizeof(*b)); 435 436 if (b == NULL) { 437 SPDK_ERRLOG("could not allocate blob_bdev\n"); 438 return -ENOMEM; 439 } 440 441 rc = spdk_bdev_open_ext(bdev_name, true, event_cb, event_ctx, &desc); 442 if (rc != 0) { 443 free(b); 444 return rc; 445 } 446 447 blob_bdev_init(b, desc); 448 449 *_bs_dev = &b->bs_dev; 450 451 return 0; 452 } 453