1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob_bdev.h" 10 #include "spdk/blob.h" 11 #include "spdk/thread.h" 12 #include "spdk/log.h" 13 #include "spdk/endian.h" 14 #define __SPDK_BDEV_MODULE_ONLY 15 #include "spdk/bdev_module.h" 16 17 struct blob_bdev { 18 struct spdk_bs_dev bs_dev; 19 struct spdk_bdev *bdev; 20 struct spdk_bdev_desc *desc; 21 }; 22 23 struct blob_resubmit { 24 struct spdk_bdev_io_wait_entry bdev_io_wait; 25 enum spdk_bdev_io_type io_type; 26 struct spdk_bs_dev *dev; 27 struct spdk_io_channel *channel; 28 void *payload; 29 int iovcnt; 30 uint64_t lba; 31 uint64_t src_lba; 32 uint32_t lba_count; 33 struct spdk_bs_dev_cb_args *cb_args; 34 struct spdk_blob_ext_io_opts *ext_io_opts; 35 }; 36 static void bdev_blob_resubmit(void *); 37 38 static inline struct spdk_bdev_desc * 39 __get_desc(struct spdk_bs_dev *dev) 40 { 41 return ((struct blob_bdev *)dev)->desc; 42 } 43 44 static inline struct spdk_bdev * 45 __get_bdev(struct spdk_bs_dev *dev) 46 { 47 return ((struct blob_bdev *)dev)->bdev; 48 } 49 50 static void 51 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 52 { 53 struct spdk_bs_dev_cb_args *cb_args = arg; 54 int bserrno; 55 56 if (success) { 57 bserrno = 0; 58 } else { 59 bserrno = -EIO; 60 } 61 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 62 spdk_bdev_free_io(bdev_io); 63 } 64 65 static void 66 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 67 int iovcnt, uint64_t lba, uint64_t src_lba, uint32_t lba_count, 68 enum spdk_bdev_io_type io_type, struct spdk_bs_dev_cb_args *cb_args, 69 struct spdk_blob_ext_io_opts *ext_io_opts) 70 { 71 int rc; 72 struct spdk_bdev *bdev = __get_bdev(dev); 73 struct blob_resubmit *ctx; 74 75 ctx = calloc(1, sizeof(struct blob_resubmit)); 76 77 if (ctx == NULL) { 78 SPDK_ERRLOG("Not enough memory to queue io\n"); 79 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 80 return; 81 } 82 83 ctx->io_type = io_type; 84 ctx->dev = dev; 85 ctx->channel = channel; 86 ctx->payload = payload; 87 ctx->iovcnt = iovcnt; 88 ctx->lba = lba; 89 ctx->src_lba = src_lba; 90 ctx->lba_count = lba_count; 91 ctx->cb_args = cb_args; 92 ctx->bdev_io_wait.bdev = bdev; 93 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 94 ctx->bdev_io_wait.cb_arg = ctx; 95 ctx->ext_io_opts = ext_io_opts; 96 97 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 98 if (rc != 0) { 99 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 100 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 101 free(ctx); 102 assert(false); 103 } 104 } 105 106 static void 107 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 108 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 109 { 110 int rc; 111 112 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 113 lba_count, bdev_blob_io_complete, cb_args); 114 if (rc == -ENOMEM) { 115 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 116 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 117 } else if (rc != 0) { 118 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 119 } 120 } 121 122 static void 123 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 124 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 125 { 126 int rc; 127 128 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 129 lba_count, bdev_blob_io_complete, cb_args); 130 if (rc == -ENOMEM) { 131 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 132 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 133 } else if (rc != 0) { 134 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 135 } 136 } 137 138 static void 139 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 140 struct iovec *iov, int iovcnt, 141 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 142 { 143 int rc; 144 145 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 146 lba_count, bdev_blob_io_complete, cb_args); 147 if (rc == -ENOMEM) { 148 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 149 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 150 } else if (rc != 0) { 151 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 152 } 153 } 154 155 static void 156 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 157 struct iovec *iov, int iovcnt, 158 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 159 { 160 int rc; 161 162 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 163 lba_count, bdev_blob_io_complete, cb_args); 164 if (rc == -ENOMEM) { 165 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 166 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 167 } else if (rc != 0) { 168 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 169 } 170 } 171 172 static void 173 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 174 struct iovec *iov, int iovcnt, 175 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 176 struct spdk_blob_ext_io_opts *io_opts) 177 { 178 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 179 int rc; 180 181 if (io_opts) { 182 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 183 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 184 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 185 bdev_io_opts = io_opts->user_ctx; 186 assert(bdev_io_opts); 187 } 188 189 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 190 bdev_blob_io_complete, cb_args, bdev_io_opts); 191 if (rc == -ENOMEM) { 192 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 193 io_opts); 194 } else if (rc != 0) { 195 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 196 } 197 } 198 199 static void 200 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 201 struct iovec *iov, int iovcnt, 202 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 203 struct spdk_blob_ext_io_opts *io_opts) 204 { 205 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 206 int rc; 207 208 if (io_opts) { 209 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 210 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 211 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 212 bdev_io_opts = io_opts->user_ctx; 213 assert(bdev_io_opts); 214 } 215 216 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 217 bdev_blob_io_complete, cb_args, bdev_io_opts); 218 if (rc == -ENOMEM) { 219 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 220 io_opts); 221 } else if (rc != 0) { 222 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 223 } 224 } 225 226 static void 227 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 228 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 229 { 230 int rc; 231 232 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 233 lba_count, bdev_blob_io_complete, cb_args); 234 if (rc == -ENOMEM) { 235 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 236 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 237 } else if (rc != 0) { 238 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 239 } 240 } 241 242 static void 243 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 244 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 245 { 246 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 247 int rc; 248 249 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 250 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 251 bdev_blob_io_complete, cb_args); 252 if (rc == -ENOMEM) { 253 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 254 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 255 } else if (rc != 0) { 256 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 257 } 258 } else { 259 /* 260 * If the device doesn't support unmap, immediately complete 261 * the request. Blobstore does not rely on unmap zeroing 262 * data. 263 */ 264 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 265 } 266 } 267 268 static void 269 bdev_blob_copy(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 270 uint64_t dst_lba, uint64_t src_lba, uint64_t lba_count, 271 struct spdk_bs_dev_cb_args *cb_args) 272 { 273 int rc; 274 275 rc = spdk_bdev_copy_blocks(__get_desc(dev), channel, 276 dst_lba, src_lba, lba_count, 277 bdev_blob_io_complete, cb_args); 278 if (rc == -ENOMEM) { 279 bdev_blob_queue_io(dev, channel, NULL, 0, dst_lba, src_lba, 280 lba_count, SPDK_BDEV_IO_TYPE_COPY, cb_args, NULL); 281 } else if (rc != 0) { 282 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 283 } 284 } 285 286 static void 287 bdev_blob_resubmit(void *arg) 288 { 289 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 290 291 switch (ctx->io_type) { 292 case SPDK_BDEV_IO_TYPE_READ: 293 if (ctx->iovcnt > 0) { 294 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 295 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 296 } else { 297 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 298 ctx->lba, ctx->lba_count, ctx->cb_args); 299 } 300 break; 301 case SPDK_BDEV_IO_TYPE_WRITE: 302 if (ctx->iovcnt > 0) { 303 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 304 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 305 } else { 306 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 307 ctx->lba, ctx->lba_count, ctx->cb_args); 308 } 309 break; 310 case SPDK_BDEV_IO_TYPE_UNMAP: 311 bdev_blob_unmap(ctx->dev, ctx->channel, 312 ctx->lba, ctx->lba_count, ctx->cb_args); 313 break; 314 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 315 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 316 ctx->lba, ctx->lba_count, ctx->cb_args); 317 break; 318 case SPDK_BDEV_IO_TYPE_COPY: 319 bdev_blob_copy(ctx->dev, ctx->channel, 320 ctx->lba, ctx->src_lba, ctx->lba_count, ctx->cb_args); 321 break; 322 default: 323 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 324 assert(false); 325 break; 326 } 327 free(ctx); 328 } 329 330 int 331 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 332 { 333 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 334 int rc; 335 336 rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, 337 NULL, module); 338 if (rc != 0) { 339 SPDK_ERRLOG("could not claim bs dev\n"); 340 return rc; 341 } 342 343 return rc; 344 } 345 346 static struct spdk_io_channel * 347 bdev_blob_create_channel(struct spdk_bs_dev *dev) 348 { 349 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 350 351 return spdk_bdev_get_io_channel(blob_bdev->desc); 352 } 353 354 static void 355 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 356 { 357 spdk_put_io_channel(channel); 358 } 359 360 static void 361 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 362 { 363 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 364 365 spdk_bdev_close(desc); 366 free(bs_dev); 367 } 368 369 static struct spdk_bdev * 370 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 371 { 372 return __get_bdev(bs_dev); 373 } 374 375 static bool 376 bdev_blob_is_zeroes(struct spdk_bs_dev *dev, uint64_t lba, uint64_t lba_count) 377 { 378 return false; 379 } 380 381 static bool 382 bdev_blob_translate_lba(struct spdk_bs_dev *dev, uint64_t lba, uint64_t *base_lba) 383 { 384 *base_lba = lba; 385 return true; 386 } 387 388 static void 389 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 390 { 391 struct spdk_bdev *bdev; 392 393 bdev = spdk_bdev_desc_get_bdev(desc); 394 assert(bdev != NULL); 395 396 b->bdev = bdev; 397 b->desc = desc; 398 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 399 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 400 b->bs_dev.create_channel = bdev_blob_create_channel; 401 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 402 b->bs_dev.destroy = bdev_blob_destroy; 403 b->bs_dev.read = bdev_blob_read; 404 b->bs_dev.write = bdev_blob_write; 405 b->bs_dev.readv = bdev_blob_readv; 406 b->bs_dev.writev = bdev_blob_writev; 407 b->bs_dev.readv_ext = bdev_blob_readv_ext; 408 b->bs_dev.writev_ext = bdev_blob_writev_ext; 409 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 410 b->bs_dev.unmap = bdev_blob_unmap; 411 if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 412 b->bs_dev.copy = bdev_blob_copy; 413 } 414 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 415 b->bs_dev.is_zeroes = bdev_blob_is_zeroes; 416 b->bs_dev.translate_lba = bdev_blob_translate_lba; 417 } 418 419 int 420 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 421 void *event_ctx, struct spdk_bs_dev **_bs_dev) 422 { 423 struct blob_bdev *b; 424 struct spdk_bdev_desc *desc; 425 int rc; 426 427 b = calloc(1, sizeof(*b)); 428 429 if (b == NULL) { 430 SPDK_ERRLOG("could not allocate blob_bdev\n"); 431 return -ENOMEM; 432 } 433 434 rc = spdk_bdev_open_ext(bdev_name, true, event_cb, event_ctx, &desc); 435 if (rc != 0) { 436 free(b); 437 return rc; 438 } 439 440 blob_bdev_init(b, desc); 441 442 *_bs_dev = &b->bs_dev; 443 444 return 0; 445 } 446