1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob_bdev.h" 10 #include "spdk/blob.h" 11 #include "spdk/thread.h" 12 #include "spdk/log.h" 13 #include "spdk/endian.h" 14 #define __SPDK_BDEV_MODULE_ONLY 15 #include "spdk/bdev_module.h" 16 17 struct blob_bdev { 18 struct spdk_bs_dev bs_dev; 19 struct spdk_bdev *bdev; 20 struct spdk_bdev_desc *desc; 21 bool write; 22 }; 23 24 struct blob_resubmit { 25 struct spdk_bdev_io_wait_entry bdev_io_wait; 26 enum spdk_bdev_io_type io_type; 27 struct spdk_bs_dev *dev; 28 struct spdk_io_channel *channel; 29 void *payload; 30 int iovcnt; 31 uint64_t lba; 32 uint64_t src_lba; 33 uint32_t lba_count; 34 struct spdk_bs_dev_cb_args *cb_args; 35 struct spdk_blob_ext_io_opts *ext_io_opts; 36 }; 37 static void bdev_blob_resubmit(void *); 38 39 static inline struct spdk_bdev_desc * 40 __get_desc(struct spdk_bs_dev *dev) 41 { 42 return ((struct blob_bdev *)dev)->desc; 43 } 44 45 static inline struct spdk_bdev * 46 __get_bdev(struct spdk_bs_dev *dev) 47 { 48 return ((struct blob_bdev *)dev)->bdev; 49 } 50 51 static void 52 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 53 { 54 struct spdk_bs_dev_cb_args *cb_args = arg; 55 int bserrno; 56 57 if (success) { 58 bserrno = 0; 59 } else { 60 bserrno = -EIO; 61 } 62 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 63 spdk_bdev_free_io(bdev_io); 64 } 65 66 static void 67 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 68 int iovcnt, uint64_t lba, uint64_t src_lba, uint32_t lba_count, 69 enum spdk_bdev_io_type io_type, struct spdk_bs_dev_cb_args *cb_args, 70 struct spdk_blob_ext_io_opts *ext_io_opts) 71 { 72 int rc; 73 struct spdk_bdev *bdev = __get_bdev(dev); 74 struct blob_resubmit *ctx; 75 76 ctx = calloc(1, sizeof(struct blob_resubmit)); 77 78 if (ctx == NULL) { 79 SPDK_ERRLOG("Not enough memory to queue io\n"); 80 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 81 return; 82 } 83 84 ctx->io_type = io_type; 85 ctx->dev = dev; 86 ctx->channel = channel; 87 ctx->payload = payload; 88 ctx->iovcnt = iovcnt; 89 ctx->lba = lba; 90 ctx->src_lba = src_lba; 91 ctx->lba_count = lba_count; 92 ctx->cb_args = cb_args; 93 ctx->bdev_io_wait.bdev = bdev; 94 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 95 ctx->bdev_io_wait.cb_arg = ctx; 96 ctx->ext_io_opts = ext_io_opts; 97 98 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 99 if (rc != 0) { 100 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 101 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 102 free(ctx); 103 assert(false); 104 } 105 } 106 107 static void 108 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 109 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 110 { 111 int rc; 112 113 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 114 lba_count, bdev_blob_io_complete, cb_args); 115 if (rc == -ENOMEM) { 116 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 117 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 118 } else if (rc != 0) { 119 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 120 } 121 } 122 123 static void 124 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 125 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 126 { 127 int rc; 128 129 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 130 lba_count, bdev_blob_io_complete, cb_args); 131 if (rc == -ENOMEM) { 132 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 133 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 134 } else if (rc != 0) { 135 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 136 } 137 } 138 139 static void 140 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 141 struct iovec *iov, int iovcnt, 142 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 143 { 144 int rc; 145 146 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 147 lba_count, bdev_blob_io_complete, cb_args); 148 if (rc == -ENOMEM) { 149 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 150 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 151 } else if (rc != 0) { 152 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 153 } 154 } 155 156 static void 157 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 158 struct iovec *iov, int iovcnt, 159 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 160 { 161 int rc; 162 163 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 164 lba_count, bdev_blob_io_complete, cb_args); 165 if (rc == -ENOMEM) { 166 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 167 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 168 } else if (rc != 0) { 169 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 170 } 171 } 172 173 static inline void 174 blob_ext_io_opts_to_bdev_opts(struct spdk_bdev_ext_io_opts *dst, struct spdk_blob_ext_io_opts *src) 175 { 176 memset(dst, 0, sizeof(*dst)); 177 dst->size = sizeof(*dst); 178 dst->memory_domain = src->memory_domain; 179 dst->memory_domain_ctx = src->memory_domain_ctx; 180 } 181 182 static void 183 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 184 struct iovec *iov, int iovcnt, 185 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 186 struct spdk_blob_ext_io_opts *io_opts) 187 { 188 struct spdk_bdev_ext_io_opts bdev_io_opts; 189 int rc; 190 191 blob_ext_io_opts_to_bdev_opts(&bdev_io_opts, io_opts); 192 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 193 bdev_blob_io_complete, cb_args, &bdev_io_opts); 194 if (rc == -ENOMEM) { 195 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 196 io_opts); 197 } else if (rc != 0) { 198 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 199 } 200 } 201 202 static void 203 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 204 struct iovec *iov, int iovcnt, 205 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 206 struct spdk_blob_ext_io_opts *io_opts) 207 { 208 struct spdk_bdev_ext_io_opts bdev_io_opts; 209 int rc; 210 211 blob_ext_io_opts_to_bdev_opts(&bdev_io_opts, io_opts); 212 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 213 bdev_blob_io_complete, cb_args, &bdev_io_opts); 214 if (rc == -ENOMEM) { 215 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 216 io_opts); 217 } else if (rc != 0) { 218 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 219 } 220 } 221 222 static void 223 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 224 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 225 { 226 int rc; 227 228 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 229 lba_count, bdev_blob_io_complete, cb_args); 230 if (rc == -ENOMEM) { 231 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 232 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 233 } else if (rc != 0) { 234 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 235 } 236 } 237 238 static void 239 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 240 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 241 { 242 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 243 int rc; 244 245 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 246 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 247 bdev_blob_io_complete, cb_args); 248 if (rc == -ENOMEM) { 249 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 250 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 251 } else if (rc != 0) { 252 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 253 } 254 } else { 255 /* 256 * If the device doesn't support unmap, immediately complete 257 * the request. Blobstore does not rely on unmap zeroing 258 * data. 259 */ 260 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 261 } 262 } 263 264 static void 265 bdev_blob_copy(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 266 uint64_t dst_lba, uint64_t src_lba, uint64_t lba_count, 267 struct spdk_bs_dev_cb_args *cb_args) 268 { 269 int rc; 270 271 rc = spdk_bdev_copy_blocks(__get_desc(dev), channel, 272 dst_lba, src_lba, lba_count, 273 bdev_blob_io_complete, cb_args); 274 if (rc == -ENOMEM) { 275 bdev_blob_queue_io(dev, channel, NULL, 0, dst_lba, src_lba, 276 lba_count, SPDK_BDEV_IO_TYPE_COPY, cb_args, NULL); 277 } else if (rc != 0) { 278 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 279 } 280 } 281 282 static void 283 bdev_blob_resubmit(void *arg) 284 { 285 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 286 287 switch (ctx->io_type) { 288 case SPDK_BDEV_IO_TYPE_READ: 289 if (ctx->iovcnt > 0) { 290 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 291 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 292 } else { 293 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 294 ctx->lba, ctx->lba_count, ctx->cb_args); 295 } 296 break; 297 case SPDK_BDEV_IO_TYPE_WRITE: 298 if (ctx->iovcnt > 0) { 299 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 300 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 301 } else { 302 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 303 ctx->lba, ctx->lba_count, ctx->cb_args); 304 } 305 break; 306 case SPDK_BDEV_IO_TYPE_UNMAP: 307 bdev_blob_unmap(ctx->dev, ctx->channel, 308 ctx->lba, ctx->lba_count, ctx->cb_args); 309 break; 310 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 311 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 312 ctx->lba, ctx->lba_count, ctx->cb_args); 313 break; 314 case SPDK_BDEV_IO_TYPE_COPY: 315 bdev_blob_copy(ctx->dev, ctx->channel, 316 ctx->lba, ctx->src_lba, ctx->lba_count, ctx->cb_args); 317 break; 318 default: 319 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 320 assert(false); 321 break; 322 } 323 free(ctx); 324 } 325 326 int 327 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 328 { 329 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 330 struct spdk_bdev_desc *desc = blob_bdev->desc; 331 enum spdk_bdev_claim_type claim_type; 332 int rc; 333 334 claim_type = blob_bdev->write ? SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE : 335 SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE; 336 rc = spdk_bdev_module_claim_bdev_desc(desc, claim_type, NULL, module); 337 if (rc != 0) { 338 SPDK_ERRLOG("could not claim bs dev\n"); 339 return rc; 340 } 341 342 return rc; 343 } 344 345 static struct spdk_io_channel * 346 bdev_blob_create_channel(struct spdk_bs_dev *dev) 347 { 348 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 349 350 return spdk_bdev_get_io_channel(blob_bdev->desc); 351 } 352 353 static void 354 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 355 { 356 spdk_put_io_channel(channel); 357 } 358 359 static void 360 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 361 { 362 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 363 364 spdk_bdev_close(desc); 365 free(bs_dev); 366 } 367 368 static struct spdk_bdev * 369 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 370 { 371 return __get_bdev(bs_dev); 372 } 373 374 static bool 375 bdev_blob_is_zeroes(struct spdk_bs_dev *dev, uint64_t lba, uint64_t lba_count) 376 { 377 return false; 378 } 379 380 static bool 381 bdev_blob_translate_lba(struct spdk_bs_dev *dev, uint64_t lba, uint64_t *base_lba) 382 { 383 *base_lba = lba; 384 return true; 385 } 386 387 static void 388 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 389 { 390 struct spdk_bdev *bdev; 391 392 bdev = spdk_bdev_desc_get_bdev(desc); 393 assert(bdev != NULL); 394 395 b->bdev = bdev; 396 b->desc = desc; 397 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 398 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 399 b->bs_dev.create_channel = bdev_blob_create_channel; 400 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 401 b->bs_dev.destroy = bdev_blob_destroy; 402 b->bs_dev.read = bdev_blob_read; 403 b->bs_dev.write = bdev_blob_write; 404 b->bs_dev.readv = bdev_blob_readv; 405 b->bs_dev.writev = bdev_blob_writev; 406 b->bs_dev.readv_ext = bdev_blob_readv_ext; 407 b->bs_dev.writev_ext = bdev_blob_writev_ext; 408 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 409 b->bs_dev.unmap = bdev_blob_unmap; 410 if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 411 b->bs_dev.copy = bdev_blob_copy; 412 } 413 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 414 b->bs_dev.is_zeroes = bdev_blob_is_zeroes; 415 b->bs_dev.translate_lba = bdev_blob_translate_lba; 416 } 417 418 int 419 spdk_bdev_create_bs_dev(const char *bdev_name, bool write, 420 struct spdk_bdev_bs_dev_opts *opts, size_t opts_size, 421 spdk_bdev_event_cb_t event_cb, void *event_ctx, 422 struct spdk_bs_dev **bs_dev) 423 { 424 struct blob_bdev *b; 425 struct spdk_bdev_desc *desc; 426 int rc; 427 428 if (opts != NULL && opts_size != sizeof(*opts)) { 429 SPDK_ERRLOG("bdev name '%s': unsupported options\n", bdev_name); 430 return -EINVAL; 431 } 432 433 b = calloc(1, sizeof(*b)); 434 435 if (b == NULL) { 436 SPDK_ERRLOG("could not allocate blob_bdev\n"); 437 return -ENOMEM; 438 } 439 440 rc = spdk_bdev_open_ext(bdev_name, write, event_cb, event_ctx, &desc); 441 if (rc != 0) { 442 free(b); 443 return rc; 444 } 445 446 blob_bdev_init(b, desc); 447 448 *bs_dev = &b->bs_dev; 449 b->write = write; 450 451 return 0; 452 } 453 454 int 455 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 456 void *event_ctx, struct spdk_bs_dev **bs_dev) 457 { 458 return spdk_bdev_create_bs_dev(bdev_name, true, NULL, 0, event_cb, event_ctx, bs_dev); 459 } 460