1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob_bdev.h" 10 #include "spdk/blob.h" 11 #include "spdk/thread.h" 12 #include "spdk/log.h" 13 #include "spdk/endian.h" 14 #define __SPDK_BDEV_MODULE_ONLY 15 #include "spdk/bdev_module.h" 16 17 struct blob_bdev { 18 struct spdk_bs_dev bs_dev; 19 struct spdk_bdev *bdev; 20 struct spdk_bdev_desc *desc; 21 }; 22 23 struct blob_resubmit { 24 struct spdk_bdev_io_wait_entry bdev_io_wait; 25 enum spdk_bdev_io_type io_type; 26 struct spdk_bs_dev *dev; 27 struct spdk_io_channel *channel; 28 void *payload; 29 int iovcnt; 30 uint64_t lba; 31 uint64_t src_lba; 32 uint32_t lba_count; 33 struct spdk_bs_dev_cb_args *cb_args; 34 struct spdk_blob_ext_io_opts *ext_io_opts; 35 }; 36 static void bdev_blob_resubmit(void *); 37 38 static inline struct spdk_bdev_desc * 39 __get_desc(struct spdk_bs_dev *dev) 40 { 41 return ((struct blob_bdev *)dev)->desc; 42 } 43 44 static inline struct spdk_bdev * 45 __get_bdev(struct spdk_bs_dev *dev) 46 { 47 return ((struct blob_bdev *)dev)->bdev; 48 } 49 50 static void 51 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 52 { 53 struct spdk_bs_dev_cb_args *cb_args = arg; 54 int bserrno; 55 56 if (success) { 57 bserrno = 0; 58 } else { 59 bserrno = -EIO; 60 } 61 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 62 spdk_bdev_free_io(bdev_io); 63 } 64 65 static void 66 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 67 int iovcnt, uint64_t lba, uint64_t src_lba, uint32_t lba_count, 68 enum spdk_bdev_io_type io_type, struct spdk_bs_dev_cb_args *cb_args, 69 struct spdk_blob_ext_io_opts *ext_io_opts) 70 { 71 int rc; 72 struct spdk_bdev *bdev = __get_bdev(dev); 73 struct blob_resubmit *ctx; 74 75 ctx = calloc(1, sizeof(struct blob_resubmit)); 76 77 if (ctx == NULL) { 78 SPDK_ERRLOG("Not enough memory to queue io\n"); 79 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 80 return; 81 } 82 83 ctx->io_type = io_type; 84 ctx->dev = dev; 85 ctx->channel = channel; 86 ctx->payload = payload; 87 ctx->iovcnt = iovcnt; 88 ctx->lba = lba; 89 ctx->src_lba = src_lba; 90 ctx->lba_count = lba_count; 91 ctx->cb_args = cb_args; 92 ctx->bdev_io_wait.bdev = bdev; 93 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 94 ctx->bdev_io_wait.cb_arg = ctx; 95 ctx->ext_io_opts = ext_io_opts; 96 97 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 98 if (rc != 0) { 99 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 100 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 101 free(ctx); 102 assert(false); 103 } 104 } 105 106 static void 107 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 108 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 109 { 110 int rc; 111 112 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 113 lba_count, bdev_blob_io_complete, cb_args); 114 if (rc == -ENOMEM) { 115 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 116 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 117 } else if (rc != 0) { 118 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 119 } 120 } 121 122 static void 123 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 124 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 125 { 126 int rc; 127 128 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 129 lba_count, bdev_blob_io_complete, cb_args); 130 if (rc == -ENOMEM) { 131 bdev_blob_queue_io(dev, channel, payload, 0, lba, 0, 132 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 133 } else if (rc != 0) { 134 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 135 } 136 } 137 138 static void 139 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 140 struct iovec *iov, int iovcnt, 141 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 142 { 143 int rc; 144 145 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 146 lba_count, bdev_blob_io_complete, cb_args); 147 if (rc == -ENOMEM) { 148 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 149 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 150 } else if (rc != 0) { 151 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 152 } 153 } 154 155 static void 156 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 157 struct iovec *iov, int iovcnt, 158 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 159 { 160 int rc; 161 162 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 163 lba_count, bdev_blob_io_complete, cb_args); 164 if (rc == -ENOMEM) { 165 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, 166 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 167 } else if (rc != 0) { 168 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 169 } 170 } 171 172 static inline void 173 blob_ext_io_opts_to_bdev_opts(struct spdk_bdev_ext_io_opts *dst, struct spdk_blob_ext_io_opts *src) 174 { 175 memset(dst, 0, sizeof(*dst)); 176 dst->size = sizeof(*dst); 177 dst->memory_domain = src->memory_domain; 178 dst->memory_domain_ctx = src->memory_domain_ctx; 179 } 180 181 static void 182 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 183 struct iovec *iov, int iovcnt, 184 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 185 struct spdk_blob_ext_io_opts *io_opts) 186 { 187 struct spdk_bdev_ext_io_opts bdev_io_opts; 188 int rc; 189 190 blob_ext_io_opts_to_bdev_opts(&bdev_io_opts, io_opts); 191 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 192 bdev_blob_io_complete, cb_args, &bdev_io_opts); 193 if (rc == -ENOMEM) { 194 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 195 io_opts); 196 } else if (rc != 0) { 197 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 198 } 199 } 200 201 static void 202 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 203 struct iovec *iov, int iovcnt, 204 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 205 struct spdk_blob_ext_io_opts *io_opts) 206 { 207 struct spdk_bdev_ext_io_opts bdev_io_opts; 208 int rc; 209 210 blob_ext_io_opts_to_bdev_opts(&bdev_io_opts, io_opts); 211 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 212 bdev_blob_io_complete, cb_args, &bdev_io_opts); 213 if (rc == -ENOMEM) { 214 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 0, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 215 io_opts); 216 } else if (rc != 0) { 217 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 218 } 219 } 220 221 static void 222 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 223 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 224 { 225 int rc; 226 227 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 228 lba_count, bdev_blob_io_complete, cb_args); 229 if (rc == -ENOMEM) { 230 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 231 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 232 } else if (rc != 0) { 233 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 234 } 235 } 236 237 static void 238 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 239 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 240 { 241 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 242 int rc; 243 244 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 245 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 246 bdev_blob_io_complete, cb_args); 247 if (rc == -ENOMEM) { 248 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 0, 249 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 250 } else if (rc != 0) { 251 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 252 } 253 } else { 254 /* 255 * If the device doesn't support unmap, immediately complete 256 * the request. Blobstore does not rely on unmap zeroing 257 * data. 258 */ 259 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 260 } 261 } 262 263 static void 264 bdev_blob_copy(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 265 uint64_t dst_lba, uint64_t src_lba, uint64_t lba_count, 266 struct spdk_bs_dev_cb_args *cb_args) 267 { 268 int rc; 269 270 rc = spdk_bdev_copy_blocks(__get_desc(dev), channel, 271 dst_lba, src_lba, lba_count, 272 bdev_blob_io_complete, cb_args); 273 if (rc == -ENOMEM) { 274 bdev_blob_queue_io(dev, channel, NULL, 0, dst_lba, src_lba, 275 lba_count, SPDK_BDEV_IO_TYPE_COPY, cb_args, NULL); 276 } else if (rc != 0) { 277 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 278 } 279 } 280 281 static void 282 bdev_blob_resubmit(void *arg) 283 { 284 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 285 286 switch (ctx->io_type) { 287 case SPDK_BDEV_IO_TYPE_READ: 288 if (ctx->iovcnt > 0) { 289 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 290 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 291 } else { 292 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 293 ctx->lba, ctx->lba_count, ctx->cb_args); 294 } 295 break; 296 case SPDK_BDEV_IO_TYPE_WRITE: 297 if (ctx->iovcnt > 0) { 298 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 299 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 300 } else { 301 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 302 ctx->lba, ctx->lba_count, ctx->cb_args); 303 } 304 break; 305 case SPDK_BDEV_IO_TYPE_UNMAP: 306 bdev_blob_unmap(ctx->dev, ctx->channel, 307 ctx->lba, ctx->lba_count, ctx->cb_args); 308 break; 309 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 310 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 311 ctx->lba, ctx->lba_count, ctx->cb_args); 312 break; 313 case SPDK_BDEV_IO_TYPE_COPY: 314 bdev_blob_copy(ctx->dev, ctx->channel, 315 ctx->lba, ctx->src_lba, ctx->lba_count, ctx->cb_args); 316 break; 317 default: 318 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 319 assert(false); 320 break; 321 } 322 free(ctx); 323 } 324 325 int 326 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 327 { 328 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 329 int rc; 330 331 rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, 332 NULL, module); 333 if (rc != 0) { 334 SPDK_ERRLOG("could not claim bs dev\n"); 335 return rc; 336 } 337 338 return rc; 339 } 340 341 static struct spdk_io_channel * 342 bdev_blob_create_channel(struct spdk_bs_dev *dev) 343 { 344 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 345 346 return spdk_bdev_get_io_channel(blob_bdev->desc); 347 } 348 349 static void 350 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 351 { 352 spdk_put_io_channel(channel); 353 } 354 355 static void 356 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 357 { 358 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 359 360 spdk_bdev_close(desc); 361 free(bs_dev); 362 } 363 364 static struct spdk_bdev * 365 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 366 { 367 return __get_bdev(bs_dev); 368 } 369 370 static bool 371 bdev_blob_is_zeroes(struct spdk_bs_dev *dev, uint64_t lba, uint64_t lba_count) 372 { 373 return false; 374 } 375 376 static bool 377 bdev_blob_translate_lba(struct spdk_bs_dev *dev, uint64_t lba, uint64_t *base_lba) 378 { 379 *base_lba = lba; 380 return true; 381 } 382 383 static void 384 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 385 { 386 struct spdk_bdev *bdev; 387 388 bdev = spdk_bdev_desc_get_bdev(desc); 389 assert(bdev != NULL); 390 391 b->bdev = bdev; 392 b->desc = desc; 393 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 394 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 395 b->bs_dev.create_channel = bdev_blob_create_channel; 396 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 397 b->bs_dev.destroy = bdev_blob_destroy; 398 b->bs_dev.read = bdev_blob_read; 399 b->bs_dev.write = bdev_blob_write; 400 b->bs_dev.readv = bdev_blob_readv; 401 b->bs_dev.writev = bdev_blob_writev; 402 b->bs_dev.readv_ext = bdev_blob_readv_ext; 403 b->bs_dev.writev_ext = bdev_blob_writev_ext; 404 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 405 b->bs_dev.unmap = bdev_blob_unmap; 406 if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 407 b->bs_dev.copy = bdev_blob_copy; 408 } 409 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 410 b->bs_dev.is_zeroes = bdev_blob_is_zeroes; 411 b->bs_dev.translate_lba = bdev_blob_translate_lba; 412 } 413 414 int 415 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 416 void *event_ctx, struct spdk_bs_dev **_bs_dev) 417 { 418 struct blob_bdev *b; 419 struct spdk_bdev_desc *desc; 420 int rc; 421 422 b = calloc(1, sizeof(*b)); 423 424 if (b == NULL) { 425 SPDK_ERRLOG("could not allocate blob_bdev\n"); 426 return -ENOMEM; 427 } 428 429 rc = spdk_bdev_open_ext(bdev_name, true, event_cb, event_ctx, &desc); 430 if (rc != 0) { 431 free(b); 432 return rc; 433 } 434 435 blob_bdev_init(b, desc); 436 437 *_bs_dev = &b->bs_dev; 438 439 return 0; 440 } 441