1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob_bdev.h" 10 #include "spdk/blob.h" 11 #include "spdk/thread.h" 12 #include "spdk/log.h" 13 #include "spdk/endian.h" 14 #define __SPDK_BDEV_MODULE_ONLY 15 #include "spdk/bdev_module.h" 16 17 struct blob_bdev { 18 struct spdk_bs_dev bs_dev; 19 struct spdk_bdev *bdev; 20 struct spdk_bdev_desc *desc; 21 bool claimed; 22 }; 23 24 struct blob_resubmit { 25 struct spdk_bdev_io_wait_entry bdev_io_wait; 26 enum spdk_bdev_io_type io_type; 27 struct spdk_bs_dev *dev; 28 struct spdk_io_channel *channel; 29 void *payload; 30 int iovcnt; 31 uint64_t lba; 32 uint32_t lba_count; 33 struct spdk_bs_dev_cb_args *cb_args; 34 struct spdk_blob_ext_io_opts *ext_io_opts; 35 }; 36 static void bdev_blob_resubmit(void *); 37 38 static inline struct spdk_bdev_desc * 39 __get_desc(struct spdk_bs_dev *dev) 40 { 41 return ((struct blob_bdev *)dev)->desc; 42 } 43 44 static inline struct spdk_bdev * 45 __get_bdev(struct spdk_bs_dev *dev) 46 { 47 return ((struct blob_bdev *)dev)->bdev; 48 } 49 50 static void 51 bdev_blob_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *arg) 52 { 53 struct spdk_bs_dev_cb_args *cb_args = arg; 54 int bserrno; 55 56 if (success) { 57 bserrno = 0; 58 } else { 59 bserrno = -EIO; 60 } 61 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, bserrno); 62 spdk_bdev_free_io(bdev_io); 63 } 64 65 static void 66 bdev_blob_queue_io(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 67 int iovcnt, uint64_t lba, uint32_t lba_count, enum spdk_bdev_io_type io_type, 68 struct spdk_bs_dev_cb_args *cb_args, struct spdk_blob_ext_io_opts *ext_io_opts) 69 { 70 int rc; 71 struct spdk_bdev *bdev = __get_bdev(dev); 72 struct blob_resubmit *ctx; 73 74 ctx = calloc(1, sizeof(struct blob_resubmit)); 75 76 if (ctx == NULL) { 77 SPDK_ERRLOG("Not enough memory to queue io\n"); 78 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, -ENOMEM); 79 return; 80 } 81 82 ctx->io_type = io_type; 83 ctx->dev = dev; 84 ctx->channel = channel; 85 ctx->payload = payload; 86 ctx->iovcnt = iovcnt; 87 ctx->lba = lba; 88 ctx->lba_count = lba_count; 89 ctx->cb_args = cb_args; 90 ctx->bdev_io_wait.bdev = bdev; 91 ctx->bdev_io_wait.cb_fn = bdev_blob_resubmit; 92 ctx->bdev_io_wait.cb_arg = ctx; 93 ctx->ext_io_opts = ext_io_opts; 94 95 rc = spdk_bdev_queue_io_wait(bdev, channel, &ctx->bdev_io_wait); 96 if (rc != 0) { 97 SPDK_ERRLOG("Queue io failed, rc=%d\n", rc); 98 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 99 free(ctx); 100 assert(false); 101 } 102 } 103 104 static void 105 bdev_blob_read(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 106 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 107 { 108 int rc; 109 110 rc = spdk_bdev_read_blocks(__get_desc(dev), channel, payload, lba, 111 lba_count, bdev_blob_io_complete, cb_args); 112 if (rc == -ENOMEM) { 113 bdev_blob_queue_io(dev, channel, payload, 0, lba, 114 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 115 } else if (rc != 0) { 116 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 117 } 118 } 119 120 static void 121 bdev_blob_write(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, void *payload, 122 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 123 { 124 int rc; 125 126 rc = spdk_bdev_write_blocks(__get_desc(dev), channel, payload, lba, 127 lba_count, bdev_blob_io_complete, cb_args); 128 if (rc == -ENOMEM) { 129 bdev_blob_queue_io(dev, channel, payload, 0, lba, 130 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 131 } else if (rc != 0) { 132 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 133 } 134 } 135 136 static void 137 bdev_blob_readv(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 138 struct iovec *iov, int iovcnt, 139 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 140 { 141 int rc; 142 143 rc = spdk_bdev_readv_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 144 lba_count, bdev_blob_io_complete, cb_args); 145 if (rc == -ENOMEM) { 146 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 147 lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, NULL); 148 } else if (rc != 0) { 149 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 150 } 151 } 152 153 static void 154 bdev_blob_writev(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 155 struct iovec *iov, int iovcnt, 156 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 157 { 158 int rc; 159 160 rc = spdk_bdev_writev_blocks(__get_desc(dev), channel, iov, iovcnt, lba, 161 lba_count, bdev_blob_io_complete, cb_args); 162 if (rc == -ENOMEM) { 163 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, 164 lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, NULL); 165 } else if (rc != 0) { 166 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 167 } 168 } 169 170 static void 171 bdev_blob_readv_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 172 struct iovec *iov, int iovcnt, 173 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 174 struct spdk_blob_ext_io_opts *io_opts) 175 { 176 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 177 int rc; 178 179 if (io_opts) { 180 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 181 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 182 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 183 bdev_io_opts = io_opts->user_ctx; 184 assert(bdev_io_opts); 185 } 186 187 rc = spdk_bdev_readv_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 188 bdev_blob_io_complete, cb_args, bdev_io_opts); 189 if (rc == -ENOMEM) { 190 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, lba_count, SPDK_BDEV_IO_TYPE_READ, cb_args, 191 io_opts); 192 } else if (rc != 0) { 193 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 194 } 195 } 196 197 static void 198 bdev_blob_writev_ext(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, 199 struct iovec *iov, int iovcnt, 200 uint64_t lba, uint32_t lba_count, struct spdk_bs_dev_cb_args *cb_args, 201 struct spdk_blob_ext_io_opts *io_opts) 202 { 203 struct spdk_bdev_ext_io_opts *bdev_io_opts = NULL; 204 int rc; 205 206 if (io_opts) { 207 /* bdev ext API requires ext_io_opts to be allocated by the user, we don't have enough context to allocate 208 * bdev ext_opts structure here. Also blob and bdev ext_opts are not API/ABI compatible, so we can't use the given 209 * io_opts. Restore ext_opts passed by the user of this bs_dev */ 210 bdev_io_opts = io_opts->user_ctx; 211 assert(bdev_io_opts); 212 } 213 214 rc = spdk_bdev_writev_blocks_ext(__get_desc(dev), channel, iov, iovcnt, lba, lba_count, 215 bdev_blob_io_complete, cb_args, bdev_io_opts); 216 if (rc == -ENOMEM) { 217 bdev_blob_queue_io(dev, channel, iov, iovcnt, lba, lba_count, SPDK_BDEV_IO_TYPE_WRITE, cb_args, 218 io_opts); 219 } else if (rc != 0) { 220 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 221 } 222 } 223 224 static void 225 bdev_blob_write_zeroes(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 226 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 227 { 228 int rc; 229 230 rc = spdk_bdev_write_zeroes_blocks(__get_desc(dev), channel, lba, 231 lba_count, bdev_blob_io_complete, cb_args); 232 if (rc == -ENOMEM) { 233 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 234 lba_count, SPDK_BDEV_IO_TYPE_WRITE_ZEROES, cb_args, NULL); 235 } else if (rc != 0) { 236 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 237 } 238 } 239 240 static void 241 bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64_t lba, 242 uint64_t lba_count, struct spdk_bs_dev_cb_args *cb_args) 243 { 244 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 245 int rc; 246 247 if (spdk_bdev_io_type_supported(blob_bdev->bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { 248 rc = spdk_bdev_unmap_blocks(__get_desc(dev), channel, lba, lba_count, 249 bdev_blob_io_complete, cb_args); 250 if (rc == -ENOMEM) { 251 bdev_blob_queue_io(dev, channel, NULL, 0, lba, 252 lba_count, SPDK_BDEV_IO_TYPE_UNMAP, cb_args, NULL); 253 } else if (rc != 0) { 254 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, rc); 255 } 256 } else { 257 /* 258 * If the device doesn't support unmap, immediately complete 259 * the request. Blobstore does not rely on unmap zeroing 260 * data. 261 */ 262 cb_args->cb_fn(cb_args->channel, cb_args->cb_arg, 0); 263 } 264 } 265 266 static void 267 bdev_blob_resubmit(void *arg) 268 { 269 struct blob_resubmit *ctx = (struct blob_resubmit *) arg; 270 271 switch (ctx->io_type) { 272 case SPDK_BDEV_IO_TYPE_READ: 273 if (ctx->iovcnt > 0) { 274 bdev_blob_readv_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 275 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 276 } else { 277 bdev_blob_read(ctx->dev, ctx->channel, ctx->payload, 278 ctx->lba, ctx->lba_count, ctx->cb_args); 279 } 280 break; 281 case SPDK_BDEV_IO_TYPE_WRITE: 282 if (ctx->iovcnt > 0) { 283 bdev_blob_writev_ext(ctx->dev, ctx->channel, (struct iovec *) ctx->payload, ctx->iovcnt, 284 ctx->lba, ctx->lba_count, ctx->cb_args, ctx->ext_io_opts); 285 } else { 286 bdev_blob_write(ctx->dev, ctx->channel, ctx->payload, 287 ctx->lba, ctx->lba_count, ctx->cb_args); 288 } 289 break; 290 case SPDK_BDEV_IO_TYPE_UNMAP: 291 bdev_blob_unmap(ctx->dev, ctx->channel, 292 ctx->lba, ctx->lba_count, ctx->cb_args); 293 break; 294 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 295 bdev_blob_write_zeroes(ctx->dev, ctx->channel, 296 ctx->lba, ctx->lba_count, ctx->cb_args); 297 break; 298 default: 299 SPDK_ERRLOG("Unsupported io type %d\n", ctx->io_type); 300 assert(false); 301 break; 302 } 303 free(ctx); 304 } 305 306 int 307 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) 308 { 309 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 310 int rc; 311 312 rc = spdk_bdev_module_claim_bdev(blob_bdev->bdev, NULL, module); 313 if (rc != 0) { 314 SPDK_ERRLOG("could not claim bs dev\n"); 315 return rc; 316 } 317 318 blob_bdev->claimed = true; 319 320 return rc; 321 } 322 323 static struct spdk_io_channel * 324 bdev_blob_create_channel(struct spdk_bs_dev *dev) 325 { 326 struct blob_bdev *blob_bdev = (struct blob_bdev *)dev; 327 328 return spdk_bdev_get_io_channel(blob_bdev->desc); 329 } 330 331 static void 332 bdev_blob_destroy_channel(struct spdk_bs_dev *dev, struct spdk_io_channel *channel) 333 { 334 spdk_put_io_channel(channel); 335 } 336 337 static void 338 bdev_blob_destroy(struct spdk_bs_dev *bs_dev) 339 { 340 struct spdk_bdev_desc *desc = __get_desc(bs_dev); 341 struct blob_bdev *blob_bdev = (struct blob_bdev *)bs_dev; 342 343 if (blob_bdev->claimed) { 344 spdk_bdev_module_release_bdev(blob_bdev->bdev); 345 } 346 347 spdk_bdev_close(desc); 348 free(bs_dev); 349 } 350 351 static struct spdk_bdev * 352 bdev_blob_get_base_bdev(struct spdk_bs_dev *bs_dev) 353 { 354 return __get_bdev(bs_dev); 355 } 356 357 static bool 358 bdev_blob_is_zeroes(struct spdk_bs_dev *dev, uint64_t lba, uint64_t lba_count) 359 { 360 return false; 361 } 362 363 static void 364 blob_bdev_init(struct blob_bdev *b, struct spdk_bdev_desc *desc) 365 { 366 struct spdk_bdev *bdev; 367 368 bdev = spdk_bdev_desc_get_bdev(desc); 369 assert(bdev != NULL); 370 371 b->bdev = bdev; 372 b->desc = desc; 373 b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev); 374 b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev); 375 b->bs_dev.create_channel = bdev_blob_create_channel; 376 b->bs_dev.destroy_channel = bdev_blob_destroy_channel; 377 b->bs_dev.destroy = bdev_blob_destroy; 378 b->bs_dev.read = bdev_blob_read; 379 b->bs_dev.write = bdev_blob_write; 380 b->bs_dev.readv = bdev_blob_readv; 381 b->bs_dev.writev = bdev_blob_writev; 382 b->bs_dev.readv_ext = bdev_blob_readv_ext; 383 b->bs_dev.writev_ext = bdev_blob_writev_ext; 384 b->bs_dev.write_zeroes = bdev_blob_write_zeroes; 385 b->bs_dev.unmap = bdev_blob_unmap; 386 b->bs_dev.get_base_bdev = bdev_blob_get_base_bdev; 387 b->bs_dev.is_zeroes = bdev_blob_is_zeroes; 388 } 389 390 int 391 spdk_bdev_create_bs_dev_ext(const char *bdev_name, spdk_bdev_event_cb_t event_cb, 392 void *event_ctx, struct spdk_bs_dev **_bs_dev) 393 { 394 struct blob_bdev *b; 395 struct spdk_bdev_desc *desc; 396 int rc; 397 398 b = calloc(1, sizeof(*b)); 399 400 if (b == NULL) { 401 SPDK_ERRLOG("could not allocate blob_bdev\n"); 402 return -ENOMEM; 403 } 404 405 rc = spdk_bdev_open_ext(bdev_name, true, event_cb, event_ctx, &desc); 406 if (rc != 0) { 407 free(b); 408 return rc; 409 } 410 411 blob_bdev_init(b, desc); 412 413 *_bs_dev = &b->bs_dev; 414 415 return 0; 416 } 417