1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 /* 7 * This is a simple example of a virtual block device that takes a single 8 * bdev and slices it into multiple smaller bdevs. 9 */ 10 11 #include "vbdev_split.h" 12 13 #include "spdk/rpc.h" 14 #include "spdk/endian.h" 15 #include "spdk/string.h" 16 #include "spdk/thread.h" 17 #include "spdk/util.h" 18 19 #include "spdk/bdev_module.h" 20 #include "spdk/log.h" 21 22 struct spdk_vbdev_split_config { 23 char *base_bdev; 24 unsigned split_count; 25 uint64_t split_size_mb; 26 27 SPDK_BDEV_PART_TAILQ splits; 28 struct spdk_bdev_part_base *split_base; 29 30 TAILQ_ENTRY(spdk_vbdev_split_config) tailq; 31 }; 32 33 static TAILQ_HEAD(, spdk_vbdev_split_config) g_split_config = TAILQ_HEAD_INITIALIZER( 34 g_split_config); 35 36 struct vbdev_split_channel { 37 struct spdk_bdev_part_channel part_ch; 38 }; 39 40 struct vbdev_split_bdev_io { 41 struct spdk_io_channel *ch; 42 struct spdk_bdev_io *bdev_io; 43 44 /* for bdev_io_wait */ 45 struct spdk_bdev_io_wait_entry bdev_io_wait; 46 }; 47 48 static void vbdev_split_del_config(struct spdk_vbdev_split_config *cfg); 49 50 static int vbdev_split_init(void); 51 static void vbdev_split_fini(void); 52 static void vbdev_split_examine(struct spdk_bdev *bdev); 53 static int vbdev_split_config_json(struct spdk_json_write_ctx *w); 54 static int vbdev_split_get_ctx_size(void); 55 56 static void 57 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io); 58 59 static struct spdk_bdev_module split_if = { 60 .name = "split", 61 .module_init = vbdev_split_init, 62 .module_fini = vbdev_split_fini, 63 .get_ctx_size = vbdev_split_get_ctx_size, 64 .examine_config = vbdev_split_examine, 65 .config_json = vbdev_split_config_json, 66 }; 67 68 SPDK_BDEV_MODULE_REGISTER(split, &split_if) 69 70 static void 71 vbdev_split_base_free(void *ctx) 72 { 73 struct spdk_vbdev_split_config *cfg = ctx; 74 75 vbdev_split_del_config(cfg); 76 } 77 78 static int 79 _vbdev_split_destruct(void *ctx) 80 { 81 struct spdk_bdev_part *part = ctx; 82 83 return spdk_bdev_part_free(part); 84 } 85 86 static void 87 vbdev_split_base_bdev_hotremove_cb(void *_part_base) 88 { 89 struct spdk_bdev_part_base *part_base = _part_base; 90 struct spdk_vbdev_split_config *cfg = spdk_bdev_part_base_get_ctx(part_base); 91 92 spdk_bdev_part_base_hotremove(part_base, &cfg->splits); 93 } 94 95 static void 96 vbdev_split_resubmit_io(void *arg) 97 { 98 struct vbdev_split_bdev_io *split_io = (struct vbdev_split_bdev_io *)arg; 99 100 _vbdev_split_submit_request(split_io->ch, split_io->bdev_io); 101 } 102 103 static void 104 vbdev_split_queue_io(struct vbdev_split_bdev_io *split_io) 105 { 106 struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(split_io->ch); 107 int rc; 108 109 split_io->bdev_io_wait.bdev = split_io->bdev_io->bdev; 110 split_io->bdev_io_wait.cb_fn = vbdev_split_resubmit_io; 111 split_io->bdev_io_wait.cb_arg = split_io; 112 113 rc = spdk_bdev_queue_io_wait(split_io->bdev_io->bdev, 114 ch->part_ch.base_ch, &split_io->bdev_io_wait); 115 if (rc != 0) { 116 SPDK_ERRLOG("Queue io failed in vbdev_split_queue_io, rc=%d\n", rc); 117 spdk_bdev_io_complete(split_io->bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 118 } 119 } 120 121 static void 122 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 123 { 124 struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(_ch); 125 struct vbdev_split_bdev_io *io_ctx = (struct vbdev_split_bdev_io *)bdev_io->driver_ctx; 126 int rc; 127 128 rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); 129 if (rc) { 130 if (rc == -ENOMEM) { 131 SPDK_DEBUGLOG(vbdev_split, "split: no memory, queue io.\n"); 132 io_ctx->ch = _ch; 133 io_ctx->bdev_io = bdev_io; 134 vbdev_split_queue_io(io_ctx); 135 } else { 136 SPDK_ERRLOG("split: error on io submission, rc=%d.\n", rc); 137 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 138 } 139 } 140 } 141 142 static void 143 vbdev_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success) 144 { 145 if (!success) { 146 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 147 return; 148 } 149 150 _vbdev_split_submit_request(ch, bdev_io); 151 } 152 153 static void 154 vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 155 { 156 switch (bdev_io->type) { 157 case SPDK_BDEV_IO_TYPE_READ: 158 spdk_bdev_io_get_buf(bdev_io, vbdev_split_get_buf_cb, 159 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 160 break; 161 default: 162 _vbdev_split_submit_request(_ch, bdev_io); 163 break; 164 } 165 } 166 167 static int 168 vbdev_split_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 169 { 170 struct spdk_bdev_part *part = ctx; 171 struct spdk_bdev *split_base_bdev = spdk_bdev_part_get_base_bdev(part); 172 uint64_t offset_blocks = spdk_bdev_part_get_offset_blocks(part); 173 174 spdk_json_write_named_object_begin(w, "split"); 175 176 spdk_json_write_named_string(w, "base_bdev", spdk_bdev_get_name(split_base_bdev)); 177 spdk_json_write_named_uint64(w, "offset_blocks", offset_blocks); 178 179 spdk_json_write_object_end(w); 180 181 return 0; 182 } 183 184 static void 185 vbdev_split_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 186 { 187 /* No config per bdev needed */ 188 } 189 190 static struct spdk_bdev_fn_table vbdev_split_fn_table = { 191 .destruct = _vbdev_split_destruct, 192 .submit_request = vbdev_split_submit_request, 193 .dump_info_json = vbdev_split_dump_info_json, 194 .write_config_json = vbdev_split_write_config_json 195 }; 196 197 static int 198 vbdev_split_create(struct spdk_vbdev_split_config *cfg) 199 { 200 uint64_t split_size_blocks, offset_blocks; 201 uint64_t split_count, max_split_count; 202 uint64_t mb = 1024 * 1024; 203 uint64_t i; 204 int rc; 205 char *name; 206 struct spdk_bdev *base_bdev; 207 struct bdev_part_tailq *split_base_tailq; 208 209 assert(cfg->split_count > 0); 210 211 TAILQ_INIT(&cfg->splits); 212 rc = spdk_bdev_part_base_construct_ext(cfg->base_bdev, 213 vbdev_split_base_bdev_hotremove_cb, 214 &split_if, &vbdev_split_fn_table, 215 &cfg->splits, vbdev_split_base_free, cfg, 216 sizeof(struct vbdev_split_channel), 217 NULL, NULL, &cfg->split_base); 218 if (rc != 0) { 219 if (rc != -ENODEV) { 220 SPDK_ERRLOG("Cannot construct bdev part base\n"); 221 } 222 return rc; 223 } 224 225 base_bdev = spdk_bdev_part_base_get_bdev(cfg->split_base); 226 227 if (cfg->split_size_mb) { 228 if (((cfg->split_size_mb * mb) % base_bdev->blocklen) != 0) { 229 SPDK_ERRLOG("Split size %" PRIu64 " MB is not possible with block size " 230 "%" PRIu32 "\n", 231 cfg->split_size_mb, base_bdev->blocklen); 232 rc = -EINVAL; 233 goto err; 234 } 235 split_size_blocks = (cfg->split_size_mb * mb) / base_bdev->blocklen; 236 SPDK_DEBUGLOG(vbdev_split, "Split size %" PRIu64 " MB specified by user\n", 237 cfg->split_size_mb); 238 } else { 239 split_size_blocks = base_bdev->blockcnt / cfg->split_count; 240 SPDK_DEBUGLOG(vbdev_split, "Split size not specified by user\n"); 241 } 242 243 max_split_count = base_bdev->blockcnt / split_size_blocks; 244 split_count = cfg->split_count; 245 if (split_count > max_split_count) { 246 SPDK_WARNLOG("Split count %" PRIu64 " is greater than maximum possible split count " 247 "%" PRIu64 " - clamping\n", split_count, max_split_count); 248 split_count = max_split_count; 249 } 250 251 SPDK_DEBUGLOG(vbdev_split, "base_bdev: %s split_count: %" PRIu64 252 " split_size_blocks: %" PRIu64 "\n", 253 cfg->base_bdev, split_count, split_size_blocks); 254 255 offset_blocks = 0; 256 for (i = 0; i < split_count; i++) { 257 struct spdk_bdev_part *d; 258 259 d = calloc(1, sizeof(*d)); 260 if (d == NULL) { 261 SPDK_ERRLOG("could not allocate bdev part\n"); 262 rc = -ENOMEM; 263 goto err; 264 } 265 266 name = spdk_sprintf_alloc("%sp%" PRIu64, cfg->base_bdev, i); 267 if (!name) { 268 SPDK_ERRLOG("could not allocate name\n"); 269 free(d); 270 rc = -ENOMEM; 271 goto err; 272 } 273 274 rc = spdk_bdev_part_construct(d, cfg->split_base, name, offset_blocks, split_size_blocks, 275 "Split Disk"); 276 free(name); 277 if (rc) { 278 SPDK_ERRLOG("could not construct bdev part\n"); 279 /* spdk_bdev_part_construct will free name if it fails */ 280 free(d); 281 rc = -ENOMEM; 282 goto err; 283 } 284 285 offset_blocks += split_size_blocks; 286 } 287 288 return 0; 289 err: 290 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 291 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 292 spdk_bdev_part_base_free(cfg->split_base); 293 return rc; 294 } 295 296 static void 297 vbdev_split_del_config(struct spdk_vbdev_split_config *cfg) 298 { 299 TAILQ_REMOVE(&g_split_config, cfg, tailq); 300 free(cfg->base_bdev); 301 free(cfg); 302 } 303 304 static void 305 vbdev_split_destruct_config(struct spdk_vbdev_split_config *cfg) 306 { 307 struct bdev_part_tailq *split_base_tailq; 308 309 if (cfg->split_base != NULL) { 310 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 311 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 312 } else { 313 vbdev_split_del_config(cfg); 314 } 315 } 316 317 static void 318 vbdev_split_clear_config(void) 319 { 320 struct spdk_vbdev_split_config *cfg, *tmp_cfg; 321 322 TAILQ_FOREACH_SAFE(cfg, &g_split_config, tailq, tmp_cfg) { 323 vbdev_split_destruct_config(cfg); 324 } 325 } 326 327 static struct spdk_vbdev_split_config * 328 vbdev_split_config_find_by_base_name(const char *base_bdev_name) 329 { 330 struct spdk_vbdev_split_config *cfg; 331 332 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 333 if (strcmp(cfg->base_bdev, base_bdev_name) == 0) { 334 return cfg; 335 } 336 } 337 338 return NULL; 339 } 340 341 static int 342 vbdev_split_add_config(const char *base_bdev_name, unsigned split_count, uint64_t split_size, 343 struct spdk_vbdev_split_config **config) 344 { 345 struct spdk_vbdev_split_config *cfg; 346 assert(base_bdev_name); 347 348 if (base_bdev_name == NULL) { 349 SPDK_ERRLOG("Split bdev config: no base bdev provided."); 350 return -EINVAL; 351 } 352 353 if (split_count == 0) { 354 SPDK_ERRLOG("Split bdev config: split_count can't be 0."); 355 return -EINVAL; 356 } 357 358 /* Check if we already have 'base_bdev_name' registered in config */ 359 cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 360 if (cfg) { 361 SPDK_ERRLOG("Split bdev config for base bdev '%s' already exist.", base_bdev_name); 362 return -EEXIST; 363 } 364 365 cfg = calloc(1, sizeof(*cfg)); 366 if (!cfg) { 367 SPDK_ERRLOG("calloc(): Out of memory"); 368 return -ENOMEM; 369 } 370 371 cfg->base_bdev = strdup(base_bdev_name); 372 if (!cfg->base_bdev) { 373 SPDK_ERRLOG("strdup(): Out of memory"); 374 free(cfg); 375 return -ENOMEM; 376 } 377 378 cfg->split_count = split_count; 379 cfg->split_size_mb = split_size; 380 TAILQ_INSERT_TAIL(&g_split_config, cfg, tailq); 381 if (config) { 382 *config = cfg; 383 } 384 385 return 0; 386 } 387 388 static int 389 vbdev_split_init(void) 390 { 391 return 0; 392 } 393 394 static void 395 vbdev_split_fini(void) 396 { 397 vbdev_split_clear_config(); 398 } 399 400 static void 401 vbdev_split_examine(struct spdk_bdev *bdev) 402 { 403 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(bdev->name); 404 405 if (cfg != NULL) { 406 assert(cfg->split_base == NULL); 407 408 if (vbdev_split_create(cfg)) { 409 SPDK_ERRLOG("could not split bdev %s\n", bdev->name); 410 } 411 } 412 spdk_bdev_module_examine_done(&split_if); 413 } 414 415 static int 416 vbdev_split_config_json(struct spdk_json_write_ctx *w) 417 { 418 struct spdk_vbdev_split_config *cfg; 419 420 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 421 spdk_json_write_object_begin(w); 422 423 spdk_json_write_named_string(w, "method", "bdev_split_create"); 424 425 spdk_json_write_named_object_begin(w, "params"); 426 spdk_json_write_named_string(w, "base_bdev", cfg->base_bdev); 427 spdk_json_write_named_uint32(w, "split_count", cfg->split_count); 428 spdk_json_write_named_uint64(w, "split_size_mb", cfg->split_size_mb); 429 spdk_json_write_object_end(w); 430 431 spdk_json_write_object_end(w); 432 } 433 434 return 0; 435 } 436 437 int 438 create_vbdev_split(const char *base_bdev_name, unsigned split_count, uint64_t split_size_mb) 439 { 440 int rc; 441 struct spdk_vbdev_split_config *cfg; 442 443 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size_mb, &cfg); 444 if (rc) { 445 return rc; 446 } 447 448 rc = vbdev_split_create(cfg); 449 if (rc == -ENODEV) { 450 /* It is ok if base bdev does not exist yet. */ 451 rc = 0; 452 } 453 454 return rc; 455 } 456 457 int 458 vbdev_split_destruct(const char *base_bdev_name) 459 { 460 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 461 462 if (!cfg) { 463 SPDK_ERRLOG("Split configuration for '%s' not found\n", base_bdev_name); 464 return -ENOENT; 465 } 466 467 vbdev_split_destruct_config(cfg); 468 return 0; 469 } 470 471 struct spdk_bdev_part_base * 472 vbdev_split_get_part_base(struct spdk_bdev *bdev) 473 { 474 struct spdk_vbdev_split_config *cfg; 475 476 cfg = vbdev_split_config_find_by_base_name(spdk_bdev_get_name(bdev)); 477 478 if (cfg == NULL) { 479 return NULL; 480 } 481 482 return cfg->split_base; 483 } 484 485 /* 486 * During init we'll be asked how much memory we'd like passed to us 487 * in bev_io structures as context. Here's where we specify how 488 * much context we want per IO. 489 */ 490 static int 491 vbdev_split_get_ctx_size(void) 492 { 493 return sizeof(struct vbdev_split_bdev_io); 494 } 495 496 SPDK_LOG_REGISTER_COMPONENT(vbdev_split) 497