1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * This is a simple example of a virtual block device that takes a single 36 * bdev and slices it into multiple smaller bdevs. 37 */ 38 39 #include "vbdev_split.h" 40 41 #include "spdk/rpc.h" 42 #include "spdk/conf.h" 43 #include "spdk/endian.h" 44 #include "spdk/string.h" 45 #include "spdk/thread.h" 46 #include "spdk/util.h" 47 48 #include "spdk/bdev_module.h" 49 #include "spdk_internal/log.h" 50 51 struct spdk_vbdev_split_config { 52 char *base_bdev; 53 unsigned split_count; 54 uint64_t split_size_mb; 55 56 SPDK_BDEV_PART_TAILQ splits; 57 struct spdk_bdev_part_base *split_base; 58 59 TAILQ_ENTRY(spdk_vbdev_split_config) tailq; 60 }; 61 62 static TAILQ_HEAD(, spdk_vbdev_split_config) g_split_config = TAILQ_HEAD_INITIALIZER( 63 g_split_config); 64 65 struct vbdev_split_channel { 66 struct spdk_bdev_part_channel part_ch; 67 }; 68 69 struct vbdev_split_bdev_io { 70 struct spdk_io_channel *ch; 71 struct spdk_bdev_io *bdev_io; 72 73 /* for bdev_io_wait */ 74 struct spdk_bdev_io_wait_entry bdev_io_wait; 75 }; 76 77 static void vbdev_split_del_config(struct spdk_vbdev_split_config *cfg); 78 79 static int vbdev_split_init(void); 80 static void vbdev_split_fini(void); 81 static void vbdev_split_examine(struct spdk_bdev *bdev); 82 static int vbdev_split_config_json(struct spdk_json_write_ctx *w); 83 static int vbdev_split_get_ctx_size(void); 84 85 static void 86 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io); 87 88 static struct spdk_bdev_module split_if = { 89 .name = "split", 90 .module_init = vbdev_split_init, 91 .module_fini = vbdev_split_fini, 92 .get_ctx_size = vbdev_split_get_ctx_size, 93 .examine_config = vbdev_split_examine, 94 .config_json = vbdev_split_config_json, 95 }; 96 97 SPDK_BDEV_MODULE_REGISTER(split, &split_if) 98 99 static void 100 vbdev_split_base_free(void *ctx) 101 { 102 struct spdk_vbdev_split_config *cfg = ctx; 103 104 vbdev_split_del_config(cfg); 105 } 106 107 static int 108 _vbdev_split_destruct(void *ctx) 109 { 110 struct spdk_bdev_part *part = ctx; 111 112 return spdk_bdev_part_free(part); 113 } 114 115 static void 116 vbdev_split_base_bdev_hotremove_cb(void *_part_base) 117 { 118 struct spdk_bdev_part_base *part_base = _part_base; 119 struct spdk_vbdev_split_config *cfg = spdk_bdev_part_base_get_ctx(part_base); 120 121 spdk_bdev_part_base_hotremove(part_base, &cfg->splits); 122 } 123 124 static void 125 vbdev_split_resubmit_io(void *arg) 126 { 127 struct vbdev_split_bdev_io *split_io = (struct vbdev_split_bdev_io *)arg; 128 129 _vbdev_split_submit_request(split_io->ch, split_io->bdev_io); 130 } 131 132 static void 133 vbdev_split_queue_io(struct vbdev_split_bdev_io *split_io) 134 { 135 struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(split_io->ch); 136 int rc; 137 138 split_io->bdev_io_wait.bdev = split_io->bdev_io->bdev; 139 split_io->bdev_io_wait.cb_fn = vbdev_split_resubmit_io; 140 split_io->bdev_io_wait.cb_arg = split_io; 141 142 rc = spdk_bdev_queue_io_wait(split_io->bdev_io->bdev, 143 ch->part_ch.base_ch, &split_io->bdev_io_wait); 144 if (rc != 0) { 145 SPDK_ERRLOG("Queue io failed in vbdev_split_queue_io, rc=%d\n", rc); 146 spdk_bdev_io_complete(split_io->bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 147 } 148 } 149 150 static void 151 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 152 { 153 struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(_ch); 154 struct vbdev_split_bdev_io *io_ctx = (struct vbdev_split_bdev_io *)bdev_io->driver_ctx; 155 int rc; 156 157 rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); 158 if (rc) { 159 if (rc == -ENOMEM) { 160 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "split: no memory, queue io.\n"); 161 io_ctx->ch = _ch; 162 io_ctx->bdev_io = bdev_io; 163 vbdev_split_queue_io(io_ctx); 164 } else { 165 SPDK_ERRLOG("split: error on io submission, rc=%d.\n", rc); 166 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 167 } 168 } 169 } 170 171 static void 172 vbdev_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success) 173 { 174 if (!success) { 175 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 176 return; 177 } 178 179 _vbdev_split_submit_request(ch, bdev_io); 180 } 181 182 static void 183 vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 184 { 185 switch (bdev_io->type) { 186 case SPDK_BDEV_IO_TYPE_READ: 187 spdk_bdev_io_get_buf(bdev_io, vbdev_split_get_buf_cb, 188 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 189 break; 190 default: 191 _vbdev_split_submit_request(_ch, bdev_io); 192 break; 193 } 194 } 195 196 static int 197 vbdev_split_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 198 { 199 struct spdk_bdev_part *part = ctx; 200 struct spdk_bdev *split_base_bdev = spdk_bdev_part_get_base_bdev(part); 201 uint64_t offset_blocks = spdk_bdev_part_get_offset_blocks(part); 202 203 spdk_json_write_named_object_begin(w, "split"); 204 205 spdk_json_write_named_string(w, "base_bdev", spdk_bdev_get_name(split_base_bdev)); 206 spdk_json_write_named_uint64(w, "offset_blocks", offset_blocks); 207 208 spdk_json_write_object_end(w); 209 210 return 0; 211 } 212 213 static void 214 vbdev_split_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 215 { 216 /* No config per bdev needed */ 217 } 218 219 static struct spdk_bdev_fn_table vbdev_split_fn_table = { 220 .destruct = _vbdev_split_destruct, 221 .submit_request = vbdev_split_submit_request, 222 .dump_info_json = vbdev_split_dump_info_json, 223 .write_config_json = vbdev_split_write_config_json 224 }; 225 226 static int 227 vbdev_split_create(struct spdk_vbdev_split_config *cfg) 228 { 229 uint64_t split_size_blocks, offset_blocks; 230 uint64_t split_count, max_split_count; 231 uint64_t mb = 1024 * 1024; 232 uint64_t i; 233 int rc; 234 char *name; 235 struct spdk_bdev *base_bdev; 236 struct bdev_part_tailq *split_base_tailq; 237 238 assert(cfg->split_count > 0); 239 240 base_bdev = spdk_bdev_get_by_name(cfg->base_bdev); 241 if (!base_bdev) { 242 return -ENODEV; 243 } 244 245 if (cfg->split_size_mb) { 246 if (((cfg->split_size_mb * mb) % base_bdev->blocklen) != 0) { 247 SPDK_ERRLOG("Split size %" PRIu64 " MB is not possible with block size " 248 "%" PRIu32 "\n", 249 cfg->split_size_mb, base_bdev->blocklen); 250 return -EINVAL; 251 } 252 split_size_blocks = (cfg->split_size_mb * mb) / base_bdev->blocklen; 253 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size %" PRIu64 " MB specified by user\n", 254 cfg->split_size_mb); 255 } else { 256 split_size_blocks = base_bdev->blockcnt / cfg->split_count; 257 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size not specified by user\n"); 258 } 259 260 max_split_count = base_bdev->blockcnt / split_size_blocks; 261 split_count = cfg->split_count; 262 if (split_count > max_split_count) { 263 SPDK_WARNLOG("Split count %" PRIu64 " is greater than maximum possible split count " 264 "%" PRIu64 " - clamping\n", split_count, max_split_count); 265 split_count = max_split_count; 266 } 267 268 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "base_bdev: %s split_count: %" PRIu64 269 " split_size_blocks: %" PRIu64 "\n", 270 spdk_bdev_get_name(base_bdev), split_count, split_size_blocks); 271 272 TAILQ_INIT(&cfg->splits); 273 cfg->split_base = spdk_bdev_part_base_construct(base_bdev, 274 vbdev_split_base_bdev_hotremove_cb, 275 &split_if, &vbdev_split_fn_table, 276 &cfg->splits, vbdev_split_base_free, cfg, 277 sizeof(struct vbdev_split_channel), NULL, NULL); 278 if (!cfg->split_base) { 279 SPDK_ERRLOG("Cannot construct bdev part base\n"); 280 return -ENOMEM; 281 } 282 283 offset_blocks = 0; 284 for (i = 0; i < split_count; i++) { 285 struct spdk_bdev_part *d; 286 287 d = calloc(1, sizeof(*d)); 288 if (d == NULL) { 289 SPDK_ERRLOG("could not allocate bdev part\n"); 290 rc = -ENOMEM; 291 goto err; 292 } 293 294 name = spdk_sprintf_alloc("%sp%" PRIu64, cfg->base_bdev, i); 295 if (!name) { 296 SPDK_ERRLOG("could not allocate name\n"); 297 free(d); 298 rc = -ENOMEM; 299 goto err; 300 } 301 302 rc = spdk_bdev_part_construct(d, cfg->split_base, name, offset_blocks, split_size_blocks, 303 "Split Disk"); 304 free(name); 305 if (rc) { 306 SPDK_ERRLOG("could not construct bdev part\n"); 307 /* spdk_bdev_part_construct will free name if it fails */ 308 free(d); 309 rc = -ENOMEM; 310 goto err; 311 } 312 313 offset_blocks += split_size_blocks; 314 } 315 316 return 0; 317 err: 318 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 319 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 320 return rc; 321 } 322 323 static void 324 vbdev_split_del_config(struct spdk_vbdev_split_config *cfg) 325 { 326 TAILQ_REMOVE(&g_split_config, cfg, tailq); 327 free(cfg->base_bdev); 328 free(cfg); 329 } 330 331 static void 332 vbdev_split_destruct_config(struct spdk_vbdev_split_config *cfg) 333 { 334 struct bdev_part_tailq *split_base_tailq; 335 336 if (cfg->split_base != NULL) { 337 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 338 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 339 } else { 340 vbdev_split_del_config(cfg); 341 } 342 } 343 344 static void 345 vbdev_split_clear_config(void) 346 { 347 struct spdk_vbdev_split_config *cfg, *tmp_cfg; 348 349 TAILQ_FOREACH_SAFE(cfg, &g_split_config, tailq, tmp_cfg) { 350 vbdev_split_destruct_config(cfg); 351 } 352 } 353 354 static struct spdk_vbdev_split_config * 355 vbdev_split_config_find_by_base_name(const char *base_bdev_name) 356 { 357 struct spdk_vbdev_split_config *cfg; 358 359 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 360 if (strcmp(cfg->base_bdev, base_bdev_name) == 0) { 361 return cfg; 362 } 363 } 364 365 return NULL; 366 } 367 368 static int 369 vbdev_split_add_config(const char *base_bdev_name, unsigned split_count, uint64_t split_size, 370 struct spdk_vbdev_split_config **config) 371 { 372 struct spdk_vbdev_split_config *cfg; 373 assert(base_bdev_name); 374 375 if (base_bdev_name == NULL) { 376 SPDK_ERRLOG("Split bdev config: no base bdev provided."); 377 return -EINVAL; 378 } 379 380 if (split_count == 0) { 381 SPDK_ERRLOG("Split bdev config: split_count can't be 0."); 382 return -EINVAL; 383 } 384 385 /* Check if we already have 'base_bdev_name' registered in config */ 386 cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 387 if (cfg) { 388 SPDK_ERRLOG("Split bdev config for base bdev '%s' already exist.", base_bdev_name); 389 return -EEXIST; 390 } 391 392 cfg = calloc(1, sizeof(*cfg)); 393 if (!cfg) { 394 SPDK_ERRLOG("calloc(): Out of memory"); 395 return -ENOMEM; 396 } 397 398 cfg->base_bdev = strdup(base_bdev_name); 399 if (!cfg->base_bdev) { 400 SPDK_ERRLOG("strdup(): Out of memory"); 401 free(cfg); 402 return -ENOMEM; 403 } 404 405 cfg->split_count = split_count; 406 cfg->split_size_mb = split_size; 407 TAILQ_INSERT_TAIL(&g_split_config, cfg, tailq); 408 if (config) { 409 *config = cfg; 410 } 411 412 return 0; 413 } 414 415 static int 416 vbdev_split_init(void) 417 { 418 419 struct spdk_conf_section *sp; 420 const char *base_bdev_name; 421 const char *split_count_str; 422 const char *split_size_str; 423 int rc, i, split_count, split_size; 424 425 sp = spdk_conf_find_section(NULL, "Split"); 426 if (sp == NULL) { 427 return 0; 428 } 429 430 for (i = 0; ; i++) { 431 if (!spdk_conf_section_get_nval(sp, "Split", i)) { 432 break; 433 } 434 435 base_bdev_name = spdk_conf_section_get_nmval(sp, "Split", i, 0); 436 if (!base_bdev_name) { 437 SPDK_ERRLOG("Split configuration missing bdev name\n"); 438 rc = -EINVAL; 439 goto err; 440 } 441 442 split_count_str = spdk_conf_section_get_nmval(sp, "Split", i, 1); 443 if (!split_count_str) { 444 SPDK_ERRLOG("Split configuration missing split count\n"); 445 rc = -EINVAL; 446 goto err; 447 } 448 449 split_count = spdk_strtol(split_count_str, 10); 450 if (split_count < 1) { 451 SPDK_ERRLOG("Invalid Split count %d\n", split_count); 452 rc = -EINVAL; 453 goto err; 454 } 455 456 /* Optional split size in MB */ 457 split_size = 0; 458 split_size_str = spdk_conf_section_get_nmval(sp, "Split", i, 2); 459 if (split_size_str) { 460 split_size = spdk_strtol(split_size_str, 10); 461 if (split_size <= 0) { 462 SPDK_ERRLOG("Invalid Split size %d\n", split_size); 463 rc = -EINVAL; 464 goto err; 465 } 466 } 467 468 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size, NULL); 469 if (rc != 0) { 470 goto err; 471 } 472 } 473 474 return 0; 475 err: 476 vbdev_split_clear_config(); 477 return rc; 478 } 479 480 static void 481 vbdev_split_fini(void) 482 { 483 vbdev_split_clear_config(); 484 } 485 486 static void 487 vbdev_split_examine(struct spdk_bdev *bdev) 488 { 489 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(bdev->name); 490 491 if (cfg != NULL) { 492 assert(cfg->split_base == NULL); 493 494 if (vbdev_split_create(cfg)) { 495 SPDK_ERRLOG("could not split bdev %s\n", bdev->name); 496 } 497 } 498 spdk_bdev_module_examine_done(&split_if); 499 } 500 501 static int 502 vbdev_split_config_json(struct spdk_json_write_ctx *w) 503 { 504 struct spdk_vbdev_split_config *cfg; 505 506 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 507 spdk_json_write_object_begin(w); 508 509 spdk_json_write_named_string(w, "method", "bdev_split_create"); 510 511 spdk_json_write_named_object_begin(w, "params"); 512 spdk_json_write_named_string(w, "base_bdev", cfg->base_bdev); 513 spdk_json_write_named_uint32(w, "split_count", cfg->split_count); 514 spdk_json_write_named_uint64(w, "split_size_mb", cfg->split_size_mb); 515 spdk_json_write_object_end(w); 516 517 spdk_json_write_object_end(w); 518 } 519 520 return 0; 521 } 522 523 int 524 create_vbdev_split(const char *base_bdev_name, unsigned split_count, uint64_t split_size_mb) 525 { 526 int rc; 527 struct spdk_vbdev_split_config *cfg; 528 529 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size_mb, &cfg); 530 if (rc) { 531 return rc; 532 } 533 534 rc = vbdev_split_create(cfg); 535 if (rc == -ENODEV) { 536 /* It is ok if base bdev does not exist yet. */ 537 rc = 0; 538 } 539 540 return rc; 541 } 542 543 int 544 vbdev_split_destruct(const char *base_bdev_name) 545 { 546 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 547 548 if (!cfg) { 549 SPDK_ERRLOG("Split configuration for '%s' not found\n", base_bdev_name); 550 return -ENOENT; 551 } 552 553 vbdev_split_destruct_config(cfg); 554 return 0; 555 } 556 557 struct spdk_bdev_part_base * 558 vbdev_split_get_part_base(struct spdk_bdev *bdev) 559 { 560 struct spdk_vbdev_split_config *cfg; 561 562 cfg = vbdev_split_config_find_by_base_name(spdk_bdev_get_name(bdev)); 563 564 if (cfg == NULL) { 565 return NULL; 566 } 567 568 return cfg->split_base; 569 } 570 571 /* 572 * During init we'll be asked how much memory we'd like passed to us 573 * in bev_io structures as context. Here's where we specify how 574 * much context we want per IO. 575 */ 576 static int 577 vbdev_split_get_ctx_size(void) 578 { 579 return sizeof(struct vbdev_split_bdev_io); 580 } 581 582 SPDK_LOG_REGISTER_COMPONENT("vbdev_split", SPDK_LOG_VBDEV_SPLIT) 583