1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * This is a simple example of a virtual block device that takes a single 36 * bdev and slices it into multiple smaller bdevs. 37 */ 38 39 #include "vbdev_split.h" 40 41 #include "spdk/rpc.h" 42 #include "spdk/conf.h" 43 #include "spdk/endian.h" 44 #include "spdk/string.h" 45 #include "spdk/thread.h" 46 #include "spdk/util.h" 47 48 #include "spdk/bdev_module.h" 49 #include "spdk_internal/log.h" 50 51 struct spdk_vbdev_split_config { 52 char *base_bdev; 53 unsigned split_count; 54 uint64_t split_size_mb; 55 56 SPDK_BDEV_PART_TAILQ splits; 57 struct spdk_bdev_part_base *split_base; 58 59 TAILQ_ENTRY(spdk_vbdev_split_config) tailq; 60 }; 61 62 static TAILQ_HEAD(, spdk_vbdev_split_config) g_split_config = TAILQ_HEAD_INITIALIZER( 63 g_split_config); 64 65 struct vbdev_split_channel { 66 struct spdk_bdev_part_channel part_ch; 67 }; 68 69 struct vbdev_split_bdev_io { 70 struct spdk_io_channel *ch; 71 struct spdk_bdev_io *bdev_io; 72 73 /* for bdev_io_wait */ 74 struct spdk_bdev_io_wait_entry bdev_io_wait; 75 }; 76 77 static void vbdev_split_del_config(struct spdk_vbdev_split_config *cfg); 78 79 static int vbdev_split_init(void); 80 static void vbdev_split_fini(void); 81 static void vbdev_split_examine(struct spdk_bdev *bdev); 82 static int vbdev_split_config_json(struct spdk_json_write_ctx *w); 83 static int vbdev_split_get_ctx_size(void); 84 85 static void 86 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io); 87 88 static struct spdk_bdev_module split_if = { 89 .name = "split", 90 .module_init = vbdev_split_init, 91 .module_fini = vbdev_split_fini, 92 .get_ctx_size = vbdev_split_get_ctx_size, 93 .examine_config = vbdev_split_examine, 94 .config_json = vbdev_split_config_json, 95 }; 96 97 SPDK_BDEV_MODULE_REGISTER(split, &split_if) 98 99 static void 100 vbdev_split_base_free(void *ctx) 101 { 102 struct spdk_vbdev_split_config *cfg = ctx; 103 104 vbdev_split_del_config(cfg); 105 } 106 107 static int 108 vbdev_split_destruct(void *ctx) 109 { 110 struct spdk_bdev_part *part = ctx; 111 112 return spdk_bdev_part_free(part); 113 } 114 115 static void 116 vbdev_split_base_bdev_hotremove_cb(void *_part_base) 117 { 118 struct spdk_bdev_part_base *part_base = _part_base; 119 struct spdk_vbdev_split_config *cfg = spdk_bdev_part_base_get_ctx(part_base); 120 121 spdk_bdev_part_base_hotremove(part_base, &cfg->splits); 122 } 123 124 static void 125 vbdev_split_resubmit_io(void *arg) 126 { 127 struct vbdev_split_bdev_io *split_io = (struct vbdev_split_bdev_io *)arg; 128 129 _vbdev_split_submit_request(split_io->ch, split_io->bdev_io); 130 } 131 132 static void 133 vbdev_split_queue_io(struct vbdev_split_bdev_io *split_io) 134 { 135 int rc; 136 137 split_io->bdev_io_wait.bdev = split_io->bdev_io->bdev; 138 split_io->bdev_io_wait.cb_fn = vbdev_split_resubmit_io; 139 split_io->bdev_io_wait.cb_arg = split_io; 140 141 rc = spdk_bdev_queue_io_wait(split_io->bdev_io->bdev, 142 split_io->ch, &split_io->bdev_io_wait); 143 if (rc != 0) { 144 SPDK_ERRLOG("Queue io failed in vbdev_split_queue_io, rc=%d\n", rc); 145 spdk_bdev_io_complete(split_io->bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 146 } 147 } 148 149 static void 150 _vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 151 { 152 struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(_ch); 153 struct vbdev_split_bdev_io *io_ctx = (struct vbdev_split_bdev_io *)bdev_io->driver_ctx; 154 int rc; 155 156 rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); 157 if (rc) { 158 if (rc == -ENOMEM) { 159 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "split: no memory, queue io.\n"); 160 io_ctx->ch = _ch; 161 io_ctx->bdev_io = bdev_io; 162 vbdev_split_queue_io(io_ctx); 163 } else { 164 SPDK_ERRLOG("split: error on io submission, rc=%d.\n", rc); 165 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 166 } 167 } 168 } 169 170 static void 171 vbdev_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success) 172 { 173 if (!success) { 174 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 175 return; 176 } 177 178 _vbdev_split_submit_request(ch, bdev_io); 179 } 180 181 static void 182 vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) 183 { 184 switch (bdev_io->type) { 185 case SPDK_BDEV_IO_TYPE_READ: 186 spdk_bdev_io_get_buf(bdev_io, vbdev_split_get_buf_cb, 187 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 188 break; 189 default: 190 _vbdev_split_submit_request(_ch, bdev_io); 191 break; 192 } 193 } 194 195 static int 196 vbdev_split_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 197 { 198 struct spdk_bdev_part *part = ctx; 199 struct spdk_bdev *split_base_bdev = spdk_bdev_part_get_base_bdev(part); 200 uint64_t offset_blocks = spdk_bdev_part_get_offset_blocks(part); 201 202 spdk_json_write_named_object_begin(w, "split"); 203 204 spdk_json_write_named_string(w, "base_bdev", spdk_bdev_get_name(split_base_bdev)); 205 spdk_json_write_named_uint64(w, "offset_blocks", offset_blocks); 206 207 spdk_json_write_object_end(w); 208 209 return 0; 210 } 211 212 static void 213 vbdev_split_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 214 { 215 /* No config per bdev needed */ 216 } 217 218 static struct spdk_bdev_fn_table vbdev_split_fn_table = { 219 .destruct = vbdev_split_destruct, 220 .submit_request = vbdev_split_submit_request, 221 .dump_info_json = vbdev_split_dump_info_json, 222 .write_config_json = vbdev_split_write_config_json 223 }; 224 225 static int 226 vbdev_split_create(struct spdk_vbdev_split_config *cfg) 227 { 228 uint64_t split_size_blocks, offset_blocks; 229 uint64_t split_count, max_split_count; 230 uint64_t mb = 1024 * 1024; 231 uint64_t i; 232 int rc; 233 char *name; 234 struct spdk_bdev *base_bdev; 235 struct bdev_part_tailq *split_base_tailq; 236 237 assert(cfg->split_count > 0); 238 239 base_bdev = spdk_bdev_get_by_name(cfg->base_bdev); 240 if (!base_bdev) { 241 return -ENODEV; 242 } 243 244 if (cfg->split_size_mb) { 245 if (((cfg->split_size_mb * mb) % base_bdev->blocklen) != 0) { 246 SPDK_ERRLOG("Split size %" PRIu64 " MB is not possible with block size " 247 "%" PRIu32 "\n", 248 cfg->split_size_mb, base_bdev->blocklen); 249 return -EINVAL; 250 } 251 split_size_blocks = (cfg->split_size_mb * mb) / base_bdev->blocklen; 252 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size %" PRIu64 " MB specified by user\n", 253 cfg->split_size_mb); 254 } else { 255 split_size_blocks = base_bdev->blockcnt / cfg->split_count; 256 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size not specified by user\n"); 257 } 258 259 max_split_count = base_bdev->blockcnt / split_size_blocks; 260 split_count = cfg->split_count; 261 if (split_count > max_split_count) { 262 SPDK_WARNLOG("Split count %" PRIu64 " is greater than maximum possible split count " 263 "%" PRIu64 " - clamping\n", split_count, max_split_count); 264 split_count = max_split_count; 265 } 266 267 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "base_bdev: %s split_count: %" PRIu64 268 " split_size_blocks: %" PRIu64 "\n", 269 spdk_bdev_get_name(base_bdev), split_count, split_size_blocks); 270 271 TAILQ_INIT(&cfg->splits); 272 cfg->split_base = spdk_bdev_part_base_construct(base_bdev, 273 vbdev_split_base_bdev_hotremove_cb, 274 &split_if, &vbdev_split_fn_table, 275 &cfg->splits, vbdev_split_base_free, cfg, 276 sizeof(struct vbdev_split_channel), NULL, NULL); 277 if (!cfg->split_base) { 278 SPDK_ERRLOG("Cannot construct bdev part base\n"); 279 return -ENOMEM; 280 } 281 282 offset_blocks = 0; 283 for (i = 0; i < split_count; i++) { 284 struct spdk_bdev_part *d; 285 286 d = calloc(1, sizeof(*d)); 287 if (d == NULL) { 288 SPDK_ERRLOG("could not allocate bdev part\n"); 289 rc = -ENOMEM; 290 goto err; 291 } 292 293 name = spdk_sprintf_alloc("%sp%" PRIu64, cfg->base_bdev, i); 294 if (!name) { 295 SPDK_ERRLOG("could not allocate name\n"); 296 free(d); 297 rc = -ENOMEM; 298 goto err; 299 } 300 301 rc = spdk_bdev_part_construct(d, cfg->split_base, name, offset_blocks, split_size_blocks, 302 "Split Disk"); 303 free(name); 304 if (rc) { 305 SPDK_ERRLOG("could not construct bdev part\n"); 306 /* spdk_bdev_part_construct will free name if it fails */ 307 free(d); 308 rc = -ENOMEM; 309 goto err; 310 } 311 312 offset_blocks += split_size_blocks; 313 } 314 315 return 0; 316 err: 317 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 318 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 319 return rc; 320 } 321 322 static void 323 vbdev_split_del_config(struct spdk_vbdev_split_config *cfg) 324 { 325 TAILQ_REMOVE(&g_split_config, cfg, tailq); 326 free(cfg->base_bdev); 327 free(cfg); 328 } 329 330 static void 331 vbdev_split_destruct_config(struct spdk_vbdev_split_config *cfg) 332 { 333 struct bdev_part_tailq *split_base_tailq; 334 335 if (cfg->split_base != NULL) { 336 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base); 337 spdk_bdev_part_base_hotremove(cfg->split_base, split_base_tailq); 338 } else { 339 vbdev_split_del_config(cfg); 340 } 341 } 342 343 static void 344 vbdev_split_clear_config(void) 345 { 346 struct spdk_vbdev_split_config *cfg, *tmp_cfg; 347 348 TAILQ_FOREACH_SAFE(cfg, &g_split_config, tailq, tmp_cfg) { 349 vbdev_split_destruct_config(cfg); 350 } 351 } 352 353 static struct spdk_vbdev_split_config * 354 vbdev_split_config_find_by_base_name(const char *base_bdev_name) 355 { 356 struct spdk_vbdev_split_config *cfg; 357 358 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 359 if (strcmp(cfg->base_bdev, base_bdev_name) == 0) { 360 return cfg; 361 } 362 } 363 364 return NULL; 365 } 366 367 static int 368 vbdev_split_add_config(const char *base_bdev_name, unsigned split_count, uint64_t split_size, 369 struct spdk_vbdev_split_config **config) 370 { 371 struct spdk_vbdev_split_config *cfg; 372 assert(base_bdev_name); 373 374 if (base_bdev_name == NULL) { 375 SPDK_ERRLOG("Split bdev config: no base bdev provided."); 376 return -EINVAL; 377 } 378 379 if (split_count == 0) { 380 SPDK_ERRLOG("Split bdev config: split_count can't be 0."); 381 return -EINVAL; 382 } 383 384 /* Check if we already have 'base_bdev_name' registered in config */ 385 cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 386 if (cfg) { 387 SPDK_ERRLOG("Split bdev config for base bdev '%s' already exist.", base_bdev_name); 388 return -EEXIST; 389 } 390 391 cfg = calloc(1, sizeof(*cfg)); 392 if (!cfg) { 393 SPDK_ERRLOG("calloc(): Out of memory"); 394 return -ENOMEM; 395 } 396 397 cfg->base_bdev = strdup(base_bdev_name); 398 if (!cfg->base_bdev) { 399 SPDK_ERRLOG("strdup(): Out of memory"); 400 free(cfg); 401 return -ENOMEM; 402 } 403 404 cfg->split_count = split_count; 405 cfg->split_size_mb = split_size; 406 TAILQ_INSERT_TAIL(&g_split_config, cfg, tailq); 407 if (config) { 408 *config = cfg; 409 } 410 411 return 0; 412 } 413 414 static int 415 vbdev_split_init(void) 416 { 417 418 struct spdk_conf_section *sp; 419 const char *base_bdev_name; 420 const char *split_count_str; 421 const char *split_size_str; 422 int rc, i, split_count, split_size; 423 424 sp = spdk_conf_find_section(NULL, "Split"); 425 if (sp == NULL) { 426 return 0; 427 } 428 429 for (i = 0; ; i++) { 430 if (!spdk_conf_section_get_nval(sp, "Split", i)) { 431 break; 432 } 433 434 base_bdev_name = spdk_conf_section_get_nmval(sp, "Split", i, 0); 435 if (!base_bdev_name) { 436 SPDK_ERRLOG("Split configuration missing bdev name\n"); 437 rc = -EINVAL; 438 goto err; 439 } 440 441 split_count_str = spdk_conf_section_get_nmval(sp, "Split", i, 1); 442 if (!split_count_str) { 443 SPDK_ERRLOG("Split configuration missing split count\n"); 444 rc = -EINVAL; 445 goto err; 446 } 447 448 split_count = spdk_strtol(split_count_str, 10); 449 if (split_count < 1) { 450 SPDK_ERRLOG("Invalid Split count %d\n", split_count); 451 rc = -EINVAL; 452 goto err; 453 } 454 455 /* Optional split size in MB */ 456 split_size = 0; 457 split_size_str = spdk_conf_section_get_nmval(sp, "Split", i, 2); 458 if (split_size_str) { 459 split_size = spdk_strtol(split_size_str, 10); 460 if (split_size <= 0) { 461 SPDK_ERRLOG("Invalid Split size %d\n", split_size); 462 rc = -EINVAL; 463 goto err; 464 } 465 } 466 467 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size, NULL); 468 if (rc != 0) { 469 goto err; 470 } 471 } 472 473 return 0; 474 err: 475 vbdev_split_clear_config(); 476 return rc; 477 } 478 479 static void 480 vbdev_split_fini(void) 481 { 482 vbdev_split_clear_config(); 483 } 484 485 static void 486 vbdev_split_examine(struct spdk_bdev *bdev) 487 { 488 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(bdev->name); 489 490 if (cfg != NULL) { 491 assert(cfg->split_base == NULL); 492 493 if (vbdev_split_create(cfg)) { 494 SPDK_ERRLOG("could not split bdev %s\n", bdev->name); 495 } 496 } 497 spdk_bdev_module_examine_done(&split_if); 498 } 499 500 static int 501 vbdev_split_config_json(struct spdk_json_write_ctx *w) 502 { 503 struct spdk_vbdev_split_config *cfg; 504 505 TAILQ_FOREACH(cfg, &g_split_config, tailq) { 506 spdk_json_write_object_begin(w); 507 508 spdk_json_write_named_string(w, "method", "bdev_split_create"); 509 510 spdk_json_write_named_object_begin(w, "params"); 511 spdk_json_write_named_string(w, "base_bdev", cfg->base_bdev); 512 spdk_json_write_named_uint32(w, "split_count", cfg->split_count); 513 spdk_json_write_named_uint64(w, "split_size_mb", cfg->split_size_mb); 514 spdk_json_write_object_end(w); 515 516 spdk_json_write_object_end(w); 517 } 518 519 return 0; 520 } 521 522 int 523 create_vbdev_split(const char *base_bdev_name, unsigned split_count, uint64_t split_size_mb) 524 { 525 int rc; 526 struct spdk_vbdev_split_config *cfg; 527 528 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size_mb, &cfg); 529 if (rc) { 530 return rc; 531 } 532 533 rc = vbdev_split_create(cfg); 534 if (rc == -ENODEV) { 535 /* It is ok if base bdev does not exist yet. */ 536 rc = 0; 537 } 538 539 return rc; 540 } 541 542 int 543 spdk_vbdev_split_destruct(const char *base_bdev_name) 544 { 545 struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(base_bdev_name); 546 547 if (!cfg) { 548 SPDK_ERRLOG("Split configuration for '%s' not found\n", base_bdev_name); 549 return -ENOENT; 550 } 551 552 vbdev_split_destruct_config(cfg); 553 return 0; 554 } 555 556 struct spdk_bdev_part_base * 557 spdk_vbdev_split_get_part_base(struct spdk_bdev *bdev) 558 { 559 struct spdk_vbdev_split_config *cfg; 560 561 cfg = vbdev_split_config_find_by_base_name(spdk_bdev_get_name(bdev)); 562 563 if (cfg == NULL) { 564 return NULL; 565 } 566 567 return cfg->split_base; 568 } 569 570 /* 571 * During init we'll be asked how much memory we'd like passed to us 572 * in bev_io structures as context. Here's where we specify how 573 * much context we want per IO. 574 */ 575 static int 576 vbdev_split_get_ctx_size(void) 577 { 578 return sizeof(struct vbdev_split_bdev_io); 579 } 580 581 SPDK_LOG_REGISTER_COMPONENT("vbdev_split", SPDK_LOG_VBDEV_SPLIT) 582