1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "bdev_malloc.h" 37 #include "spdk/bdev.h" 38 #include "spdk/endian.h" 39 #include "spdk/env.h" 40 #include "spdk/accel_engine.h" 41 #include "spdk/json.h" 42 #include "spdk/thread.h" 43 #include "spdk/queue.h" 44 #include "spdk/string.h" 45 46 #include "spdk/bdev_module.h" 47 #include "spdk/log.h" 48 49 struct malloc_disk { 50 struct spdk_bdev disk; 51 void *malloc_buf; 52 TAILQ_ENTRY(malloc_disk) link; 53 }; 54 55 struct malloc_task { 56 int num_outstanding; 57 enum spdk_bdev_io_status status; 58 }; 59 60 static void 61 malloc_done(void *ref, int status) 62 { 63 struct malloc_task *task = (struct malloc_task *)ref; 64 65 if (status != 0) { 66 if (status == -ENOMEM) { 67 task->status = SPDK_BDEV_IO_STATUS_NOMEM; 68 } else { 69 task->status = SPDK_BDEV_IO_STATUS_FAILED; 70 } 71 } 72 73 if (--task->num_outstanding == 0) { 74 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), task->status); 75 } 76 } 77 78 static TAILQ_HEAD(, malloc_disk) g_malloc_disks = TAILQ_HEAD_INITIALIZER(g_malloc_disks); 79 80 int malloc_disk_count = 0; 81 82 static int bdev_malloc_initialize(void); 83 84 static int 85 bdev_malloc_get_ctx_size(void) 86 { 87 return sizeof(struct malloc_task); 88 } 89 90 static struct spdk_bdev_module malloc_if = { 91 .name = "malloc", 92 .module_init = bdev_malloc_initialize, 93 .get_ctx_size = bdev_malloc_get_ctx_size, 94 95 }; 96 97 SPDK_BDEV_MODULE_REGISTER(malloc, &malloc_if) 98 99 static void 100 malloc_disk_free(struct malloc_disk *malloc_disk) 101 { 102 if (!malloc_disk) { 103 return; 104 } 105 106 free(malloc_disk->disk.name); 107 spdk_free(malloc_disk->malloc_buf); 108 free(malloc_disk); 109 } 110 111 static int 112 bdev_malloc_destruct(void *ctx) 113 { 114 struct malloc_disk *malloc_disk = ctx; 115 116 TAILQ_REMOVE(&g_malloc_disks, malloc_disk, link); 117 malloc_disk_free(malloc_disk); 118 return 0; 119 } 120 121 static int 122 bdev_malloc_check_iov_len(struct iovec *iovs, int iovcnt, size_t nbytes) 123 { 124 int i; 125 126 for (i = 0; i < iovcnt; i++) { 127 if (nbytes < iovs[i].iov_len) { 128 return 0; 129 } 130 131 nbytes -= iovs[i].iov_len; 132 } 133 134 return nbytes != 0; 135 } 136 137 static void 138 bdev_malloc_readv(struct malloc_disk *mdisk, struct spdk_io_channel *ch, 139 struct malloc_task *task, 140 struct iovec *iov, int iovcnt, size_t len, uint64_t offset) 141 { 142 int64_t res = 0; 143 void *src = mdisk->malloc_buf + offset; 144 int i; 145 146 if (bdev_malloc_check_iov_len(iov, iovcnt, len)) { 147 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), 148 SPDK_BDEV_IO_STATUS_FAILED); 149 return; 150 } 151 152 SPDK_DEBUGLOG(bdev_malloc, "read %zu bytes from offset %#" PRIx64 "\n", 153 len, offset); 154 155 task->status = SPDK_BDEV_IO_STATUS_SUCCESS; 156 task->num_outstanding = iovcnt; 157 158 for (i = 0; i < iovcnt; i++) { 159 res = spdk_accel_submit_copy(ch, iov[i].iov_base, 160 src, iov[i].iov_len, malloc_done, task); 161 162 if (res != 0) { 163 malloc_done(task, res); 164 } 165 166 src += iov[i].iov_len; 167 len -= iov[i].iov_len; 168 } 169 } 170 171 static void 172 bdev_malloc_writev(struct malloc_disk *mdisk, struct spdk_io_channel *ch, 173 struct malloc_task *task, 174 struct iovec *iov, int iovcnt, size_t len, uint64_t offset) 175 { 176 int64_t res = 0; 177 void *dst = mdisk->malloc_buf + offset; 178 int i; 179 180 if (bdev_malloc_check_iov_len(iov, iovcnt, len)) { 181 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), 182 SPDK_BDEV_IO_STATUS_FAILED); 183 return; 184 } 185 186 SPDK_DEBUGLOG(bdev_malloc, "wrote %zu bytes to offset %#" PRIx64 "\n", 187 len, offset); 188 189 task->status = SPDK_BDEV_IO_STATUS_SUCCESS; 190 task->num_outstanding = iovcnt; 191 192 for (i = 0; i < iovcnt; i++) { 193 res = spdk_accel_submit_copy(ch, dst, iov[i].iov_base, 194 iov[i].iov_len, malloc_done, task); 195 196 if (res != 0) { 197 malloc_done(task, res); 198 } 199 200 dst += iov[i].iov_len; 201 } 202 } 203 204 static int 205 bdev_malloc_unmap(struct malloc_disk *mdisk, 206 struct spdk_io_channel *ch, 207 struct malloc_task *task, 208 uint64_t offset, 209 uint64_t byte_count) 210 { 211 task->status = SPDK_BDEV_IO_STATUS_SUCCESS; 212 task->num_outstanding = 1; 213 214 return spdk_accel_submit_fill(ch, mdisk->malloc_buf + offset, 0, 215 byte_count, malloc_done, task); 216 } 217 218 static int64_t 219 bdev_malloc_flush(struct malloc_disk *mdisk, struct malloc_task *task, 220 uint64_t offset, uint64_t nbytes) 221 { 222 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), SPDK_BDEV_IO_STATUS_SUCCESS); 223 224 return 0; 225 } 226 227 static int 228 bdev_malloc_reset(struct malloc_disk *mdisk, struct malloc_task *task) 229 { 230 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), SPDK_BDEV_IO_STATUS_SUCCESS); 231 232 return 0; 233 } 234 235 static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 236 { 237 uint32_t block_size = bdev_io->bdev->blocklen; 238 239 switch (bdev_io->type) { 240 case SPDK_BDEV_IO_TYPE_READ: 241 if (bdev_io->u.bdev.iovs[0].iov_base == NULL) { 242 assert(bdev_io->u.bdev.iovcnt == 1); 243 bdev_io->u.bdev.iovs[0].iov_base = 244 ((struct malloc_disk *)bdev_io->bdev->ctxt)->malloc_buf + 245 bdev_io->u.bdev.offset_blocks * block_size; 246 bdev_io->u.bdev.iovs[0].iov_len = bdev_io->u.bdev.num_blocks * block_size; 247 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 248 return 0; 249 } 250 251 bdev_malloc_readv((struct malloc_disk *)bdev_io->bdev->ctxt, 252 ch, 253 (struct malloc_task *)bdev_io->driver_ctx, 254 bdev_io->u.bdev.iovs, 255 bdev_io->u.bdev.iovcnt, 256 bdev_io->u.bdev.num_blocks * block_size, 257 bdev_io->u.bdev.offset_blocks * block_size); 258 return 0; 259 260 case SPDK_BDEV_IO_TYPE_WRITE: 261 bdev_malloc_writev((struct malloc_disk *)bdev_io->bdev->ctxt, 262 ch, 263 (struct malloc_task *)bdev_io->driver_ctx, 264 bdev_io->u.bdev.iovs, 265 bdev_io->u.bdev.iovcnt, 266 bdev_io->u.bdev.num_blocks * block_size, 267 bdev_io->u.bdev.offset_blocks * block_size); 268 return 0; 269 270 case SPDK_BDEV_IO_TYPE_RESET: 271 return bdev_malloc_reset((struct malloc_disk *)bdev_io->bdev->ctxt, 272 (struct malloc_task *)bdev_io->driver_ctx); 273 274 case SPDK_BDEV_IO_TYPE_FLUSH: 275 return bdev_malloc_flush((struct malloc_disk *)bdev_io->bdev->ctxt, 276 (struct malloc_task *)bdev_io->driver_ctx, 277 bdev_io->u.bdev.offset_blocks * block_size, 278 bdev_io->u.bdev.num_blocks * block_size); 279 280 case SPDK_BDEV_IO_TYPE_UNMAP: 281 return bdev_malloc_unmap((struct malloc_disk *)bdev_io->bdev->ctxt, 282 ch, 283 (struct malloc_task *)bdev_io->driver_ctx, 284 bdev_io->u.bdev.offset_blocks * block_size, 285 bdev_io->u.bdev.num_blocks * block_size); 286 287 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 288 /* bdev_malloc_unmap is implemented with a call to mem_cpy_fill which zeroes out all of the requested bytes. */ 289 return bdev_malloc_unmap((struct malloc_disk *)bdev_io->bdev->ctxt, 290 ch, 291 (struct malloc_task *)bdev_io->driver_ctx, 292 bdev_io->u.bdev.offset_blocks * block_size, 293 bdev_io->u.bdev.num_blocks * block_size); 294 295 case SPDK_BDEV_IO_TYPE_ZCOPY: 296 if (bdev_io->u.bdev.zcopy.start) { 297 void *buf; 298 size_t len; 299 300 buf = ((struct malloc_disk *)bdev_io->bdev->ctxt)->malloc_buf + 301 bdev_io->u.bdev.offset_blocks * block_size; 302 len = bdev_io->u.bdev.num_blocks * block_size; 303 spdk_bdev_io_set_buf(bdev_io, buf, len); 304 305 } 306 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 307 return 0; 308 case SPDK_BDEV_IO_TYPE_ABORT: 309 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 310 return 0; 311 default: 312 return -1; 313 } 314 return 0; 315 } 316 317 static void bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 318 { 319 if (_bdev_malloc_submit_request(ch, bdev_io) != 0) { 320 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 321 } 322 } 323 324 static bool 325 bdev_malloc_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 326 { 327 switch (io_type) { 328 case SPDK_BDEV_IO_TYPE_READ: 329 case SPDK_BDEV_IO_TYPE_WRITE: 330 case SPDK_BDEV_IO_TYPE_FLUSH: 331 case SPDK_BDEV_IO_TYPE_RESET: 332 case SPDK_BDEV_IO_TYPE_UNMAP: 333 case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: 334 case SPDK_BDEV_IO_TYPE_ZCOPY: 335 case SPDK_BDEV_IO_TYPE_ABORT: 336 return true; 337 338 default: 339 return false; 340 } 341 } 342 343 static struct spdk_io_channel * 344 bdev_malloc_get_io_channel(void *ctx) 345 { 346 return spdk_accel_engine_get_io_channel(); 347 } 348 349 static void 350 bdev_malloc_write_json_config(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 351 { 352 char uuid_str[SPDK_UUID_STRING_LEN]; 353 354 spdk_json_write_object_begin(w); 355 356 spdk_json_write_named_string(w, "method", "bdev_malloc_create"); 357 358 spdk_json_write_named_object_begin(w, "params"); 359 spdk_json_write_named_string(w, "name", bdev->name); 360 spdk_json_write_named_uint64(w, "num_blocks", bdev->blockcnt); 361 spdk_json_write_named_uint32(w, "block_size", bdev->blocklen); 362 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &bdev->uuid); 363 spdk_json_write_named_string(w, "uuid", uuid_str); 364 365 spdk_json_write_object_end(w); 366 367 spdk_json_write_object_end(w); 368 } 369 370 static const struct spdk_bdev_fn_table malloc_fn_table = { 371 .destruct = bdev_malloc_destruct, 372 .submit_request = bdev_malloc_submit_request, 373 .io_type_supported = bdev_malloc_io_type_supported, 374 .get_io_channel = bdev_malloc_get_io_channel, 375 .write_config_json = bdev_malloc_write_json_config, 376 }; 377 378 int 379 create_malloc_disk(struct spdk_bdev **bdev, const char *name, const struct spdk_uuid *uuid, 380 uint64_t num_blocks, uint32_t block_size) 381 { 382 struct malloc_disk *mdisk; 383 int rc; 384 385 if (num_blocks == 0) { 386 SPDK_ERRLOG("Disk num_blocks must be greater than 0"); 387 return -EINVAL; 388 } 389 390 mdisk = calloc(1, sizeof(*mdisk)); 391 if (!mdisk) { 392 SPDK_ERRLOG("mdisk calloc() failed\n"); 393 return -ENOMEM; 394 } 395 396 /* 397 * Allocate the large backend memory buffer from pinned memory. 398 * 399 * TODO: need to pass a hint so we know which socket to allocate 400 * from on multi-socket systems. 401 */ 402 mdisk->malloc_buf = spdk_zmalloc(num_blocks * block_size, 2 * 1024 * 1024, NULL, 403 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 404 if (!mdisk->malloc_buf) { 405 SPDK_ERRLOG("malloc_buf spdk_zmalloc() failed\n"); 406 malloc_disk_free(mdisk); 407 return -ENOMEM; 408 } 409 410 if (name) { 411 mdisk->disk.name = strdup(name); 412 } else { 413 /* Auto-generate a name */ 414 mdisk->disk.name = spdk_sprintf_alloc("Malloc%d", malloc_disk_count); 415 malloc_disk_count++; 416 } 417 if (!mdisk->disk.name) { 418 malloc_disk_free(mdisk); 419 return -ENOMEM; 420 } 421 mdisk->disk.product_name = "Malloc disk"; 422 423 mdisk->disk.write_cache = 1; 424 mdisk->disk.blocklen = block_size; 425 mdisk->disk.blockcnt = num_blocks; 426 if (uuid) { 427 mdisk->disk.uuid = *uuid; 428 } else { 429 spdk_uuid_generate(&mdisk->disk.uuid); 430 } 431 432 mdisk->disk.ctxt = mdisk; 433 mdisk->disk.fn_table = &malloc_fn_table; 434 mdisk->disk.module = &malloc_if; 435 436 rc = spdk_bdev_register(&mdisk->disk); 437 if (rc) { 438 malloc_disk_free(mdisk); 439 return rc; 440 } 441 442 *bdev = &(mdisk->disk); 443 444 TAILQ_INSERT_TAIL(&g_malloc_disks, mdisk, link); 445 446 return rc; 447 } 448 449 void 450 delete_malloc_disk(struct spdk_bdev *bdev, spdk_delete_malloc_complete cb_fn, void *cb_arg) 451 { 452 if (!bdev || bdev->module != &malloc_if) { 453 cb_fn(cb_arg, -ENODEV); 454 return; 455 } 456 457 spdk_bdev_unregister(bdev, cb_fn, cb_arg); 458 } 459 460 static int bdev_malloc_initialize(void) 461 { 462 /* This needs to be reset for each reinitialization of submodules. 463 * Otherwise after enough devices or reinitializations the value gets too high. 464 * TODO: Make malloc bdev name mandatory and remove this counter. */ 465 malloc_disk_count = 0; 466 return 0; 467 } 468 469 SPDK_LOG_REGISTER_COMPONENT(bdev_malloc) 470