1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "bdev_aio.h" 8 9 #include "spdk/stdinc.h" 10 11 #include "spdk/barrier.h" 12 #include "spdk/bdev.h" 13 #include "spdk/bdev_module.h" 14 #include "spdk/env.h" 15 #include "spdk/fd.h" 16 #include "spdk/likely.h" 17 #include "spdk/thread.h" 18 #include "spdk/json.h" 19 #include "spdk/util.h" 20 #include "spdk/string.h" 21 22 #include "spdk/log.h" 23 24 #include <sys/eventfd.h> 25 #include <libaio.h> 26 27 struct bdev_aio_io_channel { 28 uint64_t io_inflight; 29 io_context_t io_ctx; 30 struct bdev_aio_group_channel *group_ch; 31 TAILQ_ENTRY(bdev_aio_io_channel) link; 32 }; 33 34 struct bdev_aio_group_channel { 35 /* eventfd for io completion notification in interrupt mode. 36 * Negative value like '-1' indicates it is invalid or unused. 37 */ 38 int efd; 39 struct spdk_interrupt *intr; 40 struct spdk_poller *poller; 41 TAILQ_HEAD(, bdev_aio_io_channel) io_ch_head; 42 }; 43 44 struct bdev_aio_task { 45 struct iocb iocb; 46 uint64_t len; 47 struct bdev_aio_io_channel *ch; 48 }; 49 50 struct file_disk { 51 struct bdev_aio_task *reset_task; 52 struct spdk_poller *reset_retry_timer; 53 struct spdk_bdev disk; 54 char *filename; 55 int fd; 56 TAILQ_ENTRY(file_disk) link; 57 bool block_size_override; 58 }; 59 60 /* For user space reaping of completions */ 61 struct spdk_aio_ring { 62 uint32_t id; 63 uint32_t size; 64 uint32_t head; 65 uint32_t tail; 66 67 uint32_t version; 68 uint32_t compat_features; 69 uint32_t incompat_features; 70 uint32_t header_length; 71 }; 72 73 #define SPDK_AIO_RING_VERSION 0xa10a10a1 74 75 static int bdev_aio_initialize(void); 76 static void bdev_aio_fini(void); 77 static void aio_free_disk(struct file_disk *fdisk); 78 static TAILQ_HEAD(, file_disk) g_aio_disk_head = TAILQ_HEAD_INITIALIZER(g_aio_disk_head); 79 80 #define SPDK_AIO_QUEUE_DEPTH 128 81 #define MAX_EVENTS_PER_POLL 32 82 83 static int 84 bdev_aio_get_ctx_size(void) 85 { 86 return sizeof(struct bdev_aio_task); 87 } 88 89 static struct spdk_bdev_module aio_if = { 90 .name = "aio", 91 .module_init = bdev_aio_initialize, 92 .module_fini = bdev_aio_fini, 93 .get_ctx_size = bdev_aio_get_ctx_size, 94 }; 95 96 SPDK_BDEV_MODULE_REGISTER(aio, &aio_if) 97 98 static int 99 bdev_aio_open(struct file_disk *disk) 100 { 101 int fd; 102 103 fd = open(disk->filename, O_RDWR | O_DIRECT); 104 if (fd < 0) { 105 /* Try without O_DIRECT for non-disk files */ 106 fd = open(disk->filename, O_RDWR); 107 if (fd < 0) { 108 SPDK_ERRLOG("open() failed (file:%s), errno %d: %s\n", 109 disk->filename, errno, spdk_strerror(errno)); 110 disk->fd = -1; 111 return -1; 112 } 113 } 114 115 disk->fd = fd; 116 117 return 0; 118 } 119 120 static int 121 bdev_aio_close(struct file_disk *disk) 122 { 123 int rc; 124 125 if (disk->fd == -1) { 126 return 0; 127 } 128 129 rc = close(disk->fd); 130 if (rc < 0) { 131 SPDK_ERRLOG("close() failed (fd=%d), errno %d: %s\n", 132 disk->fd, errno, spdk_strerror(errno)); 133 return -1; 134 } 135 136 disk->fd = -1; 137 138 return 0; 139 } 140 141 static void 142 bdev_aio_readv(struct file_disk *fdisk, struct spdk_io_channel *ch, 143 struct bdev_aio_task *aio_task, 144 struct iovec *iov, int iovcnt, uint64_t nbytes, uint64_t offset) 145 { 146 struct iocb *iocb = &aio_task->iocb; 147 struct bdev_aio_io_channel *aio_ch = spdk_io_channel_get_ctx(ch); 148 int rc; 149 150 io_prep_preadv(iocb, fdisk->fd, iov, iovcnt, offset); 151 if (aio_ch->group_ch->efd >= 0) { 152 io_set_eventfd(iocb, aio_ch->group_ch->efd); 153 } 154 iocb->data = aio_task; 155 aio_task->len = nbytes; 156 aio_task->ch = aio_ch; 157 158 SPDK_DEBUGLOG(aio, "read %d iovs size %lu to off: %#lx\n", 159 iovcnt, nbytes, offset); 160 161 rc = io_submit(aio_ch->io_ctx, 1, &iocb); 162 if (spdk_unlikely(rc < 0)) { 163 if (rc == -EAGAIN) { 164 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(aio_task), SPDK_BDEV_IO_STATUS_NOMEM); 165 } else { 166 spdk_bdev_io_complete_aio_status(spdk_bdev_io_from_ctx(aio_task), rc); 167 SPDK_ERRLOG("%s: io_submit returned %d\n", __func__, rc); 168 } 169 } else { 170 aio_ch->io_inflight++; 171 } 172 } 173 174 static void 175 bdev_aio_writev(struct file_disk *fdisk, struct spdk_io_channel *ch, 176 struct bdev_aio_task *aio_task, 177 struct iovec *iov, int iovcnt, size_t len, uint64_t offset) 178 { 179 struct iocb *iocb = &aio_task->iocb; 180 struct bdev_aio_io_channel *aio_ch = spdk_io_channel_get_ctx(ch); 181 int rc; 182 183 io_prep_pwritev(iocb, fdisk->fd, iov, iovcnt, offset); 184 if (aio_ch->group_ch->efd >= 0) { 185 io_set_eventfd(iocb, aio_ch->group_ch->efd); 186 } 187 iocb->data = aio_task; 188 aio_task->len = len; 189 aio_task->ch = aio_ch; 190 191 SPDK_DEBUGLOG(aio, "write %d iovs size %lu from off: %#lx\n", 192 iovcnt, len, offset); 193 194 rc = io_submit(aio_ch->io_ctx, 1, &iocb); 195 if (spdk_unlikely(rc < 0)) { 196 if (rc == -EAGAIN) { 197 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(aio_task), SPDK_BDEV_IO_STATUS_NOMEM); 198 } else { 199 spdk_bdev_io_complete_aio_status(spdk_bdev_io_from_ctx(aio_task), rc); 200 SPDK_ERRLOG("%s: io_submit returned %d\n", __func__, rc); 201 } 202 } else { 203 aio_ch->io_inflight++; 204 } 205 } 206 207 static void 208 bdev_aio_flush(struct file_disk *fdisk, struct bdev_aio_task *aio_task) 209 { 210 int rc = fsync(fdisk->fd); 211 212 if (rc == 0) { 213 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(aio_task), SPDK_BDEV_IO_STATUS_SUCCESS); 214 } else { 215 spdk_bdev_io_complete_aio_status(spdk_bdev_io_from_ctx(aio_task), -errno); 216 } 217 } 218 219 static void 220 bdev_aio_destruct_cb(void *io_device) 221 { 222 struct file_disk *fdisk = io_device; 223 int rc = 0; 224 225 TAILQ_REMOVE(&g_aio_disk_head, fdisk, link); 226 rc = bdev_aio_close(fdisk); 227 if (rc < 0) { 228 SPDK_ERRLOG("bdev_aio_close() failed\n"); 229 } 230 231 aio_free_disk(fdisk); 232 } 233 234 static int 235 bdev_aio_destruct(void *ctx) 236 { 237 struct file_disk *fdisk = ctx; 238 239 spdk_io_device_unregister(fdisk, bdev_aio_destruct_cb); 240 241 return 0; 242 } 243 244 static int 245 bdev_user_io_getevents(io_context_t io_ctx, unsigned int max, struct io_event *uevents) 246 { 247 uint32_t head, tail, count; 248 struct spdk_aio_ring *ring; 249 struct timespec timeout; 250 struct io_event *kevents; 251 252 ring = (struct spdk_aio_ring *)io_ctx; 253 254 if (spdk_unlikely(ring->version != SPDK_AIO_RING_VERSION || ring->incompat_features != 0)) { 255 timeout.tv_sec = 0; 256 timeout.tv_nsec = 0; 257 258 return io_getevents(io_ctx, 0, max, uevents, &timeout); 259 } 260 261 /* Read the current state out of the ring */ 262 head = ring->head; 263 tail = ring->tail; 264 265 /* This memory barrier is required to prevent the loads above 266 * from being re-ordered with stores to the events array 267 * potentially occurring on other threads. */ 268 spdk_smp_rmb(); 269 270 /* Calculate how many items are in the circular ring */ 271 count = tail - head; 272 if (tail < head) { 273 count += ring->size; 274 } 275 276 /* Reduce the count to the limit provided by the user */ 277 count = spdk_min(max, count); 278 279 /* Grab the memory location of the event array */ 280 kevents = (struct io_event *)((uintptr_t)ring + ring->header_length); 281 282 /* Copy the events out of the ring. */ 283 if ((head + count) <= ring->size) { 284 /* Only one copy is required */ 285 memcpy(uevents, &kevents[head], count * sizeof(struct io_event)); 286 } else { 287 uint32_t first_part = ring->size - head; 288 /* Two copies are required */ 289 memcpy(uevents, &kevents[head], first_part * sizeof(struct io_event)); 290 memcpy(&uevents[first_part], &kevents[0], (count - first_part) * sizeof(struct io_event)); 291 } 292 293 /* Update the head pointer. On x86, stores will not be reordered with older loads, 294 * so the copies out of the event array will always be complete prior to this 295 * update becoming visible. On other architectures this is not guaranteed, so 296 * add a barrier. */ 297 #if defined(__i386__) || defined(__x86_64__) 298 spdk_compiler_barrier(); 299 #else 300 spdk_smp_mb(); 301 #endif 302 ring->head = (head + count) % ring->size; 303 304 return count; 305 } 306 307 static int 308 bdev_aio_io_channel_poll(struct bdev_aio_io_channel *io_ch) 309 { 310 int nr, i = 0; 311 struct bdev_aio_task *aio_task; 312 struct io_event events[SPDK_AIO_QUEUE_DEPTH]; 313 314 nr = bdev_user_io_getevents(io_ch->io_ctx, SPDK_AIO_QUEUE_DEPTH, events); 315 if (nr < 0) { 316 return 0; 317 } 318 319 for (i = 0; i < nr; i++) { 320 aio_task = events[i].data; 321 aio_task->ch->io_inflight--; 322 if (events[i].res == aio_task->len) { 323 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(aio_task), SPDK_BDEV_IO_STATUS_SUCCESS); 324 } else { 325 /* From aio_abi.h, io_event.res is defined __s64, negative errno 326 * will be assigned to io_event.res for error situation. 327 * But from libaio.h, io_event.res is defined unsigned long, so 328 * convert it to signed value for error detection. 329 */ 330 SPDK_ERRLOG("failed to complete aio: rc %"PRId64"\n", events[i].res); 331 spdk_bdev_io_complete_aio_status(spdk_bdev_io_from_ctx(aio_task), (int)events[i].res); 332 } 333 } 334 335 return nr; 336 } 337 338 static int 339 bdev_aio_group_poll(void *arg) 340 { 341 struct bdev_aio_group_channel *group_ch = arg; 342 struct bdev_aio_io_channel *io_ch; 343 int nr = 0; 344 345 TAILQ_FOREACH(io_ch, &group_ch->io_ch_head, link) { 346 nr += bdev_aio_io_channel_poll(io_ch); 347 } 348 349 return nr > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 350 } 351 352 static int 353 bdev_aio_group_interrupt(void *arg) 354 { 355 struct bdev_aio_group_channel *group_ch = arg; 356 int rc; 357 uint64_t num_events; 358 359 assert(group_ch->efd >= 0); 360 361 /* if completed IO number is larger than SPDK_AIO_QUEUE_DEPTH, 362 * io_getevent should be called again to ensure all completed IO are processed. 363 */ 364 rc = read(group_ch->efd, &num_events, sizeof(num_events)); 365 if (rc < 0) { 366 SPDK_ERRLOG("failed to acknowledge aio group: %s.\n", spdk_strerror(errno)); 367 return -errno; 368 } 369 370 if (num_events > SPDK_AIO_QUEUE_DEPTH) { 371 num_events -= SPDK_AIO_QUEUE_DEPTH; 372 rc = write(group_ch->efd, &num_events, sizeof(num_events)); 373 if (rc < 0) { 374 SPDK_ERRLOG("failed to notify aio group: %s.\n", spdk_strerror(errno)); 375 } 376 } 377 378 return bdev_aio_group_poll(group_ch); 379 } 380 381 static void 382 _bdev_aio_get_io_inflight(struct spdk_io_channel_iter *i) 383 { 384 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 385 struct bdev_aio_io_channel *aio_ch = spdk_io_channel_get_ctx(ch); 386 387 if (aio_ch->io_inflight) { 388 spdk_for_each_channel_continue(i, -1); 389 return; 390 } 391 392 spdk_for_each_channel_continue(i, 0); 393 } 394 395 static int bdev_aio_reset_retry_timer(void *arg); 396 397 static void 398 _bdev_aio_get_io_inflight_done(struct spdk_io_channel_iter *i, int status) 399 { 400 struct file_disk *fdisk = spdk_io_channel_iter_get_ctx(i); 401 402 if (status == -1) { 403 fdisk->reset_retry_timer = SPDK_POLLER_REGISTER(bdev_aio_reset_retry_timer, fdisk, 500); 404 return; 405 } 406 407 spdk_bdev_io_complete(spdk_bdev_io_from_ctx(fdisk->reset_task), SPDK_BDEV_IO_STATUS_SUCCESS); 408 } 409 410 static int 411 bdev_aio_reset_retry_timer(void *arg) 412 { 413 struct file_disk *fdisk = arg; 414 415 if (fdisk->reset_retry_timer) { 416 spdk_poller_unregister(&fdisk->reset_retry_timer); 417 } 418 419 spdk_for_each_channel(fdisk, 420 _bdev_aio_get_io_inflight, 421 fdisk, 422 _bdev_aio_get_io_inflight_done); 423 424 return SPDK_POLLER_BUSY; 425 } 426 427 static void 428 bdev_aio_reset(struct file_disk *fdisk, struct bdev_aio_task *aio_task) 429 { 430 fdisk->reset_task = aio_task; 431 432 bdev_aio_reset_retry_timer(fdisk); 433 } 434 435 static void 436 bdev_aio_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 437 bool success) 438 { 439 if (!success) { 440 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 441 return; 442 } 443 444 switch (bdev_io->type) { 445 case SPDK_BDEV_IO_TYPE_READ: 446 bdev_aio_readv((struct file_disk *)bdev_io->bdev->ctxt, 447 ch, 448 (struct bdev_aio_task *)bdev_io->driver_ctx, 449 bdev_io->u.bdev.iovs, 450 bdev_io->u.bdev.iovcnt, 451 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen, 452 bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen); 453 break; 454 case SPDK_BDEV_IO_TYPE_WRITE: 455 bdev_aio_writev((struct file_disk *)bdev_io->bdev->ctxt, 456 ch, 457 (struct bdev_aio_task *)bdev_io->driver_ctx, 458 bdev_io->u.bdev.iovs, 459 bdev_io->u.bdev.iovcnt, 460 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen, 461 bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen); 462 break; 463 default: 464 SPDK_ERRLOG("Wrong io type\n"); 465 break; 466 } 467 } 468 469 static int 470 _bdev_aio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 471 { 472 switch (bdev_io->type) { 473 /* Read and write operations must be performed on buffers aligned to 474 * bdev->required_alignment. If user specified unaligned buffers, 475 * get the aligned buffer from the pool by calling spdk_bdev_io_get_buf. */ 476 case SPDK_BDEV_IO_TYPE_READ: 477 case SPDK_BDEV_IO_TYPE_WRITE: 478 spdk_bdev_io_get_buf(bdev_io, bdev_aio_get_buf_cb, 479 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 480 return 0; 481 case SPDK_BDEV_IO_TYPE_FLUSH: 482 bdev_aio_flush((struct file_disk *)bdev_io->bdev->ctxt, 483 (struct bdev_aio_task *)bdev_io->driver_ctx); 484 return 0; 485 486 case SPDK_BDEV_IO_TYPE_RESET: 487 bdev_aio_reset((struct file_disk *)bdev_io->bdev->ctxt, 488 (struct bdev_aio_task *)bdev_io->driver_ctx); 489 return 0; 490 default: 491 return -1; 492 } 493 } 494 495 static void 496 bdev_aio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 497 { 498 if (_bdev_aio_submit_request(ch, bdev_io) < 0) { 499 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 500 } 501 } 502 503 static bool 504 bdev_aio_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 505 { 506 switch (io_type) { 507 case SPDK_BDEV_IO_TYPE_READ: 508 case SPDK_BDEV_IO_TYPE_WRITE: 509 case SPDK_BDEV_IO_TYPE_FLUSH: 510 case SPDK_BDEV_IO_TYPE_RESET: 511 return true; 512 513 default: 514 return false; 515 } 516 } 517 518 static int 519 bdev_aio_create_cb(void *io_device, void *ctx_buf) 520 { 521 struct bdev_aio_io_channel *ch = ctx_buf; 522 523 if (io_setup(SPDK_AIO_QUEUE_DEPTH, &ch->io_ctx) < 0) { 524 SPDK_ERRLOG("async I/O context setup failure\n"); 525 return -1; 526 } 527 528 ch->group_ch = spdk_io_channel_get_ctx(spdk_get_io_channel(&aio_if)); 529 TAILQ_INSERT_TAIL(&ch->group_ch->io_ch_head, ch, link); 530 531 return 0; 532 } 533 534 static void 535 bdev_aio_destroy_cb(void *io_device, void *ctx_buf) 536 { 537 struct bdev_aio_io_channel *ch = ctx_buf; 538 539 io_destroy(ch->io_ctx); 540 541 assert(ch->group_ch); 542 TAILQ_REMOVE(&ch->group_ch->io_ch_head, ch, link); 543 544 spdk_put_io_channel(spdk_io_channel_from_ctx(ch->group_ch)); 545 } 546 547 static struct spdk_io_channel * 548 bdev_aio_get_io_channel(void *ctx) 549 { 550 struct file_disk *fdisk = ctx; 551 552 return spdk_get_io_channel(fdisk); 553 } 554 555 556 static int 557 bdev_aio_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 558 { 559 struct file_disk *fdisk = ctx; 560 561 spdk_json_write_named_object_begin(w, "aio"); 562 563 spdk_json_write_named_string(w, "filename", fdisk->filename); 564 565 spdk_json_write_object_end(w); 566 567 return 0; 568 } 569 570 static void 571 bdev_aio_write_json_config(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 572 { 573 struct file_disk *fdisk = bdev->ctxt; 574 575 spdk_json_write_object_begin(w); 576 577 spdk_json_write_named_string(w, "method", "bdev_aio_create"); 578 579 spdk_json_write_named_object_begin(w, "params"); 580 spdk_json_write_named_string(w, "name", bdev->name); 581 if (fdisk->block_size_override) { 582 spdk_json_write_named_uint32(w, "block_size", bdev->blocklen); 583 } 584 spdk_json_write_named_string(w, "filename", fdisk->filename); 585 spdk_json_write_object_end(w); 586 587 spdk_json_write_object_end(w); 588 } 589 590 static const struct spdk_bdev_fn_table aio_fn_table = { 591 .destruct = bdev_aio_destruct, 592 .submit_request = bdev_aio_submit_request, 593 .io_type_supported = bdev_aio_io_type_supported, 594 .get_io_channel = bdev_aio_get_io_channel, 595 .dump_info_json = bdev_aio_dump_info_json, 596 .write_config_json = bdev_aio_write_json_config, 597 }; 598 599 static void 600 aio_free_disk(struct file_disk *fdisk) 601 { 602 if (fdisk == NULL) { 603 return; 604 } 605 free(fdisk->filename); 606 free(fdisk->disk.name); 607 free(fdisk); 608 } 609 610 static int 611 bdev_aio_register_interrupt(struct bdev_aio_group_channel *ch) 612 { 613 int efd; 614 615 efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); 616 if (efd < 0) { 617 return -1; 618 } 619 620 ch->intr = SPDK_INTERRUPT_REGISTER(efd, bdev_aio_group_interrupt, ch); 621 if (ch->intr == NULL) { 622 close(efd); 623 return -1; 624 } 625 ch->efd = efd; 626 627 return 0; 628 } 629 630 static void 631 bdev_aio_unregister_interrupt(struct bdev_aio_group_channel *ch) 632 { 633 spdk_interrupt_unregister(&ch->intr); 634 close(ch->efd); 635 ch->efd = -1; 636 } 637 638 static void 639 bdev_aio_poller_set_interrupt_mode(struct spdk_poller *poller, void *cb_arg, bool interrupt_mode) 640 { 641 return; 642 } 643 644 static int 645 bdev_aio_group_create_cb(void *io_device, void *ctx_buf) 646 { 647 struct bdev_aio_group_channel *ch = ctx_buf; 648 int rc; 649 650 TAILQ_INIT(&ch->io_ch_head); 651 /* Initialize ch->efd to be invalid and unused. */ 652 ch->efd = -1; 653 if (spdk_interrupt_mode_is_enabled()) { 654 rc = bdev_aio_register_interrupt(ch); 655 if (rc < 0) { 656 SPDK_ERRLOG("Failed to prepare intr resource to bdev_aio\n"); 657 return rc; 658 } 659 } 660 661 ch->poller = SPDK_POLLER_REGISTER(bdev_aio_group_poll, ch, 0); 662 spdk_poller_register_interrupt(ch->poller, bdev_aio_poller_set_interrupt_mode, NULL); 663 664 return 0; 665 } 666 667 static void 668 bdev_aio_group_destroy_cb(void *io_device, void *ctx_buf) 669 { 670 struct bdev_aio_group_channel *ch = ctx_buf; 671 672 if (!TAILQ_EMPTY(&ch->io_ch_head)) { 673 SPDK_ERRLOG("Group channel of bdev aio has uncleared io channel\n"); 674 } 675 676 spdk_poller_unregister(&ch->poller); 677 if (spdk_interrupt_mode_is_enabled()) { 678 bdev_aio_unregister_interrupt(ch); 679 } 680 } 681 682 int 683 create_aio_bdev(const char *name, const char *filename, uint32_t block_size) 684 { 685 struct file_disk *fdisk; 686 uint32_t detected_block_size; 687 uint64_t disk_size; 688 int rc; 689 690 fdisk = calloc(1, sizeof(*fdisk)); 691 if (!fdisk) { 692 SPDK_ERRLOG("Unable to allocate enough memory for aio backend\n"); 693 return -ENOMEM; 694 } 695 696 fdisk->filename = strdup(filename); 697 if (!fdisk->filename) { 698 rc = -ENOMEM; 699 goto error_return; 700 } 701 702 if (bdev_aio_open(fdisk)) { 703 SPDK_ERRLOG("Unable to open file %s. fd: %d errno: %d\n", filename, fdisk->fd, errno); 704 rc = -errno; 705 goto error_return; 706 } 707 708 disk_size = spdk_fd_get_size(fdisk->fd); 709 710 fdisk->disk.name = strdup(name); 711 if (!fdisk->disk.name) { 712 rc = -ENOMEM; 713 goto error_return; 714 } 715 fdisk->disk.product_name = "AIO disk"; 716 fdisk->disk.module = &aio_if; 717 718 fdisk->disk.write_cache = 1; 719 720 detected_block_size = spdk_fd_get_blocklen(fdisk->fd); 721 if (block_size == 0) { 722 /* User did not specify block size - use autodetected block size. */ 723 if (detected_block_size == 0) { 724 SPDK_ERRLOG("Block size could not be auto-detected\n"); 725 rc = -EINVAL; 726 goto error_return; 727 } 728 fdisk->block_size_override = false; 729 block_size = detected_block_size; 730 } else { 731 if (block_size < detected_block_size) { 732 SPDK_ERRLOG("Specified block size %" PRIu32 " is smaller than " 733 "auto-detected block size %" PRIu32 "\n", 734 block_size, detected_block_size); 735 rc = -EINVAL; 736 goto error_return; 737 } else if (detected_block_size != 0 && block_size != detected_block_size) { 738 SPDK_WARNLOG("Specified block size %" PRIu32 " does not match " 739 "auto-detected block size %" PRIu32 "\n", 740 block_size, detected_block_size); 741 } 742 fdisk->block_size_override = true; 743 } 744 745 if (block_size < 512) { 746 SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be at least 512).\n", block_size); 747 rc = -EINVAL; 748 goto error_return; 749 } 750 751 if (!spdk_u32_is_pow2(block_size)) { 752 SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be a power of 2.)\n", block_size); 753 rc = -EINVAL; 754 goto error_return; 755 } 756 757 fdisk->disk.blocklen = block_size; 758 if (fdisk->block_size_override && detected_block_size) { 759 fdisk->disk.required_alignment = spdk_u32log2(detected_block_size); 760 } else { 761 fdisk->disk.required_alignment = spdk_u32log2(block_size); 762 } 763 764 if (disk_size % fdisk->disk.blocklen != 0) { 765 SPDK_ERRLOG("Disk size %" PRIu64 " is not a multiple of block size %" PRIu32 "\n", 766 disk_size, fdisk->disk.blocklen); 767 rc = -EINVAL; 768 goto error_return; 769 } 770 771 fdisk->disk.blockcnt = disk_size / fdisk->disk.blocklen; 772 fdisk->disk.ctxt = fdisk; 773 774 fdisk->disk.fn_table = &aio_fn_table; 775 776 spdk_io_device_register(fdisk, bdev_aio_create_cb, bdev_aio_destroy_cb, 777 sizeof(struct bdev_aio_io_channel), 778 fdisk->disk.name); 779 rc = spdk_bdev_register(&fdisk->disk); 780 if (rc) { 781 spdk_io_device_unregister(fdisk, NULL); 782 goto error_return; 783 } 784 785 TAILQ_INSERT_TAIL(&g_aio_disk_head, fdisk, link); 786 return 0; 787 788 error_return: 789 bdev_aio_close(fdisk); 790 aio_free_disk(fdisk); 791 return rc; 792 } 793 794 static void 795 dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 796 { 797 } 798 799 int 800 bdev_aio_rescan(const char *name) 801 { 802 struct spdk_bdev_desc *desc; 803 struct spdk_bdev *bdev; 804 struct file_disk *fdisk; 805 uint64_t disk_size, blockcnt; 806 int rc; 807 808 rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &desc); 809 if (rc != 0) { 810 return rc; 811 } 812 813 bdev = spdk_bdev_desc_get_bdev(desc); 814 if (bdev->module != &aio_if) { 815 rc = -ENODEV; 816 goto exit; 817 } 818 819 fdisk = SPDK_CONTAINEROF(bdev, struct file_disk, disk); 820 disk_size = spdk_fd_get_size(fdisk->fd); 821 blockcnt = disk_size / bdev->blocklen; 822 823 if (bdev->blockcnt != blockcnt) { 824 SPDK_NOTICELOG("AIO device is resized: bdev name %s, old block count %" PRIu64 ", new block count %" 825 PRIu64 "\n", 826 fdisk->filename, 827 bdev->blockcnt, 828 blockcnt); 829 rc = spdk_bdev_notify_blockcnt_change(bdev, blockcnt); 830 if (rc != 0) { 831 SPDK_ERRLOG("Could not change num blocks for aio bdev: name %s, errno: %d.\n", 832 fdisk->filename, rc); 833 goto exit; 834 } 835 } 836 837 exit: 838 spdk_bdev_close(desc); 839 return rc; 840 } 841 842 struct delete_aio_bdev_ctx { 843 delete_aio_bdev_complete cb_fn; 844 void *cb_arg; 845 }; 846 847 static void 848 aio_bdev_unregister_cb(void *arg, int bdeverrno) 849 { 850 struct delete_aio_bdev_ctx *ctx = arg; 851 852 ctx->cb_fn(ctx->cb_arg, bdeverrno); 853 free(ctx); 854 } 855 856 void 857 bdev_aio_delete(const char *name, delete_aio_bdev_complete cb_fn, void *cb_arg) 858 { 859 struct delete_aio_bdev_ctx *ctx; 860 int rc; 861 862 ctx = calloc(1, sizeof(*ctx)); 863 if (ctx == NULL) { 864 cb_fn(cb_arg, -ENOMEM); 865 return; 866 } 867 868 ctx->cb_fn = cb_fn; 869 ctx->cb_arg = cb_arg; 870 rc = spdk_bdev_unregister_by_name(name, &aio_if, aio_bdev_unregister_cb, ctx); 871 if (rc != 0) { 872 aio_bdev_unregister_cb(ctx, rc); 873 } 874 } 875 876 static int 877 bdev_aio_initialize(void) 878 { 879 spdk_io_device_register(&aio_if, bdev_aio_group_create_cb, bdev_aio_group_destroy_cb, 880 sizeof(struct bdev_aio_group_channel), "aio_module"); 881 882 return 0; 883 } 884 885 static void 886 bdev_aio_fini(void) 887 { 888 spdk_io_device_unregister(&aio_if, NULL); 889 } 890 891 SPDK_LOG_REGISTER_COMPONENT(aio) 892