1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 #include "spdk/stdinc.h" 5 #include "spdk/util.h" 6 #include "spdk/log.h" 7 #include "aio_mgr.h" 8 #include <libaio.h> 9 10 #define MAX_EVENTS 1024 11 12 struct spdk_aio_mgr_io { 13 struct spdk_aio_mgr *mgr; 14 TAILQ_ENTRY(spdk_aio_mgr_io) link; 15 struct iocb io; 16 fsdev_aio_done_cb clb; 17 void *ctx; 18 uint32_t data_size; 19 int err; 20 }; 21 22 struct spdk_aio_mgr { 23 TAILQ_HEAD(, spdk_aio_mgr_io) in_flight; 24 io_context_t io_ctx; 25 struct { 26 struct spdk_aio_mgr_io *arr; 27 uint32_t size; 28 TAILQ_HEAD(, spdk_aio_mgr_io) pool; 29 } aios; 30 uint32_t num_completions; 31 }; 32 33 static struct spdk_aio_mgr_io * 34 aio_mgr_get_aio(struct spdk_aio_mgr *mgr, fsdev_aio_done_cb clb, void *ctx) 35 { 36 struct spdk_aio_mgr_io *aio = TAILQ_FIRST(&mgr->aios.pool); 37 38 if (aio) { 39 aio->mgr = mgr; 40 aio->clb = clb; 41 aio->ctx = ctx; 42 aio->err = 0; 43 aio->data_size = 0; 44 TAILQ_REMOVE(&mgr->aios.pool, aio, link); 45 } 46 47 return aio; 48 } 49 50 static inline void 51 aio_mgr_put_aio(struct spdk_aio_mgr *mgr, struct spdk_aio_mgr_io *aio) 52 { 53 TAILQ_INSERT_TAIL(&aio->mgr->aios.pool, aio, link); 54 } 55 56 static void 57 spdk_aio_mgr_io_cpl_cb(io_context_t ctx, struct iocb *iocb, long res, long res2) 58 { 59 struct spdk_aio_mgr_io *aio = SPDK_CONTAINEROF(iocb, struct spdk_aio_mgr_io, io); 60 61 TAILQ_REMOVE(&aio->mgr->in_flight, aio, link); 62 63 aio->clb(aio->ctx, res, -res2); 64 65 aio->mgr->num_completions++; 66 67 aio_mgr_put_aio(aio->mgr, aio); 68 } 69 70 static struct spdk_aio_mgr_io * 71 spdk_aio_mgr_submit_io(struct spdk_aio_mgr *mgr, fsdev_aio_done_cb clb, void *ctx, int fd, 72 uint64_t offs, uint32_t size, struct iovec *iovs, uint32_t iovcnt, bool read) 73 { 74 struct spdk_aio_mgr_io *aio; 75 int res; 76 struct iocb *ios[1]; 77 78 SPDK_DEBUGLOG(spdk_aio_mgr_io, "%s: fd=%d offs=%" PRIu64 " size=%" PRIu32 " iovcnt=%" PRIu32 "\n", 79 read ? "read" : "write", fd, offs, size, iovcnt); 80 81 aio = aio_mgr_get_aio(mgr, clb, ctx); 82 if (!aio) { 83 SPDK_ERRLOG("Cannot get aio\n"); 84 clb(ctx, 0, EFAULT); 85 return NULL; 86 } 87 88 if (read) { 89 io_prep_preadv(&aio->io, fd, iovs, iovcnt, offs); 90 } else { 91 io_prep_pwritev(&aio->io, fd, iovs, iovcnt, offs); 92 } 93 io_set_callback(&aio->io, spdk_aio_mgr_io_cpl_cb); 94 95 96 ios[0] = &aio->io; 97 res = io_submit(mgr->io_ctx, 1, ios); 98 SPDK_DEBUGLOG(spdk_aio_mgr_io, "%s: aio=%p submitted with res=%d\n", read ? "read" : "write", aio, 99 res); 100 if (res) { 101 TAILQ_INSERT_TAIL(&aio->mgr->in_flight, aio, link); 102 return aio; 103 } else { 104 aio->clb(aio->ctx, 0, aio->err); 105 aio_mgr_put_aio(mgr, aio); 106 return NULL; 107 } 108 109 } 110 111 struct spdk_aio_mgr * 112 spdk_aio_mgr_create(uint32_t max_aios) 113 { 114 struct spdk_aio_mgr *mgr; 115 int res; 116 uint32_t i; 117 118 mgr = calloc(1, sizeof(*mgr)); 119 if (!mgr) { 120 SPDK_ERRLOG("cannot alloc mgr of %zu bytes\n", sizeof(*mgr)); 121 return NULL; 122 } 123 124 res = io_queue_init(max_aios, &mgr->io_ctx); 125 if (res) { 126 SPDK_ERRLOG("io_setup(%" PRIu32 ") failed with %d\n", max_aios, res); 127 free(mgr); 128 return NULL; 129 } 130 131 mgr->aios.arr = calloc(max_aios, sizeof(mgr->aios.arr[0])); 132 if (!mgr->aios.arr) { 133 SPDK_ERRLOG("cannot alloc aios pool of %" PRIu32 "\n", max_aios); 134 io_queue_release(mgr->io_ctx); 135 free(mgr); 136 return NULL; 137 } 138 139 TAILQ_INIT(&mgr->in_flight); 140 TAILQ_INIT(&mgr->aios.pool); 141 142 for (i = 0; i < max_aios; i++) { 143 TAILQ_INSERT_TAIL(&mgr->aios.pool, &mgr->aios.arr[i], link); 144 } 145 146 return mgr; 147 } 148 149 struct spdk_aio_mgr_io * 150 spdk_aio_mgr_read(struct spdk_aio_mgr *mgr, fsdev_aio_done_cb clb, void *ctx, 151 int fd, uint64_t offs, uint32_t size, struct iovec *iovs, uint32_t iovcnt) 152 { 153 return spdk_aio_mgr_submit_io(mgr, clb, ctx, fd, offs, size, iovs, iovcnt, true); 154 } 155 156 struct spdk_aio_mgr_io * 157 spdk_aio_mgr_write(struct spdk_aio_mgr *mgr, fsdev_aio_done_cb clb, void *ctx, 158 int fd, uint64_t offs, uint32_t size, const struct iovec *iovs, uint32_t iovcnt) 159 { 160 return spdk_aio_mgr_submit_io(mgr, clb, ctx, fd, offs, size, (struct iovec *)iovs, iovcnt, false); 161 } 162 163 void 164 spdk_aio_mgr_cancel(struct spdk_aio_mgr *mgr, struct spdk_aio_mgr_io *aio) 165 { 166 int res; 167 struct io_event result; 168 169 assert(mgr == aio->mgr); 170 171 res = io_cancel(mgr->io_ctx, &aio->io, &result); 172 if (res) { 173 SPDK_DEBUGLOG(spdk_aio_mgr_io, "aio=%p cancelled\n", aio); 174 spdk_aio_mgr_io_cpl_cb(mgr->io_ctx, &aio->io, ECANCELED, 0); 175 } else { 176 SPDK_WARNLOG("aio=%p cancellation failed with err=%d\n", aio, res); 177 } 178 } 179 180 bool 181 spdk_aio_mgr_poll(struct spdk_aio_mgr *mgr) 182 { 183 int res; 184 185 mgr->num_completions = 0; 186 187 res = io_queue_run(mgr->io_ctx); 188 if (res) { 189 SPDK_WARNLOG("polling failed with err=%d\n", res); 190 } 191 192 return mgr->num_completions ? true : false; 193 } 194 195 void 196 spdk_aio_mgr_delete(struct spdk_aio_mgr *mgr) 197 { 198 assert(TAILQ_EMPTY(&mgr->in_flight)); 199 free(mgr->aios.arr); 200 io_queue_release(mgr->io_ctx); 201 free(mgr); 202 } 203 204 SPDK_LOG_REGISTER_COMPONENT(spdk_aio_mgr_io) 205