1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/queue.h" 8 #include "spdk/util.h" 9 #include "spdk/env.h" 10 #include "spdk/likely.h" 11 12 #include "ftl_mempool.h" 13 #include "ftl_bitmap.h" 14 15 struct ftl_mempool_element { 16 SLIST_ENTRY(ftl_mempool_element) entry; 17 }; 18 19 struct ftl_mempool { 20 SLIST_HEAD(, ftl_mempool_element) list; 21 size_t element_size; 22 void *buffer; 23 size_t buffer_size; 24 size_t count; 25 size_t alignment; 26 int socket_id; 27 struct ftl_bitmap *inuse_bmp; 28 void *inuse_buf; 29 }; 30 31 static inline bool is_element_valid(struct ftl_mempool *mpool, 32 void *element) __attribute__((unused)); 33 34 static inline bool ftl_mempool_is_initialized(struct ftl_mempool *mpool) __attribute__((unused)); 35 36 static size_t 37 element_size_aligned(size_t size, size_t alignment) 38 { 39 if (!alignment) { 40 return size; 41 } 42 43 if (size % alignment) { 44 return (size / alignment + 1) * alignment; 45 } 46 47 return size; 48 } 49 50 static inline bool 51 is_element_valid(struct ftl_mempool *mpool, void *element) 52 { 53 if (element < mpool->buffer) { 54 return false; 55 } 56 57 if (element + mpool->element_size > mpool->buffer + mpool->buffer_size) { 58 return false; 59 } 60 61 if (!mpool->alignment) { 62 return true; 63 } 64 65 if ((size_t)element % mpool->alignment) { 66 return false; 67 } 68 69 if ((element - mpool->buffer) % mpool->element_size != 0) { 70 return false; 71 } 72 73 return true; 74 } 75 76 struct ftl_mempool *ftl_mempool_create(size_t count, size_t size, 77 size_t alignment, int socket_id) 78 { 79 struct ftl_mempool *mp; 80 void *buffer; 81 size_t i; 82 83 assert(count > 0); 84 assert(size > 0); 85 86 if (!spdk_u64_is_pow2(alignment)) { 87 return NULL; 88 } 89 90 mp = calloc(1, sizeof(*mp)); 91 if (!mp) { 92 return NULL; 93 } 94 95 size = spdk_max(size, sizeof(struct ftl_mempool_element)); 96 97 mp->count = count; 98 mp->element_size = element_size_aligned(size, alignment); 99 mp->alignment = alignment; 100 mp->socket_id = socket_id; 101 SLIST_INIT(&mp->list); 102 103 mp->buffer_size = mp->element_size * mp->count; 104 mp->buffer = spdk_dma_malloc_socket(mp->buffer_size, mp->alignment, 105 NULL, socket_id); 106 if (!mp->buffer) { 107 free(mp); 108 return NULL; 109 } 110 111 buffer = mp->buffer; 112 for (i = 0; i < count; i++, buffer += mp->element_size) { 113 struct ftl_mempool_element *el = buffer; 114 assert(is_element_valid(mp, el)); 115 SLIST_INSERT_HEAD(&mp->list, el, entry); 116 } 117 118 return mp; 119 } 120 121 void 122 ftl_mempool_destroy(struct ftl_mempool *mpool) 123 { 124 if (!mpool) { 125 return; 126 } 127 128 spdk_dma_free(mpool->buffer); 129 free(mpool); 130 } 131 132 static inline bool 133 ftl_mempool_is_initialized(struct ftl_mempool *mpool) 134 { 135 return mpool->inuse_buf == NULL; 136 } 137 138 void * 139 ftl_mempool_get(struct ftl_mempool *mpool) 140 { 141 struct ftl_mempool_element *el; 142 143 assert(ftl_mempool_is_initialized(mpool)); 144 if (spdk_unlikely(SLIST_EMPTY(&mpool->list))) { 145 return NULL; 146 } 147 148 el = SLIST_FIRST(&mpool->list); 149 SLIST_REMOVE_HEAD(&mpool->list, entry); 150 151 return el; 152 } 153 154 void 155 ftl_mempool_put(struct ftl_mempool *mpool, void *element) 156 { 157 struct ftl_mempool_element *el = element; 158 159 assert(ftl_mempool_is_initialized(mpool)); 160 assert(is_element_valid(mpool, element)); 161 SLIST_INSERT_HEAD(&mpool->list, el, entry); 162 } 163 164 struct ftl_mempool * 165 ftl_mempool_create_ext(void *buffer, size_t count, size_t size, size_t alignment) 166 { 167 struct ftl_mempool *mp; 168 size_t inuse_buf_sz; 169 170 assert(buffer); 171 assert(count > 0); 172 assert(size > 0); 173 174 mp = calloc(1, sizeof(*mp)); 175 if (!mp) { 176 goto error; 177 } 178 179 size = spdk_max(size, sizeof(struct ftl_mempool_element)); 180 181 mp->count = count; 182 mp->element_size = element_size_aligned(size, alignment); 183 mp->alignment = alignment; 184 SLIST_INIT(&mp->list); 185 186 /* Calculate underlying inuse_bmp's buf size */ 187 inuse_buf_sz = spdk_divide_round_up(mp->count, 8); 188 /* The bitmap size must be a multiple of word size (8b) - round up */ 189 if (inuse_buf_sz & 7UL) { 190 inuse_buf_sz &= ~7UL; 191 inuse_buf_sz += 8; 192 } 193 194 mp->inuse_buf = calloc(1, inuse_buf_sz); 195 if (!mp->inuse_buf) { 196 goto error; 197 } 198 199 mp->inuse_bmp = ftl_bitmap_create(mp->inuse_buf, inuse_buf_sz); 200 if (!mp->inuse_bmp) { 201 goto error; 202 } 203 204 /* Map the buffer */ 205 mp->buffer_size = mp->element_size * mp->count; 206 mp->buffer = buffer; 207 208 return mp; 209 210 error: 211 ftl_mempool_destroy_ext(mp); 212 return NULL; 213 } 214 215 void 216 ftl_mempool_destroy_ext(struct ftl_mempool *mpool) 217 { 218 if (!mpool) { 219 return; 220 } 221 222 if (mpool->inuse_bmp) { 223 ftl_bitmap_destroy(mpool->inuse_bmp); 224 } 225 free(mpool->inuse_buf); 226 free(mpool); 227 } 228 229 void 230 ftl_mempool_initialize_ext(struct ftl_mempool *mpool) 231 { 232 struct ftl_mempool_element *el; 233 void *buffer = mpool->buffer; 234 size_t i; 235 236 assert(!ftl_mempool_is_initialized(mpool)); 237 238 for (i = 0; i < mpool->count; i++, buffer += mpool->element_size) { 239 if (ftl_bitmap_get(mpool->inuse_bmp, i)) { 240 continue; 241 } 242 el = buffer; 243 assert(is_element_valid(mpool, el)); 244 SLIST_INSERT_HEAD(&mpool->list, el, entry); 245 } 246 247 ftl_bitmap_destroy(mpool->inuse_bmp); 248 mpool->inuse_bmp = NULL; 249 250 free(mpool->inuse_buf); 251 mpool->inuse_buf = NULL; 252 } 253 254 ftl_df_obj_id 255 ftl_mempool_get_df_obj_id(struct ftl_mempool *mpool, void *df_obj_ptr) 256 { 257 return ftl_df_get_obj_id(mpool->buffer, df_obj_ptr); 258 } 259 260 size_t 261 ftl_mempool_get_df_obj_index(struct ftl_mempool *mpool, void *df_obj_ptr) 262 { 263 return ftl_mempool_get_df_obj_id(mpool, df_obj_ptr) / mpool->element_size; 264 } 265 266 void * 267 ftl_mempool_get_df_ptr(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 268 { 269 return ftl_df_get_obj_ptr(mpool->buffer, df_obj_id); 270 } 271 272 void * 273 ftl_mempool_claim_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 274 { 275 struct ftl_mempool_element *el = ftl_df_get_obj_ptr(mpool->buffer, df_obj_id); 276 277 assert(!ftl_mempool_is_initialized(mpool)); 278 assert(df_obj_id % mpool->element_size == 0); 279 assert(df_obj_id / mpool->element_size < mpool->count); 280 281 ftl_bitmap_set(mpool->inuse_bmp, df_obj_id / mpool->element_size); 282 return el; 283 } 284 285 void 286 ftl_mempool_release_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 287 { 288 assert(!ftl_mempool_is_initialized(mpool)); 289 assert(df_obj_id % mpool->element_size == 0); 290 assert(df_obj_id / mpool->element_size < mpool->count); 291 292 ftl_bitmap_clear(mpool->inuse_bmp, df_obj_id / mpool->element_size); 293 } 294