1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/queue.h" 8 #include "spdk/util.h" 9 #include "spdk/env.h" 10 #include "spdk/likely.h" 11 12 #include "ftl_mempool.h" 13 #include "ftl_bitmap.h" 14 15 struct ftl_mempool_element { 16 SLIST_ENTRY(ftl_mempool_element) entry; 17 }; 18 19 struct ftl_mempool { 20 SLIST_HEAD(, ftl_mempool_element) list; 21 size_t element_size; 22 void *buffer; 23 size_t buffer_size; 24 size_t count; 25 size_t alignment; 26 int socket_id; 27 struct ftl_bitmap *inuse_bmp; 28 void *inuse_buf; 29 }; 30 31 static inline bool is_element_valid(struct ftl_mempool *mpool, 32 void *element) __attribute__((unused)); 33 34 static size_t 35 element_size_aligned(size_t size, size_t alignment) 36 { 37 if (!alignment) { 38 return size; 39 } 40 41 if (size % alignment) { 42 return (size / alignment + 1) * alignment; 43 } 44 45 return size; 46 } 47 48 static inline bool 49 is_element_valid(struct ftl_mempool *mpool, void *element) 50 { 51 if (element < mpool->buffer) { 52 return false; 53 } 54 55 if (element + mpool->element_size > mpool->buffer + mpool->buffer_size) { 56 return false; 57 } 58 59 if (!mpool->alignment) { 60 return true; 61 } 62 63 if ((size_t)element % mpool->alignment) { 64 return false; 65 } 66 67 if ((element - mpool->buffer) % mpool->element_size != 0) { 68 return false; 69 } 70 71 return true; 72 } 73 74 struct ftl_mempool *ftl_mempool_create(size_t count, size_t size, 75 size_t alignment, int socket_id) 76 { 77 struct ftl_mempool *mp; 78 void *buffer; 79 size_t i; 80 81 assert(count > 0); 82 assert(size > 0); 83 84 if (!spdk_u64_is_pow2(alignment)) { 85 return NULL; 86 } 87 88 mp = calloc(1, sizeof(*mp)); 89 if (!mp) { 90 return NULL; 91 } 92 93 size = spdk_max(size, sizeof(struct ftl_mempool_element)); 94 95 mp->count = count; 96 mp->element_size = element_size_aligned(size, alignment); 97 mp->alignment = alignment; 98 mp->socket_id = socket_id; 99 SLIST_INIT(&mp->list); 100 101 mp->buffer_size = mp->element_size * mp->count; 102 mp->buffer = spdk_dma_malloc_socket(mp->buffer_size, mp->alignment, 103 NULL, socket_id); 104 if (!mp->buffer) { 105 free(mp); 106 return NULL; 107 } 108 109 buffer = mp->buffer; 110 for (i = 0; i < count; i++, buffer += mp->element_size) { 111 struct ftl_mempool_element *el = buffer; 112 assert(is_element_valid(mp, el)); 113 SLIST_INSERT_HEAD(&mp->list, el, entry); 114 } 115 116 return mp; 117 } 118 119 void 120 ftl_mempool_destroy(struct ftl_mempool *mpool) 121 { 122 if (!mpool) { 123 return; 124 } 125 126 spdk_dma_free(mpool->buffer); 127 free(mpool); 128 } 129 130 static inline bool 131 ftl_mempool_is_initialized(struct ftl_mempool *mpool) 132 { 133 return mpool->inuse_buf == NULL; 134 } 135 136 void * 137 ftl_mempool_get(struct ftl_mempool *mpool) 138 { 139 struct ftl_mempool_element *el; 140 141 assert(ftl_mempool_is_initialized(mpool)); 142 if (spdk_unlikely(SLIST_EMPTY(&mpool->list))) { 143 return NULL; 144 } 145 146 el = SLIST_FIRST(&mpool->list); 147 SLIST_REMOVE_HEAD(&mpool->list, entry); 148 149 return el; 150 } 151 152 void 153 ftl_mempool_put(struct ftl_mempool *mpool, void *element) 154 { 155 struct ftl_mempool_element *el = element; 156 157 assert(ftl_mempool_is_initialized(mpool)); 158 assert(is_element_valid(mpool, element)); 159 SLIST_INSERT_HEAD(&mpool->list, el, entry); 160 } 161 162 struct ftl_mempool * 163 ftl_mempool_create_ext(void *buffer, size_t count, size_t size, size_t alignment) 164 { 165 struct ftl_mempool *mp; 166 size_t inuse_buf_sz; 167 168 assert(buffer); 169 assert(count > 0); 170 assert(size > 0); 171 172 mp = calloc(1, sizeof(*mp)); 173 if (!mp) { 174 goto error; 175 } 176 177 size = spdk_max(size, sizeof(struct ftl_mempool_element)); 178 179 mp->count = count; 180 mp->element_size = element_size_aligned(size, alignment); 181 mp->alignment = alignment; 182 SLIST_INIT(&mp->list); 183 184 /* Calculate underlying inuse_bmp's buf size */ 185 inuse_buf_sz = spdk_divide_round_up(mp->count, 8); 186 /* The bitmap size must be a multiple of word size (8b) - round up */ 187 if (inuse_buf_sz & 7UL) { 188 inuse_buf_sz &= ~7UL; 189 inuse_buf_sz += 8; 190 } 191 192 mp->inuse_buf = calloc(1, inuse_buf_sz); 193 if (!mp->inuse_buf) { 194 goto error; 195 } 196 197 mp->inuse_bmp = ftl_bitmap_create(mp->inuse_buf, inuse_buf_sz); 198 if (!mp->inuse_bmp) { 199 goto error; 200 } 201 202 /* Map the buffer */ 203 mp->buffer_size = mp->element_size * mp->count; 204 mp->buffer = buffer; 205 206 return mp; 207 208 error: 209 ftl_mempool_destroy_ext(mp); 210 return NULL; 211 } 212 213 void 214 ftl_mempool_destroy_ext(struct ftl_mempool *mpool) 215 { 216 if (!mpool) { 217 return; 218 } 219 220 if (mpool->inuse_bmp) { 221 ftl_bitmap_destroy(mpool->inuse_bmp); 222 } 223 free(mpool->inuse_buf); 224 free(mpool); 225 } 226 227 void 228 ftl_mempool_initialize_ext(struct ftl_mempool *mpool) 229 { 230 struct ftl_mempool_element *el; 231 void *buffer = mpool->buffer; 232 size_t i; 233 234 assert(!ftl_mempool_is_initialized(mpool)); 235 236 for (i = 0; i < mpool->count; i++, buffer += mpool->element_size) { 237 if (ftl_bitmap_get(mpool->inuse_bmp, i)) { 238 continue; 239 } 240 el = buffer; 241 assert(is_element_valid(mpool, el)); 242 SLIST_INSERT_HEAD(&mpool->list, el, entry); 243 } 244 245 ftl_bitmap_destroy(mpool->inuse_bmp); 246 mpool->inuse_bmp = NULL; 247 248 free(mpool->inuse_buf); 249 mpool->inuse_buf = NULL; 250 } 251 252 ftl_df_obj_id 253 ftl_mempool_get_df_obj_id(struct ftl_mempool *mpool, void *df_obj_ptr) 254 { 255 return ftl_df_get_obj_id(mpool->buffer, df_obj_ptr); 256 } 257 258 size_t 259 ftl_mempool_get_df_obj_index(struct ftl_mempool *mpool, void *df_obj_ptr) 260 { 261 return ftl_mempool_get_df_obj_id(mpool, df_obj_ptr) / mpool->element_size; 262 } 263 264 void * 265 ftl_mempool_get_df_ptr(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 266 { 267 return ftl_df_get_obj_ptr(mpool->buffer, df_obj_id); 268 } 269 270 void * 271 ftl_mempool_claim_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 272 { 273 struct ftl_mempool_element *el = ftl_df_get_obj_ptr(mpool->buffer, df_obj_id); 274 275 assert(!ftl_mempool_is_initialized(mpool)); 276 assert(df_obj_id % mpool->element_size == 0); 277 assert(df_obj_id / mpool->element_size < mpool->count); 278 279 ftl_bitmap_set(mpool->inuse_bmp, df_obj_id / mpool->element_size); 280 return el; 281 } 282 283 void 284 ftl_mempool_release_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id) 285 { 286 assert(!ftl_mempool_is_initialized(mpool)); 287 assert(df_obj_id % mpool->element_size == 0); 288 assert(df_obj_id / mpool->element_size < mpool->count); 289 290 ftl_bitmap_clear(mpool->inuse_bmp, df_obj_id / mpool->element_size); 291 } 292