1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <inttypes.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_log.h> 12 #include <rte_memory.h> 13 #include <rte_memzone.h> 14 #include <rte_eal.h> 15 #include <rte_errno.h> 16 #include <rte_string_fns.h> 17 #include <rte_common.h> 18 #include <rte_eal_trace.h> 19 20 #include "malloc_heap.h" 21 #include "malloc_elem.h" 22 #include "eal_private.h" 23 #include "eal_memcfg.h" 24 25 static inline const struct rte_memzone * 26 memzone_lookup_thread_unsafe(const char *name) 27 { 28 struct rte_mem_config *mcfg; 29 struct rte_fbarray *arr; 30 const struct rte_memzone *mz; 31 int i = 0; 32 33 /* get pointer to global configuration */ 34 mcfg = rte_eal_get_configuration()->mem_config; 35 arr = &mcfg->memzones; 36 37 /* 38 * the algorithm is not optimal (linear), but there are few 39 * zones and this function should be called at init only 40 */ 41 i = rte_fbarray_find_next_used(arr, 0); 42 while (i >= 0) { 43 mz = rte_fbarray_get(arr, i); 44 if (mz->addr != NULL && 45 !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE)) 46 return mz; 47 i = rte_fbarray_find_next_used(arr, i + 1); 48 } 49 return NULL; 50 } 51 52 #define MEMZONE_KNOWN_FLAGS (RTE_MEMZONE_2MB \ 53 | RTE_MEMZONE_1GB \ 54 | RTE_MEMZONE_16MB \ 55 | RTE_MEMZONE_16GB \ 56 | RTE_MEMZONE_256KB \ 57 | RTE_MEMZONE_256MB \ 58 | RTE_MEMZONE_512MB \ 59 | RTE_MEMZONE_4GB \ 60 | RTE_MEMZONE_SIZE_HINT_ONLY \ 61 | RTE_MEMZONE_IOVA_CONTIG \ 62 ) 63 64 static const struct rte_memzone * 65 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, 66 int socket_id, unsigned int flags, unsigned int align, 67 unsigned int bound) 68 { 69 struct rte_memzone *mz; 70 struct rte_mem_config *mcfg; 71 struct rte_fbarray *arr; 72 void *mz_addr; 73 size_t requested_len; 74 int mz_idx; 75 bool contig; 76 77 /* get pointer to global configuration */ 78 mcfg = rte_eal_get_configuration()->mem_config; 79 arr = &mcfg->memzones; 80 81 /* no more room in config */ 82 if (arr->count >= arr->len) { 83 RTE_LOG(ERR, EAL, 84 "%s(): Number of requested memzone segments exceeds RTE_MAX_MEMZONE\n", 85 __func__); 86 rte_errno = ENOSPC; 87 return NULL; 88 } 89 90 if (strlen(name) > sizeof(mz->name) - 1) { 91 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n", 92 __func__, name); 93 rte_errno = ENAMETOOLONG; 94 return NULL; 95 } 96 97 /* zone already exist */ 98 if ((memzone_lookup_thread_unsafe(name)) != NULL) { 99 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n", 100 __func__, name); 101 rte_errno = EEXIST; 102 return NULL; 103 } 104 105 /* if alignment is not a power of two */ 106 if (align && !rte_is_power_of_2(align)) { 107 RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__, 108 align); 109 rte_errno = EINVAL; 110 return NULL; 111 } 112 113 /* alignment less than cache size is not allowed */ 114 if (align < RTE_CACHE_LINE_SIZE) 115 align = RTE_CACHE_LINE_SIZE; 116 117 /* align length on cache boundary. Check for overflow before doing so */ 118 if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) { 119 rte_errno = EINVAL; /* requested size too big */ 120 return NULL; 121 } 122 123 len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE); 124 125 /* save minimal requested length */ 126 requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len); 127 128 /* check that boundary condition is valid */ 129 if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) { 130 rte_errno = EINVAL; 131 return NULL; 132 } 133 134 if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) { 135 rte_errno = EINVAL; 136 return NULL; 137 } 138 139 if ((flags & ~MEMZONE_KNOWN_FLAGS) != 0) { 140 rte_errno = EINVAL; 141 return NULL; 142 } 143 144 /* only set socket to SOCKET_ID_ANY if we aren't allocating for an 145 * external heap. 146 */ 147 if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES) 148 socket_id = SOCKET_ID_ANY; 149 150 contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0; 151 /* malloc only cares about size flags, remove contig flag from flags */ 152 flags &= ~RTE_MEMZONE_IOVA_CONTIG; 153 154 if (len == 0 && bound == 0) { 155 /* no size constraints were placed, so use malloc elem len */ 156 requested_len = 0; 157 mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags, 158 align, contig); 159 } else { 160 if (len == 0) 161 requested_len = bound; 162 /* allocate memory on heap */ 163 mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, 164 flags, align, bound, contig); 165 } 166 if (mz_addr == NULL) { 167 rte_errno = ENOMEM; 168 return NULL; 169 } 170 171 struct malloc_elem *elem = malloc_elem_from_data(mz_addr); 172 173 /* fill the zone in config */ 174 mz_idx = rte_fbarray_find_next_free(arr, 0); 175 176 if (mz_idx < 0) { 177 mz = NULL; 178 } else { 179 rte_fbarray_set_used(arr, mz_idx); 180 mz = rte_fbarray_get(arr, mz_idx); 181 } 182 183 if (mz == NULL) { 184 RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__); 185 malloc_heap_free(elem); 186 rte_errno = ENOSPC; 187 return NULL; 188 } 189 190 strlcpy(mz->name, name, sizeof(mz->name)); 191 mz->iova = rte_malloc_virt2iova(mz_addr); 192 mz->addr = mz_addr; 193 mz->len = requested_len == 0 ? 194 elem->size - elem->pad - MALLOC_ELEM_OVERHEAD : 195 requested_len; 196 mz->hugepage_sz = elem->msl->page_sz; 197 mz->socket_id = elem->msl->socket_id; 198 mz->flags = 0; 199 200 return mz; 201 } 202 203 static const struct rte_memzone * 204 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id, 205 unsigned int flags, unsigned int align, unsigned int bound) 206 { 207 struct rte_mem_config *mcfg; 208 const struct rte_memzone *mz = NULL; 209 210 /* get pointer to global configuration */ 211 mcfg = rte_eal_get_configuration()->mem_config; 212 213 rte_rwlock_write_lock(&mcfg->mlock); 214 215 mz = memzone_reserve_aligned_thread_unsafe( 216 name, len, socket_id, flags, align, bound); 217 218 rte_eal_trace_memzone_reserve(name, len, socket_id, flags, align, 219 bound, mz); 220 221 rte_rwlock_write_unlock(&mcfg->mlock); 222 223 return mz; 224 } 225 226 /* 227 * Return a pointer to a correctly filled memzone descriptor (with a 228 * specified alignment and boundary). If the allocation cannot be done, 229 * return NULL. 230 */ 231 const struct rte_memzone * 232 rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id, 233 unsigned flags, unsigned align, unsigned bound) 234 { 235 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, 236 align, bound); 237 } 238 239 /* 240 * Return a pointer to a correctly filled memzone descriptor (with a 241 * specified alignment). If the allocation cannot be done, return NULL. 242 */ 243 const struct rte_memzone * 244 rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 245 unsigned flags, unsigned align) 246 { 247 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, 248 align, 0); 249 } 250 251 /* 252 * Return a pointer to a correctly filled memzone descriptor. If the 253 * allocation cannot be done, return NULL. 254 */ 255 const struct rte_memzone * 256 rte_memzone_reserve(const char *name, size_t len, int socket_id, 257 unsigned flags) 258 { 259 return rte_memzone_reserve_thread_safe(name, len, socket_id, 260 flags, RTE_CACHE_LINE_SIZE, 0); 261 } 262 263 int 264 rte_memzone_free(const struct rte_memzone *mz) 265 { 266 char name[RTE_MEMZONE_NAMESIZE]; 267 struct rte_mem_config *mcfg; 268 struct rte_fbarray *arr; 269 struct rte_memzone *found_mz; 270 int ret = 0; 271 void *addr = NULL; 272 unsigned idx; 273 274 if (mz == NULL) 275 return -EINVAL; 276 277 rte_strlcpy(name, mz->name, RTE_MEMZONE_NAMESIZE); 278 mcfg = rte_eal_get_configuration()->mem_config; 279 arr = &mcfg->memzones; 280 281 rte_rwlock_write_lock(&mcfg->mlock); 282 283 idx = rte_fbarray_find_idx(arr, mz); 284 found_mz = rte_fbarray_get(arr, idx); 285 286 if (found_mz == NULL) { 287 ret = -EINVAL; 288 } else if (found_mz->addr == NULL) { 289 RTE_LOG(ERR, EAL, "Memzone is not allocated\n"); 290 ret = -EINVAL; 291 } else { 292 addr = found_mz->addr; 293 memset(found_mz, 0, sizeof(*found_mz)); 294 rte_fbarray_set_free(arr, idx); 295 } 296 297 rte_rwlock_write_unlock(&mcfg->mlock); 298 299 rte_free(addr); 300 301 rte_eal_trace_memzone_free(name, addr, ret); 302 return ret; 303 } 304 305 /* 306 * Lookup for the memzone identified by the given name 307 */ 308 const struct rte_memzone * 309 rte_memzone_lookup(const char *name) 310 { 311 struct rte_mem_config *mcfg; 312 const struct rte_memzone *memzone = NULL; 313 314 mcfg = rte_eal_get_configuration()->mem_config; 315 316 rte_rwlock_read_lock(&mcfg->mlock); 317 318 memzone = memzone_lookup_thread_unsafe(name); 319 320 rte_rwlock_read_unlock(&mcfg->mlock); 321 322 rte_eal_trace_memzone_lookup(name, memzone); 323 return memzone; 324 } 325 326 static void 327 dump_memzone(const struct rte_memzone *mz, void *arg) 328 { 329 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 330 struct rte_memseg_list *msl = NULL; 331 void *cur_addr, *mz_end; 332 struct rte_memseg *ms; 333 int mz_idx, ms_idx; 334 size_t page_sz; 335 FILE *f = arg; 336 337 mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz); 338 339 fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, " 340 "socket_id:%"PRId32", flags:%"PRIx32"\n", 341 mz_idx, 342 mz->name, 343 mz->len, 344 mz->addr, 345 mz->socket_id, 346 mz->flags); 347 348 /* go through each page occupied by this memzone */ 349 msl = rte_mem_virt2memseg_list(mz->addr); 350 if (!msl) { 351 RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n"); 352 return; 353 } 354 page_sz = (size_t)mz->hugepage_sz; 355 cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz); 356 mz_end = RTE_PTR_ADD(cur_addr, mz->len); 357 358 fprintf(f, "physical segments used:\n"); 359 ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz; 360 ms = rte_fbarray_get(&msl->memseg_arr, ms_idx); 361 362 do { 363 fprintf(f, " addr: %p iova: 0x%" PRIx64 " " 364 "len: 0x%zx " 365 "pagesz: 0x%zx\n", 366 cur_addr, ms->iova, ms->len, page_sz); 367 368 /* advance VA to next page */ 369 cur_addr = RTE_PTR_ADD(cur_addr, page_sz); 370 371 /* memzones occupy contiguous segments */ 372 ++ms; 373 } while (cur_addr < mz_end); 374 } 375 376 /* Dump all reserved memory zones on console */ 377 void 378 rte_memzone_dump(FILE *f) 379 { 380 rte_memzone_walk(dump_memzone, f); 381 } 382 383 /* 384 * Init the memzone subsystem 385 */ 386 int 387 rte_eal_memzone_init(void) 388 { 389 struct rte_mem_config *mcfg; 390 int ret = 0; 391 392 /* get pointer to global configuration */ 393 mcfg = rte_eal_get_configuration()->mem_config; 394 395 rte_rwlock_write_lock(&mcfg->mlock); 396 397 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 398 rte_fbarray_init(&mcfg->memzones, "memzone", 399 RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) { 400 RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n"); 401 ret = -1; 402 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY && 403 rte_fbarray_attach(&mcfg->memzones)) { 404 RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n"); 405 ret = -1; 406 } 407 408 rte_rwlock_write_unlock(&mcfg->mlock); 409 410 return ret; 411 } 412 413 /* Walk all reserved memory zones */ 414 void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *), 415 void *arg) 416 { 417 struct rte_mem_config *mcfg; 418 struct rte_fbarray *arr; 419 int i; 420 421 mcfg = rte_eal_get_configuration()->mem_config; 422 arr = &mcfg->memzones; 423 424 rte_rwlock_read_lock(&mcfg->mlock); 425 i = rte_fbarray_find_next_used(arr, 0); 426 while (i >= 0) { 427 struct rte_memzone *mz = rte_fbarray_get(arr, i); 428 (*func)(mz, arg); 429 i = rte_fbarray_find_next_used(arr, i + 1); 430 } 431 rte_rwlock_read_unlock(&mcfg->mlock); 432 } 433