1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <inttypes.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <eal_trace_internal.h> 12 #include <rte_log.h> 13 #include <rte_memory.h> 14 #include <rte_memzone.h> 15 #include <rte_eal.h> 16 #include <rte_errno.h> 17 #include <rte_string_fns.h> 18 #include <rte_common.h> 19 20 #include "malloc_heap.h" 21 #include "malloc_elem.h" 22 #include "eal_private.h" 23 #include "eal_memcfg.h" 24 25 /* Default count used until rte_memzone_max_set() is called */ 26 #define DEFAULT_MAX_MEMZONE_COUNT 2560 27 28 int 29 rte_memzone_max_set(size_t max) 30 { 31 struct rte_mem_config *mcfg; 32 33 if (eal_get_internal_configuration()->init_complete > 0) { 34 EAL_LOG(ERR, "Max memzone cannot be set after EAL init"); 35 return -1; 36 } 37 38 mcfg = rte_eal_get_configuration()->mem_config; 39 if (mcfg == NULL) { 40 EAL_LOG(ERR, "Failed to set max memzone count"); 41 return -1; 42 } 43 44 mcfg->max_memzone = max; 45 46 return 0; 47 } 48 49 size_t 50 rte_memzone_max_get(void) 51 { 52 struct rte_mem_config *mcfg; 53 54 mcfg = rte_eal_get_configuration()->mem_config; 55 if (mcfg == NULL || mcfg->max_memzone == 0) 56 return DEFAULT_MAX_MEMZONE_COUNT; 57 58 return mcfg->max_memzone; 59 } 60 61 static inline const struct rte_memzone * 62 memzone_lookup_thread_unsafe(const char *name) 63 { 64 struct rte_mem_config *mcfg; 65 struct rte_fbarray *arr; 66 const struct rte_memzone *mz; 67 int i = 0; 68 69 /* get pointer to global configuration */ 70 mcfg = rte_eal_get_configuration()->mem_config; 71 arr = &mcfg->memzones; 72 73 /* 74 * the algorithm is not optimal (linear), but there are few 75 * zones and this function should be called at init only 76 */ 77 i = rte_fbarray_find_next_used(arr, 0); 78 while (i >= 0) { 79 mz = rte_fbarray_get(arr, i); 80 if (mz->addr != NULL && 81 !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE)) 82 return mz; 83 i = rte_fbarray_find_next_used(arr, i + 1); 84 } 85 return NULL; 86 } 87 88 #define MEMZONE_KNOWN_FLAGS (RTE_MEMZONE_2MB \ 89 | RTE_MEMZONE_1GB \ 90 | RTE_MEMZONE_16MB \ 91 | RTE_MEMZONE_16GB \ 92 | RTE_MEMZONE_256KB \ 93 | RTE_MEMZONE_256MB \ 94 | RTE_MEMZONE_512MB \ 95 | RTE_MEMZONE_4GB \ 96 | RTE_MEMZONE_SIZE_HINT_ONLY \ 97 | RTE_MEMZONE_IOVA_CONTIG \ 98 ) 99 100 static const struct rte_memzone * 101 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, 102 int socket_id, unsigned int flags, unsigned int align, 103 unsigned int bound) 104 { 105 struct rte_memzone *mz; 106 struct rte_mem_config *mcfg; 107 struct rte_fbarray *arr; 108 void *mz_addr; 109 size_t requested_len; 110 int mz_idx; 111 bool contig; 112 113 /* get pointer to global configuration */ 114 mcfg = rte_eal_get_configuration()->mem_config; 115 arr = &mcfg->memzones; 116 117 /* no more room in config */ 118 if (arr->count >= arr->len) { 119 EAL_LOG(ERR, 120 "%s(): Number of requested memzone segments exceeds maximum " 121 "%u", __func__, arr->len); 122 123 rte_errno = ENOSPC; 124 return NULL; 125 } 126 127 if (strlen(name) > sizeof(mz->name) - 1) { 128 EAL_LOG(DEBUG, "%s(): memzone <%s>: name too long", 129 __func__, name); 130 rte_errno = ENAMETOOLONG; 131 return NULL; 132 } 133 134 /* zone already exist */ 135 if ((memzone_lookup_thread_unsafe(name)) != NULL) { 136 EAL_LOG(DEBUG, "%s(): memzone <%s> already exists", 137 __func__, name); 138 rte_errno = EEXIST; 139 return NULL; 140 } 141 142 /* if alignment is not a power of two */ 143 if (align && !rte_is_power_of_2(align)) { 144 EAL_LOG(ERR, "%s(): Invalid alignment: %u", __func__, 145 align); 146 rte_errno = EINVAL; 147 return NULL; 148 } 149 150 /* alignment less than cache size is not allowed */ 151 if (align < RTE_CACHE_LINE_SIZE) 152 align = RTE_CACHE_LINE_SIZE; 153 154 /* align length on cache boundary. Check for overflow before doing so */ 155 if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) { 156 rte_errno = EINVAL; /* requested size too big */ 157 return NULL; 158 } 159 160 len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE); 161 162 /* save minimal requested length */ 163 requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len); 164 165 /* check that boundary condition is valid */ 166 if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) { 167 rte_errno = EINVAL; 168 return NULL; 169 } 170 171 if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) { 172 rte_errno = EINVAL; 173 return NULL; 174 } 175 176 if ((flags & ~MEMZONE_KNOWN_FLAGS) != 0) { 177 rte_errno = EINVAL; 178 return NULL; 179 } 180 181 /* only set socket to SOCKET_ID_ANY if we aren't allocating for an 182 * external heap. 183 */ 184 if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES) 185 socket_id = SOCKET_ID_ANY; 186 187 contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0; 188 /* malloc only cares about size flags, remove contig flag from flags */ 189 flags &= ~RTE_MEMZONE_IOVA_CONTIG; 190 191 if (len == 0 && bound == 0) { 192 /* no size constraints were placed, so use malloc elem len */ 193 requested_len = 0; 194 mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags, 195 align, contig); 196 } else { 197 if (len == 0) 198 requested_len = bound; 199 /* allocate memory on heap */ 200 mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, 201 flags, align, bound, contig); 202 } 203 if (mz_addr == NULL) { 204 rte_errno = ENOMEM; 205 return NULL; 206 } 207 208 struct malloc_elem *elem = malloc_elem_from_data(mz_addr); 209 210 /* fill the zone in config */ 211 mz_idx = rte_fbarray_find_next_free(arr, 0); 212 213 if (mz_idx < 0) { 214 mz = NULL; 215 } else { 216 rte_fbarray_set_used(arr, mz_idx); 217 mz = rte_fbarray_get(arr, mz_idx); 218 } 219 220 if (mz == NULL) { 221 EAL_LOG(ERR, "%s(): Cannot find free memzone", __func__); 222 malloc_heap_free(elem); 223 rte_errno = ENOSPC; 224 return NULL; 225 } 226 227 strlcpy(mz->name, name, sizeof(mz->name)); 228 mz->iova = rte_malloc_virt2iova(mz_addr); 229 mz->addr = mz_addr; 230 mz->len = requested_len == 0 ? 231 elem->size - elem->pad - MALLOC_ELEM_OVERHEAD : 232 requested_len; 233 mz->hugepage_sz = elem->msl->page_sz; 234 mz->socket_id = elem->msl->socket_id; 235 mz->flags = 0; 236 237 return mz; 238 } 239 240 static const struct rte_memzone * 241 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id, 242 unsigned int flags, unsigned int align, unsigned int bound) 243 { 244 struct rte_mem_config *mcfg; 245 const struct rte_memzone *mz = NULL; 246 247 /* get pointer to global configuration */ 248 mcfg = rte_eal_get_configuration()->mem_config; 249 250 rte_rwlock_write_lock(&mcfg->mlock); 251 252 mz = memzone_reserve_aligned_thread_unsafe( 253 name, len, socket_id, flags, align, bound); 254 255 rte_eal_trace_memzone_reserve(name, len, socket_id, flags, align, 256 bound, mz); 257 258 rte_rwlock_write_unlock(&mcfg->mlock); 259 260 return mz; 261 } 262 263 /* 264 * Return a pointer to a correctly filled memzone descriptor (with a 265 * specified alignment and boundary). If the allocation cannot be done, 266 * return NULL. 267 */ 268 const struct rte_memzone * 269 rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id, 270 unsigned flags, unsigned align, unsigned bound) 271 { 272 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, 273 align, bound); 274 } 275 276 /* 277 * Return a pointer to a correctly filled memzone descriptor (with a 278 * specified alignment). If the allocation cannot be done, return NULL. 279 */ 280 const struct rte_memzone * 281 rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 282 unsigned flags, unsigned align) 283 { 284 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, 285 align, 0); 286 } 287 288 /* 289 * Return a pointer to a correctly filled memzone descriptor. If the 290 * allocation cannot be done, return NULL. 291 */ 292 const struct rte_memzone * 293 rte_memzone_reserve(const char *name, size_t len, int socket_id, 294 unsigned flags) 295 { 296 return rte_memzone_reserve_thread_safe(name, len, socket_id, 297 flags, RTE_CACHE_LINE_SIZE, 0); 298 } 299 300 int 301 rte_memzone_free(const struct rte_memzone *mz) 302 { 303 char name[RTE_MEMZONE_NAMESIZE]; 304 struct rte_mem_config *mcfg; 305 struct rte_fbarray *arr; 306 struct rte_memzone *found_mz; 307 int ret = 0; 308 void *addr = NULL; 309 unsigned idx; 310 311 if (mz == NULL) 312 return -EINVAL; 313 314 rte_strlcpy(name, mz->name, RTE_MEMZONE_NAMESIZE); 315 mcfg = rte_eal_get_configuration()->mem_config; 316 arr = &mcfg->memzones; 317 318 rte_rwlock_write_lock(&mcfg->mlock); 319 320 idx = rte_fbarray_find_idx(arr, mz); 321 found_mz = rte_fbarray_get(arr, idx); 322 323 if (found_mz == NULL) { 324 ret = -EINVAL; 325 } else if (found_mz->addr == NULL) { 326 EAL_LOG(ERR, "Memzone is not allocated"); 327 ret = -EINVAL; 328 } else { 329 addr = found_mz->addr; 330 memset(found_mz, 0, sizeof(*found_mz)); 331 rte_fbarray_set_free(arr, idx); 332 } 333 334 rte_rwlock_write_unlock(&mcfg->mlock); 335 336 rte_free(addr); 337 338 rte_eal_trace_memzone_free(name, addr, ret); 339 return ret; 340 } 341 342 /* 343 * Lookup for the memzone identified by the given name 344 */ 345 const struct rte_memzone * 346 rte_memzone_lookup(const char *name) 347 { 348 struct rte_mem_config *mcfg; 349 const struct rte_memzone *memzone = NULL; 350 351 mcfg = rte_eal_get_configuration()->mem_config; 352 353 rte_rwlock_read_lock(&mcfg->mlock); 354 355 memzone = memzone_lookup_thread_unsafe(name); 356 357 rte_rwlock_read_unlock(&mcfg->mlock); 358 359 rte_eal_trace_memzone_lookup(name, memzone); 360 return memzone; 361 } 362 363 static void 364 dump_memzone(const struct rte_memzone *mz, void *arg) 365 { 366 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 367 struct rte_memseg_list *msl = NULL; 368 void *cur_addr, *mz_end; 369 struct rte_memseg *ms; 370 int mz_idx, ms_idx; 371 size_t page_sz; 372 FILE *f = arg; 373 374 mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz); 375 376 fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, " 377 "socket_id:%"PRId32", flags:%"PRIx32"\n", 378 mz_idx, 379 mz->name, 380 mz->len, 381 mz->addr, 382 mz->socket_id, 383 mz->flags); 384 385 /* go through each page occupied by this memzone */ 386 msl = rte_mem_virt2memseg_list(mz->addr); 387 if (!msl) { 388 EAL_LOG(DEBUG, "Skipping bad memzone"); 389 return; 390 } 391 page_sz = (size_t)mz->hugepage_sz; 392 cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz); 393 mz_end = RTE_PTR_ADD(cur_addr, mz->len); 394 395 fprintf(f, "physical segments used:\n"); 396 ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz; 397 ms = rte_fbarray_get(&msl->memseg_arr, ms_idx); 398 399 do { 400 fprintf(f, " addr: %p iova: 0x%" PRIx64 " " 401 "len: 0x%zx " 402 "pagesz: 0x%zx\n", 403 cur_addr, ms->iova, ms->len, page_sz); 404 405 /* advance VA to next page */ 406 cur_addr = RTE_PTR_ADD(cur_addr, page_sz); 407 408 /* memzones occupy contiguous segments */ 409 ++ms; 410 } while (cur_addr < mz_end); 411 } 412 413 /* Dump all reserved memory zones on console */ 414 void 415 rte_memzone_dump(FILE *f) 416 { 417 rte_memzone_walk(dump_memzone, f); 418 } 419 420 /* 421 * Init the memzone subsystem 422 */ 423 int 424 rte_eal_memzone_init(void) 425 { 426 struct rte_mem_config *mcfg; 427 int ret = 0; 428 429 /* get pointer to global configuration */ 430 mcfg = rte_eal_get_configuration()->mem_config; 431 432 rte_rwlock_write_lock(&mcfg->mlock); 433 434 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 435 rte_fbarray_init(&mcfg->memzones, "memzone", 436 rte_memzone_max_get(), sizeof(struct rte_memzone))) { 437 EAL_LOG(ERR, "Cannot allocate memzone list"); 438 ret = -1; 439 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY && 440 rte_fbarray_attach(&mcfg->memzones)) { 441 EAL_LOG(ERR, "Cannot attach to memzone list"); 442 ret = -1; 443 } 444 445 rte_rwlock_write_unlock(&mcfg->mlock); 446 447 return ret; 448 } 449 450 /* Walk all reserved memory zones */ 451 void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *), 452 void *arg) 453 { 454 struct rte_mem_config *mcfg; 455 struct rte_fbarray *arr; 456 int i; 457 458 mcfg = rte_eal_get_configuration()->mem_config; 459 arr = &mcfg->memzones; 460 461 rte_rwlock_read_lock(&mcfg->mlock); 462 i = rte_fbarray_find_next_used(arr, 0); 463 while (i >= 0) { 464 struct rte_memzone *mz = rte_fbarray_get(arr, i); 465 (*func)(mz, arg); 466 i = rte_fbarray_find_next_used(arr, i + 1); 467 } 468 rte_rwlock_read_unlock(&mcfg->mlock); 469 } 470