1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <errno.h> 6 #include <rte_malloc.h> 7 #include <malloc.h> 8 #include <stdbool.h> 9 #include <string.h> 10 11 #include "mlx5_common_utils.h" 12 #include "mlx5_malloc.h" 13 14 struct mlx5_sys_mem { 15 uint32_t init:1; /* Memory allocator initialized. */ 16 uint32_t enable:1; /* System memory select. */ 17 uint32_t reserve:30; /* Reserve. */ 18 struct rte_memseg_list *last_msl; 19 /* last allocated rte memory memseg list. */ 20 #ifdef RTE_LIBRTE_MLX5_DEBUG 21 uint64_t malloc_sys; 22 /* Memory allocated from system count. */ 23 uint64_t malloc_rte; 24 /* Memory allocated from hugepage count. */ 25 uint64_t realloc_sys; 26 /* Memory reallocate from system count. */ 27 uint64_t realloc_rte; 28 /* Memory reallocate from hugepage count. */ 29 uint64_t free_sys; 30 /* Memory free to system count. */ 31 uint64_t free_rte; 32 /* Memory free to hugepage count. */ 33 uint64_t msl_miss; 34 /* MSL miss count. */ 35 uint64_t msl_update; 36 /* MSL update count. */ 37 #endif 38 }; 39 40 /* Initialize default as not */ 41 static struct mlx5_sys_mem mlx5_sys_mem = { 42 .init = 0, 43 .enable = 0, 44 #ifdef RTE_LIBRTE_MLX5_DEBUG 45 .malloc_sys = 0, 46 .malloc_rte = 0, 47 .realloc_sys = 0, 48 .realloc_rte = 0, 49 .free_sys = 0, 50 .free_rte = 0, 51 .msl_miss = 0, 52 .msl_update = 0, 53 #endif 54 }; 55 56 /** 57 * Check if the address belongs to memory seg list. 58 * 59 * @param addr 60 * Memory address to be ckeced. 61 * @param msl 62 * Memory seg list. 63 * 64 * @return 65 * True if it belongs, false otherwise. 66 */ 67 static bool 68 mlx5_mem_check_msl(void *addr, struct rte_memseg_list *msl) 69 { 70 void *start, *end; 71 72 if (!msl) 73 return false; 74 start = msl->base_va; 75 end = RTE_PTR_ADD(start, msl->len); 76 if (addr >= start && addr < end) 77 return true; 78 return false; 79 } 80 81 /** 82 * Update the msl if memory belongs to new msl. 83 * 84 * @param addr 85 * Memory address. 86 */ 87 static void 88 mlx5_mem_update_msl(void *addr) 89 { 90 /* 91 * Update the cache msl if the new addr comes from the new msl 92 * different with the cached msl. 93 */ 94 if (addr && !mlx5_mem_check_msl(addr, 95 (struct rte_memseg_list *)__atomic_load_n 96 (&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) { 97 __atomic_store_n(&mlx5_sys_mem.last_msl, 98 rte_mem_virt2memseg_list(addr), 99 __ATOMIC_RELAXED); 100 #ifdef RTE_LIBRTE_MLX5_DEBUG 101 __atomic_add_fetch(&mlx5_sys_mem.msl_update, 1, 102 __ATOMIC_RELAXED); 103 #endif 104 } 105 } 106 107 /** 108 * Check if the address belongs to rte memory. 109 * 110 * @param addr 111 * Memory address to be ckeced. 112 * 113 * @return 114 * True if it belongs, false otherwise. 115 */ 116 static bool 117 mlx5_mem_is_rte(void *addr) 118 { 119 /* 120 * Check if the last cache msl matches. Drop to slow path 121 * to check if the memory belongs to rte memory. 122 */ 123 if (!mlx5_mem_check_msl(addr, (struct rte_memseg_list *) 124 __atomic_load_n(&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) { 125 if (!rte_mem_virt2memseg_list(addr)) 126 return false; 127 #ifdef RTE_LIBRTE_MLX5_DEBUG 128 __atomic_add_fetch(&mlx5_sys_mem.msl_miss, 1, __ATOMIC_RELAXED); 129 #endif 130 } 131 return true; 132 } 133 134 /** 135 * Allocate memory with alignment. 136 * 137 * @param size 138 * Memory size to be allocated. 139 * @param align 140 * Memory alignment. 141 * @param zero 142 * Clear the allocated memory or not. 143 * 144 * @return 145 * Pointer of the allocated memory, NULL otherwise. 146 */ 147 static void * 148 mlx5_alloc_align(size_t size, unsigned int align, unsigned int zero) 149 { 150 void *buf; 151 int ret; 152 153 ret = posix_memalign(&buf, align, size); 154 if (ret) { 155 DRV_LOG(ERR, 156 "Couldn't allocate buf size=%zu align=%u. Err=%d\n", 157 size, align, ret); 158 159 return NULL; 160 } 161 if (zero) 162 memset(buf, 0, size); 163 return buf; 164 } 165 166 void * 167 mlx5_malloc(uint32_t flags, size_t size, unsigned int align, int socket) 168 { 169 void *addr; 170 bool rte_mem; 171 172 /* 173 * If neither system memory nor rte memory is required, allocate 174 * memory according to mlx5_sys_mem.enable. 175 */ 176 if (flags & MLX5_MEM_RTE) 177 rte_mem = true; 178 else if (flags & MLX5_MEM_SYS) 179 rte_mem = false; 180 else 181 rte_mem = mlx5_sys_mem.enable ? false : true; 182 if (rte_mem) { 183 if (flags & MLX5_MEM_ZERO) 184 addr = rte_zmalloc_socket(NULL, size, align, socket); 185 else 186 addr = rte_malloc_socket(NULL, size, align, socket); 187 mlx5_mem_update_msl(addr); 188 #ifdef RTE_LIBRTE_MLX5_DEBUG 189 if (addr) 190 __atomic_add_fetch(&mlx5_sys_mem.malloc_rte, 1, 191 __ATOMIC_RELAXED); 192 #endif 193 return addr; 194 } 195 /* The memory will be allocated from system. */ 196 if (align > MLX5_MALLOC_ALIGNMENT) 197 addr = mlx5_alloc_align(size, align, !!(flags & MLX5_MEM_ZERO)); 198 else if (flags & MLX5_MEM_ZERO) 199 addr = calloc(1, size); 200 else 201 addr = malloc(size); 202 #ifdef RTE_LIBRTE_MLX5_DEBUG 203 if (addr) 204 __atomic_add_fetch(&mlx5_sys_mem.malloc_sys, 1, 205 __ATOMIC_RELAXED); 206 #endif 207 return addr; 208 } 209 210 void * 211 mlx5_realloc(void *addr, uint32_t flags, size_t size, unsigned int align, 212 int socket) 213 { 214 void *new_addr; 215 bool rte_mem; 216 217 /* Allocate directly if old memory address is NULL. */ 218 if (!addr) 219 return mlx5_malloc(flags, size, align, socket); 220 /* Get the memory type. */ 221 if (flags & MLX5_MEM_RTE) 222 rte_mem = true; 223 else if (flags & MLX5_MEM_SYS) 224 rte_mem = false; 225 else 226 rte_mem = mlx5_sys_mem.enable ? false : true; 227 /* Check if old memory and to be allocated memory are the same type. */ 228 if (rte_mem != mlx5_mem_is_rte(addr)) { 229 DRV_LOG(ERR, "Couldn't reallocate to different memory type."); 230 return NULL; 231 } 232 /* Allocate memory from rte memory. */ 233 if (rte_mem) { 234 new_addr = rte_realloc_socket(addr, size, align, socket); 235 mlx5_mem_update_msl(new_addr); 236 #ifdef RTE_LIBRTE_MLX5_DEBUG 237 if (new_addr) 238 __atomic_add_fetch(&mlx5_sys_mem.realloc_rte, 1, 239 __ATOMIC_RELAXED); 240 #endif 241 return new_addr; 242 } 243 /* Align is not supported for system memory. */ 244 if (align) { 245 DRV_LOG(ERR, "Couldn't reallocate with alignment"); 246 return NULL; 247 } 248 new_addr = realloc(addr, size); 249 #ifdef RTE_LIBRTE_MLX5_DEBUG 250 if (new_addr) 251 __atomic_add_fetch(&mlx5_sys_mem.realloc_sys, 1, 252 __ATOMIC_RELAXED); 253 #endif 254 return new_addr; 255 } 256 257 void 258 mlx5_free(void *addr) 259 { 260 if (addr == NULL) 261 return; 262 if (!mlx5_mem_is_rte(addr)) { 263 #ifdef RTE_LIBRTE_MLX5_DEBUG 264 __atomic_add_fetch(&mlx5_sys_mem.free_sys, 1, 265 __ATOMIC_RELAXED); 266 #endif 267 free(addr); 268 } else { 269 #ifdef RTE_LIBRTE_MLX5_DEBUG 270 __atomic_add_fetch(&mlx5_sys_mem.free_rte, 1, 271 __ATOMIC_RELAXED); 272 #endif 273 rte_free(addr); 274 } 275 } 276 277 void 278 mlx5_memory_stat_dump(void) 279 { 280 #ifdef RTE_LIBRTE_MLX5_DEBUG 281 DRV_LOG(INFO, "System memory malloc:%"PRIi64", realloc:%"PRIi64"," 282 " free:%"PRIi64"\nRTE memory malloc:%"PRIi64"," 283 " realloc:%"PRIi64", free:%"PRIi64"\nMSL miss:%"PRIi64"," 284 " update:%"PRIi64"", 285 __atomic_load_n(&mlx5_sys_mem.malloc_sys, __ATOMIC_RELAXED), 286 __atomic_load_n(&mlx5_sys_mem.realloc_sys, __ATOMIC_RELAXED), 287 __atomic_load_n(&mlx5_sys_mem.free_sys, __ATOMIC_RELAXED), 288 __atomic_load_n(&mlx5_sys_mem.malloc_rte, __ATOMIC_RELAXED), 289 __atomic_load_n(&mlx5_sys_mem.realloc_rte, __ATOMIC_RELAXED), 290 __atomic_load_n(&mlx5_sys_mem.free_rte, __ATOMIC_RELAXED), 291 __atomic_load_n(&mlx5_sys_mem.msl_miss, __ATOMIC_RELAXED), 292 __atomic_load_n(&mlx5_sys_mem.msl_update, __ATOMIC_RELAXED)); 293 #endif 294 } 295 296 void 297 mlx5_malloc_mem_select(uint32_t sys_mem_en) 298 { 299 /* 300 * The initialization should be called only once and all devices 301 * should use the same memory type. Otherwise, when new device is 302 * being attached with some different memory allocation configuration, 303 * the memory will get wrong behavior or a failure will be raised. 304 */ 305 if (!mlx5_sys_mem.init) { 306 if (sys_mem_en) 307 mlx5_sys_mem.enable = 1; 308 mlx5_sys_mem.init = 1; 309 DRV_LOG(INFO, "%s is selected.", sys_mem_en ? "SYS_MEM" : "RTE_MEM"); 310 } else if (mlx5_sys_mem.enable != sys_mem_en) { 311 DRV_LOG(WARNING, "%s is already selected.", 312 mlx5_sys_mem.enable ? "SYS_MEM" : "RTE_MEM"); 313 } 314 } 315