1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <errno.h> 6 #include <rte_malloc.h> 7 #include <malloc.h> 8 #include <stdbool.h> 9 #include <string.h> 10 11 #include <rte_atomic.h> 12 13 #include "mlx5_common_utils.h" 14 #include "mlx5_malloc.h" 15 16 struct mlx5_sys_mem { 17 uint32_t init:1; /* Memory allocator initialized. */ 18 uint32_t enable:1; /* System memory select. */ 19 uint32_t reserve:30; /* Reserve. */ 20 union { 21 struct rte_memseg_list *last_msl; 22 rte_atomic64_t a64_last_msl; 23 }; 24 /* last allocated rte memory memseg list. */ 25 #ifdef RTE_LIBRTE_MLX5_DEBUG 26 rte_atomic64_t malloc_sys; 27 /* Memory allocated from system count. */ 28 rte_atomic64_t malloc_rte; 29 /* Memory allocated from hugepage count. */ 30 rte_atomic64_t realloc_sys; 31 /* Memory reallocate from system count. */ 32 rte_atomic64_t realloc_rte; 33 /* Memory reallocate from hugepage count. */ 34 rte_atomic64_t free_sys; 35 /* Memory free to system count. */ 36 rte_atomic64_t free_rte; 37 /* Memory free to hugepage count. */ 38 rte_atomic64_t msl_miss; 39 /* MSL miss count. */ 40 rte_atomic64_t msl_update; 41 /* MSL update count. */ 42 #endif 43 }; 44 45 /* Initialize default as not */ 46 static struct mlx5_sys_mem mlx5_sys_mem = { 47 .init = 0, 48 .enable = 0, 49 #ifdef RTE_LIBRTE_MLX5_DEBUG 50 .malloc_sys = RTE_ATOMIC64_INIT(0), 51 .malloc_rte = RTE_ATOMIC64_INIT(0), 52 .realloc_sys = RTE_ATOMIC64_INIT(0), 53 .realloc_rte = RTE_ATOMIC64_INIT(0), 54 .free_sys = RTE_ATOMIC64_INIT(0), 55 .free_rte = RTE_ATOMIC64_INIT(0), 56 .msl_miss = RTE_ATOMIC64_INIT(0), 57 .msl_update = RTE_ATOMIC64_INIT(0), 58 #endif 59 }; 60 61 /** 62 * Check if the address belongs to memory seg list. 63 * 64 * @param addr 65 * Memory address to be ckeced. 66 * @param msl 67 * Memory seg list. 68 * 69 * @return 70 * True if it belongs, false otherwise. 71 */ 72 static bool 73 mlx5_mem_check_msl(void *addr, struct rte_memseg_list *msl) 74 { 75 void *start, *end; 76 77 if (!msl) 78 return false; 79 start = msl->base_va; 80 end = RTE_PTR_ADD(start, msl->len); 81 if (addr >= start && addr < end) 82 return true; 83 return false; 84 } 85 86 /** 87 * Update the msl if memory belongs to new msl. 88 * 89 * @param addr 90 * Memory address. 91 */ 92 static void 93 mlx5_mem_update_msl(void *addr) 94 { 95 /* 96 * Update the cache msl if the new addr comes from the new msl 97 * different with the cached msl. 98 */ 99 if (addr && !mlx5_mem_check_msl(addr, 100 (struct rte_memseg_list *)(uintptr_t)rte_atomic64_read 101 (&mlx5_sys_mem.a64_last_msl))) { 102 rte_atomic64_set(&mlx5_sys_mem.a64_last_msl, 103 (int64_t)(uintptr_t)rte_mem_virt2memseg_list(addr)); 104 #ifdef RTE_LIBRTE_MLX5_DEBUG 105 rte_atomic64_inc(&mlx5_sys_mem.msl_update); 106 #endif 107 } 108 } 109 110 /** 111 * Check if the address belongs to rte memory. 112 * 113 * @param addr 114 * Memory address to be ckeced. 115 * 116 * @return 117 * True if it belongs, false otherwise. 118 */ 119 static bool 120 mlx5_mem_is_rte(void *addr) 121 { 122 /* 123 * Check if the last cache msl matches. Drop to slow path 124 * to check if the memory belongs to rte memory. 125 */ 126 if (!mlx5_mem_check_msl(addr, (struct rte_memseg_list *)(uintptr_t) 127 rte_atomic64_read(&mlx5_sys_mem.a64_last_msl))) { 128 if (!rte_mem_virt2memseg_list(addr)) 129 return false; 130 #ifdef RTE_LIBRTE_MLX5_DEBUG 131 rte_atomic64_inc(&mlx5_sys_mem.msl_miss); 132 #endif 133 } 134 return true; 135 } 136 137 /** 138 * Allocate memory with alignment. 139 * 140 * @param size 141 * Memory size to be allocated. 142 * @param align 143 * Memory alignment. 144 * @param zero 145 * Clear the allocated memory or not. 146 * 147 * @return 148 * Pointer of the allocated memory, NULL otherwise. 149 */ 150 static void * 151 mlx5_alloc_align(size_t size, unsigned int align, unsigned int zero) 152 { 153 void *buf; 154 buf = memalign(align, size); 155 if (!buf) { 156 DRV_LOG(ERR, "Couldn't allocate buf.\n"); 157 return NULL; 158 } 159 if (zero) 160 memset(buf, 0, size); 161 return buf; 162 } 163 164 void * 165 mlx5_malloc(uint32_t flags, size_t size, unsigned int align, int socket) 166 { 167 void *addr; 168 bool rte_mem; 169 170 /* 171 * If neither system memory nor rte memory is required, allocate 172 * memory according to mlx5_sys_mem.enable. 173 */ 174 if (flags & MLX5_MEM_RTE) 175 rte_mem = true; 176 else if (flags & MLX5_MEM_SYS) 177 rte_mem = false; 178 else 179 rte_mem = mlx5_sys_mem.enable ? false : true; 180 if (rte_mem) { 181 if (flags & MLX5_MEM_ZERO) 182 addr = rte_zmalloc_socket(NULL, size, align, socket); 183 else 184 addr = rte_malloc_socket(NULL, size, align, socket); 185 mlx5_mem_update_msl(addr); 186 #ifdef RTE_LIBRTE_MLX5_DEBUG 187 if (addr) 188 rte_atomic64_inc(&mlx5_sys_mem.malloc_rte); 189 #endif 190 return addr; 191 } 192 /* The memory will be allocated from system. */ 193 if (align) 194 addr = mlx5_alloc_align(size, align, !!(flags & MLX5_MEM_ZERO)); 195 else if (flags & MLX5_MEM_ZERO) 196 addr = calloc(1, size); 197 else 198 addr = malloc(size); 199 #ifdef RTE_LIBRTE_MLX5_DEBUG 200 if (addr) 201 rte_atomic64_inc(&mlx5_sys_mem.malloc_sys); 202 #endif 203 return addr; 204 } 205 206 void * 207 mlx5_realloc(void *addr, uint32_t flags, size_t size, unsigned int align, 208 int socket) 209 { 210 void *new_addr; 211 bool rte_mem; 212 213 /* Allocate directly if old memory address is NULL. */ 214 if (!addr) 215 return mlx5_malloc(flags, size, align, socket); 216 /* Get the memory type. */ 217 if (flags & MLX5_MEM_RTE) 218 rte_mem = true; 219 else if (flags & MLX5_MEM_SYS) 220 rte_mem = false; 221 else 222 rte_mem = mlx5_sys_mem.enable ? false : true; 223 /* Check if old memory and to be allocated memory are the same type. */ 224 if (rte_mem != mlx5_mem_is_rte(addr)) { 225 DRV_LOG(ERR, "Couldn't reallocate to different memory type."); 226 return NULL; 227 } 228 /* Allocate memory from rte memory. */ 229 if (rte_mem) { 230 new_addr = rte_realloc_socket(addr, size, align, socket); 231 mlx5_mem_update_msl(new_addr); 232 #ifdef RTE_LIBRTE_MLX5_DEBUG 233 if (new_addr) 234 rte_atomic64_inc(&mlx5_sys_mem.realloc_rte); 235 #endif 236 return new_addr; 237 } 238 /* Align is not supported for system memory. */ 239 if (align) { 240 DRV_LOG(ERR, "Couldn't reallocate with alignment"); 241 return NULL; 242 } 243 new_addr = realloc(addr, size); 244 #ifdef RTE_LIBRTE_MLX5_DEBUG 245 if (new_addr) 246 rte_atomic64_inc(&mlx5_sys_mem.realloc_sys); 247 #endif 248 return new_addr; 249 } 250 251 void 252 mlx5_free(void *addr) 253 { 254 if (addr == NULL) 255 return; 256 if (!mlx5_mem_is_rte(addr)) { 257 #ifdef RTE_LIBRTE_MLX5_DEBUG 258 rte_atomic64_inc(&mlx5_sys_mem.free_sys); 259 #endif 260 free(addr); 261 } else { 262 #ifdef RTE_LIBRTE_MLX5_DEBUG 263 rte_atomic64_inc(&mlx5_sys_mem.free_rte); 264 #endif 265 rte_free(addr); 266 } 267 } 268 269 void 270 mlx5_memory_stat_dump(void) 271 { 272 #ifdef RTE_LIBRTE_MLX5_DEBUG 273 DRV_LOG(INFO, "System memory malloc:%"PRIi64", realloc:%"PRIi64"," 274 " free:%"PRIi64"\nRTE memory malloc:%"PRIi64"," 275 " realloc:%"PRIi64", free:%"PRIi64"\nMSL miss:%"PRIi64"," 276 " update:%"PRIi64"", 277 rte_atomic64_read(&mlx5_sys_mem.malloc_sys), 278 rte_atomic64_read(&mlx5_sys_mem.realloc_sys), 279 rte_atomic64_read(&mlx5_sys_mem.free_sys), 280 rte_atomic64_read(&mlx5_sys_mem.malloc_rte), 281 rte_atomic64_read(&mlx5_sys_mem.realloc_rte), 282 rte_atomic64_read(&mlx5_sys_mem.free_rte), 283 rte_atomic64_read(&mlx5_sys_mem.msl_miss), 284 rte_atomic64_read(&mlx5_sys_mem.msl_update)); 285 #endif 286 } 287 288 void 289 mlx5_malloc_mem_select(uint32_t sys_mem_en) 290 { 291 /* 292 * The initialization should be called only once and all devices 293 * should use the same memory type. Otherwise, when new device is 294 * being attached with some different memory allocation configuration, 295 * the memory will get wrong behavior or a failure will be raised. 296 */ 297 if (!mlx5_sys_mem.init) { 298 if (sys_mem_en) 299 mlx5_sys_mem.enable = 1; 300 mlx5_sys_mem.init = 1; 301 DRV_LOG(INFO, "%s is selected.", sys_mem_en ? "SYS_MEM" : "RTE_MEM"); 302 } else if (mlx5_sys_mem.enable != sys_mem_en) { 303 DRV_LOG(WARNING, "%s is already selected.", 304 mlx5_sys_mem.enable ? "SYS_MEM" : "RTE_MEM"); 305 } 306 } 307