1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef MALLOC_ELEM_H_ 6 #define MALLOC_ELEM_H_ 7 8 #include <stdbool.h> 9 10 #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE) 11 12 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */ 13 struct malloc_heap; 14 15 enum elem_state { 16 ELEM_FREE = 0, 17 ELEM_BUSY, 18 ELEM_PAD /* element is a padding-only header */ 19 }; 20 21 struct malloc_elem { 22 struct malloc_heap *heap; 23 struct malloc_elem *volatile prev; 24 /**< points to prev elem in memseg */ 25 struct malloc_elem *volatile next; 26 /**< points to next elem in memseg */ 27 LIST_ENTRY(malloc_elem) free_list; 28 /**< list of free elements in heap */ 29 struct rte_memseg_list *msl; 30 volatile enum elem_state state; 31 uint32_t pad; 32 size_t size; 33 struct malloc_elem *orig_elem; 34 size_t orig_size; 35 #ifdef RTE_MALLOC_DEBUG 36 uint64_t header_cookie; /* Cookie marking start of data */ 37 /* trailer cookie at start + size */ 38 #endif 39 #ifdef RTE_MALLOC_ASAN 40 size_t user_size; 41 uint64_t asan_cookie[2]; /* must be next to header_cookie */ 42 #endif 43 } __rte_cache_aligned; 44 45 static const unsigned int MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem); 46 47 #ifndef RTE_MALLOC_DEBUG 48 #ifdef RTE_MALLOC_ASAN 49 static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE; 50 #else 51 static const unsigned int MALLOC_ELEM_TRAILER_LEN; 52 #endif 53 54 /* dummy function - just check if pointer is non-null */ 55 static inline int 56 malloc_elem_cookies_ok(const struct malloc_elem *elem){ return elem != NULL; } 57 58 /* dummy function - no header if malloc_debug is not enabled */ 59 static inline void 60 set_header(struct malloc_elem *elem __rte_unused){ } 61 62 /* dummy function - no trailer if malloc_debug is not enabled */ 63 static inline void 64 set_trailer(struct malloc_elem *elem __rte_unused){ } 65 66 67 #else 68 static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE; 69 70 #define MALLOC_HEADER_COOKIE 0xbadbadbadadd2e55ULL /**< Header cookie. */ 71 #define MALLOC_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/ 72 73 /* define macros to make referencing the header and trailer cookies easier */ 74 #define MALLOC_ELEM_TRAILER(elem) (*((uint64_t*)RTE_PTR_ADD(elem, \ 75 elem->size - MALLOC_ELEM_TRAILER_LEN))) 76 #define MALLOC_ELEM_HEADER(elem) (elem->header_cookie) 77 78 static inline void 79 set_header(struct malloc_elem *elem) 80 { 81 if (elem != NULL) 82 MALLOC_ELEM_HEADER(elem) = MALLOC_HEADER_COOKIE; 83 } 84 85 static inline void 86 set_trailer(struct malloc_elem *elem) 87 { 88 if (elem != NULL) 89 MALLOC_ELEM_TRAILER(elem) = MALLOC_TRAILER_COOKIE; 90 } 91 92 /* check that the header and trailer cookies are set correctly */ 93 static inline int 94 malloc_elem_cookies_ok(const struct malloc_elem *elem) 95 { 96 return elem != NULL && 97 MALLOC_ELEM_HEADER(elem) == MALLOC_HEADER_COOKIE && 98 MALLOC_ELEM_TRAILER(elem) == MALLOC_TRAILER_COOKIE; 99 } 100 101 #endif 102 103 #define MALLOC_ELEM_OVERHEAD (MALLOC_ELEM_HEADER_LEN + MALLOC_ELEM_TRAILER_LEN) 104 105 #ifdef RTE_MALLOC_ASAN 106 107 /* 108 * ASAN_SHADOW_OFFSET should match to the corresponding 109 * value defined in gcc/libsanitizer/asan/asan_mapping.h 110 */ 111 #ifdef RTE_ARCH_X86_64 112 #define ASAN_SHADOW_OFFSET 0x00007fff8000 113 #elif defined(RTE_ARCH_ARM64) 114 #define ASAN_SHADOW_OFFSET 0x001000000000 115 #elif defined(RTE_ARCH_PPC_64) 116 #define ASAN_SHADOW_OFFSET 0x020000000000 117 #endif 118 119 #define ASAN_SHADOW_GRAIN_SIZE 8 120 #define ASAN_MEM_FREE_FLAG 0xfd 121 #define ASAN_MEM_REDZONE_FLAG 0xfa 122 #define ASAN_SHADOW_SCALE 3 123 124 #define ASAN_MEM_SHIFT(mem) ((void *)((uintptr_t)(mem) >> ASAN_SHADOW_SCALE)) 125 #define ASAN_MEM_TO_SHADOW(mem) \ 126 RTE_PTR_ADD(ASAN_MEM_SHIFT(mem), ASAN_SHADOW_OFFSET) 127 128 #if defined(__clang__) 129 #define __rte_no_asan __attribute__((no_sanitize("address", "hwaddress"))) 130 #else 131 #define __rte_no_asan __attribute__((no_sanitize_address)) 132 #endif 133 134 __rte_no_asan 135 static inline void 136 asan_set_shadow(void *addr, char val) 137 { 138 *(char *)addr = val; 139 } 140 141 static inline void 142 asan_set_zone(void *ptr, size_t len, uint32_t val) 143 { 144 size_t offset, i; 145 void *shadow; 146 size_t zone_len = len / ASAN_SHADOW_GRAIN_SIZE; 147 if (len % ASAN_SHADOW_GRAIN_SIZE != 0) 148 zone_len += 1; 149 150 for (i = 0; i < zone_len; i++) { 151 offset = i * ASAN_SHADOW_GRAIN_SIZE; 152 shadow = ASAN_MEM_TO_SHADOW((uintptr_t)ptr + offset); 153 asan_set_shadow(shadow, val); 154 } 155 } 156 157 /* 158 * When the memory is released, the release mark is 159 * set in the corresponding range of the shadow area. 160 */ 161 static inline void 162 asan_set_freezone(void *ptr, size_t size) 163 { 164 asan_set_zone(ptr, size, ASAN_MEM_FREE_FLAG); 165 } 166 167 /* 168 * When the memory is allocated, memory state must set as accessible. 169 */ 170 static inline void 171 asan_clear_alloczone(struct malloc_elem *elem) 172 { 173 asan_set_zone((void *)elem, elem->size, 0x0); 174 } 175 176 static inline void 177 asan_clear_split_alloczone(struct malloc_elem *elem) 178 { 179 void *ptr = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN); 180 asan_set_zone(ptr, MALLOC_ELEM_OVERHEAD, 0x0); 181 } 182 183 /* 184 * When the memory is allocated, the memory boundary is 185 * marked in the corresponding range of the shadow area. 186 * Requirement: redzone >= 16, is a power of two. 187 */ 188 static inline void 189 asan_set_redzone(struct malloc_elem *elem, size_t user_size) 190 { 191 uintptr_t head_redzone; 192 uintptr_t tail_redzone; 193 void *front_shadow; 194 void *tail_shadow; 195 uint32_t val; 196 197 if (elem != NULL) { 198 if (elem->state != ELEM_PAD) 199 elem = RTE_PTR_ADD(elem, elem->pad); 200 201 elem->user_size = user_size; 202 203 /* Set mark before the start of the allocated memory */ 204 head_redzone = (uintptr_t)RTE_PTR_ADD(elem, 205 MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE); 206 front_shadow = ASAN_MEM_TO_SHADOW(head_redzone); 207 asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG); 208 front_shadow = ASAN_MEM_TO_SHADOW(head_redzone 209 - ASAN_SHADOW_GRAIN_SIZE); 210 asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG); 211 212 /* Set mark after the end of the allocated memory */ 213 tail_redzone = (uintptr_t)RTE_PTR_ADD(elem, 214 MALLOC_ELEM_HEADER_LEN 215 + elem->user_size); 216 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone); 217 val = (tail_redzone % ASAN_SHADOW_GRAIN_SIZE); 218 val = (val == 0) ? ASAN_MEM_REDZONE_FLAG : val; 219 asan_set_shadow(tail_shadow, val); 220 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone 221 + ASAN_SHADOW_GRAIN_SIZE); 222 asan_set_shadow(tail_shadow, ASAN_MEM_REDZONE_FLAG); 223 } 224 } 225 226 /* 227 * When the memory is released, the mark of the memory boundary 228 * in the corresponding range of the shadow area is cleared. 229 * Requirement: redzone >= 16, is a power of two. 230 */ 231 static inline void 232 asan_clear_redzone(struct malloc_elem *elem) 233 { 234 uintptr_t head_redzone; 235 uintptr_t tail_redzone; 236 void *head_shadow; 237 void *tail_shadow; 238 239 if (elem != NULL) { 240 elem = RTE_PTR_ADD(elem, elem->pad); 241 242 /* Clear mark before the start of the allocated memory */ 243 head_redzone = (uintptr_t)RTE_PTR_ADD(elem, 244 MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE); 245 head_shadow = ASAN_MEM_TO_SHADOW(head_redzone); 246 asan_set_shadow(head_shadow, 0x00); 247 head_shadow = ASAN_MEM_TO_SHADOW(head_redzone 248 - ASAN_SHADOW_GRAIN_SIZE); 249 asan_set_shadow(head_shadow, 0x00); 250 251 /* Clear mark after the end of the allocated memory */ 252 tail_redzone = (uintptr_t)RTE_PTR_ADD(elem, 253 MALLOC_ELEM_HEADER_LEN + elem->user_size); 254 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone); 255 asan_set_shadow(tail_shadow, 0x00); 256 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone 257 + ASAN_SHADOW_GRAIN_SIZE); 258 asan_set_shadow(tail_shadow, 0x00); 259 } 260 } 261 262 static inline size_t 263 old_malloc_size(struct malloc_elem *elem) 264 { 265 if (elem->state != ELEM_PAD) 266 elem = RTE_PTR_ADD(elem, elem->pad); 267 268 return elem->user_size; 269 } 270 271 #else /* !RTE_MALLOC_ASAN */ 272 273 #define __rte_no_asan 274 275 static inline void 276 asan_set_freezone(void *ptr __rte_unused, size_t size __rte_unused) { } 277 278 static inline void 279 asan_clear_alloczone(struct malloc_elem *elem __rte_unused) { } 280 281 static inline void 282 asan_clear_split_alloczone(struct malloc_elem *elem __rte_unused) { } 283 284 static inline void 285 asan_set_redzone(struct malloc_elem *elem __rte_unused, 286 size_t user_size __rte_unused) { } 287 288 static inline void 289 asan_clear_redzone(struct malloc_elem *elem __rte_unused) { } 290 291 static inline size_t 292 old_malloc_size(struct malloc_elem *elem) 293 { 294 return elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 295 } 296 #endif /* !RTE_MALLOC_ASAN */ 297 298 /* 299 * Given a pointer to the start of a memory block returned by malloc, get 300 * the actual malloc_elem header for that block. 301 */ 302 static inline struct malloc_elem * 303 malloc_elem_from_data(const void *data) 304 { 305 if (data == NULL) 306 return NULL; 307 308 struct malloc_elem *elem = RTE_PTR_SUB(data, MALLOC_ELEM_HEADER_LEN); 309 if (!malloc_elem_cookies_ok(elem)) 310 return NULL; 311 return elem->state != ELEM_PAD ? elem: RTE_PTR_SUB(elem, elem->pad); 312 } 313 314 /* 315 * initialise a malloc_elem header 316 */ 317 void 318 malloc_elem_init(struct malloc_elem *elem, 319 struct malloc_heap *heap, 320 struct rte_memseg_list *msl, 321 size_t size, 322 struct malloc_elem *orig_elem, 323 size_t orig_size); 324 325 void 326 malloc_elem_insert(struct malloc_elem *elem); 327 328 /* 329 * return true if the current malloc_elem can hold a block of data 330 * of the requested size and with the requested alignment 331 */ 332 int 333 malloc_elem_can_hold(struct malloc_elem *elem, size_t size, 334 unsigned int align, size_t bound, bool contig); 335 336 /* 337 * reserve a block of data in an existing malloc_elem. If the malloc_elem 338 * is much larger than the data block requested, we split the element in two. 339 */ 340 struct malloc_elem * 341 malloc_elem_alloc(struct malloc_elem *elem, size_t size, 342 unsigned int align, size_t bound, bool contig); 343 344 /* 345 * free a malloc_elem block by adding it to the free list. If the 346 * blocks either immediately before or immediately after newly freed block 347 * are also free, the blocks are merged together. 348 */ 349 struct malloc_elem * 350 malloc_elem_free(struct malloc_elem *elem); 351 352 struct malloc_elem * 353 malloc_elem_join_adjacent_free(struct malloc_elem *elem); 354 355 /* 356 * attempt to resize a malloc_elem by expanding into any free space 357 * immediately after it in memory. 358 */ 359 int 360 malloc_elem_resize(struct malloc_elem *elem, size_t size); 361 362 void 363 malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len); 364 365 void 366 malloc_elem_free_list_remove(struct malloc_elem *elem); 367 368 /* 369 * dump contents of malloc elem to a file. 370 */ 371 void 372 malloc_elem_dump(const struct malloc_elem *elem, FILE *f); 373 374 /* 375 * Given an element size, compute its freelist index. 376 */ 377 size_t 378 malloc_elem_free_list_index(size_t size); 379 380 /* 381 * Add element to its heap's free list. 382 */ 383 void 384 malloc_elem_free_list_insert(struct malloc_elem *elem); 385 386 /* 387 * Find biggest IOVA-contiguous zone within an element with specified alignment. 388 */ 389 size_t 390 malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align); 391 392 #endif /* MALLOC_ELEM_H_ */ 393