1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <string.h> 6 7 #include <rte_errno.h> 8 #include <rte_lcore.h> 9 #include <rte_fbarray.h> 10 #include <rte_memzone.h> 11 #include <rte_memory.h> 12 #include <rte_string_fns.h> 13 #include <rte_rwlock.h> 14 15 #include "eal_private.h" 16 #include "eal_internal_cfg.h" 17 #include "eal_memalloc.h" 18 19 struct mem_event_callback_entry { 20 TAILQ_ENTRY(mem_event_callback_entry) next; 21 char name[RTE_MEM_EVENT_CALLBACK_NAME_LEN]; 22 rte_mem_event_callback_t clb; 23 void *arg; 24 }; 25 26 struct mem_alloc_validator_entry { 27 TAILQ_ENTRY(mem_alloc_validator_entry) next; 28 char name[RTE_MEM_ALLOC_VALIDATOR_NAME_LEN]; 29 rte_mem_alloc_validator_t clb; 30 int socket_id; 31 size_t limit; 32 }; 33 34 /** Double linked list of actions. */ 35 TAILQ_HEAD(mem_event_callback_entry_list, mem_event_callback_entry); 36 TAILQ_HEAD(mem_alloc_validator_entry_list, mem_alloc_validator_entry); 37 38 static struct mem_event_callback_entry_list mem_event_callback_list = 39 TAILQ_HEAD_INITIALIZER(mem_event_callback_list); 40 static rte_rwlock_t mem_event_rwlock = RTE_RWLOCK_INITIALIZER; 41 42 static struct mem_alloc_validator_entry_list mem_alloc_validator_list = 43 TAILQ_HEAD_INITIALIZER(mem_alloc_validator_list); 44 static rte_rwlock_t mem_alloc_validator_rwlock = RTE_RWLOCK_INITIALIZER; 45 46 static struct mem_event_callback_entry * 47 find_mem_event_callback(const char *name, void *arg) 48 { 49 struct mem_event_callback_entry *r; 50 51 TAILQ_FOREACH(r, &mem_event_callback_list, next) { 52 if (!strcmp(r->name, name) && r->arg == arg) 53 break; 54 } 55 return r; 56 } 57 58 static struct mem_alloc_validator_entry * 59 find_mem_alloc_validator(const char *name, int socket_id) 60 { 61 struct mem_alloc_validator_entry *r; 62 63 TAILQ_FOREACH(r, &mem_alloc_validator_list, next) { 64 if (!strcmp(r->name, name) && r->socket_id == socket_id) 65 break; 66 } 67 return r; 68 } 69 70 bool 71 eal_memalloc_is_contig(const struct rte_memseg_list *msl, void *start, 72 size_t len) 73 { 74 void *end, *aligned_start, *aligned_end; 75 size_t pgsz = (size_t)msl->page_sz; 76 const struct rte_memseg *ms; 77 const struct internal_config *internal_conf = 78 eal_get_internal_configuration(); 79 80 /* for IOVA_VA, it's always contiguous */ 81 if (rte_eal_iova_mode() == RTE_IOVA_VA && !msl->external) 82 return true; 83 84 /* for legacy memory, it's always contiguous */ 85 if (internal_conf->legacy_mem) 86 return true; 87 88 end = RTE_PTR_ADD(start, len); 89 90 /* for nohuge, we check pagemap, otherwise check memseg */ 91 if (!rte_eal_has_hugepages()) { 92 rte_iova_t cur, expected; 93 94 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz); 95 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz); 96 97 /* if start and end are on the same page, bail out early */ 98 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz) 99 return true; 100 101 /* skip first iteration */ 102 cur = rte_mem_virt2iova(aligned_start); 103 expected = cur + pgsz; 104 aligned_start = RTE_PTR_ADD(aligned_start, pgsz); 105 106 while (aligned_start < aligned_end) { 107 cur = rte_mem_virt2iova(aligned_start); 108 if (cur != expected) 109 return false; 110 aligned_start = RTE_PTR_ADD(aligned_start, pgsz); 111 expected += pgsz; 112 } 113 } else { 114 int start_seg, end_seg, cur_seg; 115 rte_iova_t cur, expected; 116 117 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz); 118 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz); 119 120 start_seg = RTE_PTR_DIFF(aligned_start, msl->base_va) / 121 pgsz; 122 end_seg = RTE_PTR_DIFF(aligned_end, msl->base_va) / 123 pgsz; 124 125 /* if start and end are on the same page, bail out early */ 126 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz) 127 return true; 128 129 /* skip first iteration */ 130 ms = rte_fbarray_get(&msl->memseg_arr, start_seg); 131 cur = ms->iova; 132 expected = cur + pgsz; 133 134 /* if we can't access IOVA addresses, assume non-contiguous */ 135 if (cur == RTE_BAD_IOVA) 136 return false; 137 138 for (cur_seg = start_seg + 1; cur_seg < end_seg; 139 cur_seg++, expected += pgsz) { 140 ms = rte_fbarray_get(&msl->memseg_arr, cur_seg); 141 142 if (ms->iova != expected) 143 return false; 144 } 145 } 146 return true; 147 } 148 149 int 150 eal_memalloc_mem_event_callback_register(const char *name, 151 rte_mem_event_callback_t clb, void *arg) 152 { 153 struct mem_event_callback_entry *entry; 154 int ret, len; 155 if (name == NULL || clb == NULL) { 156 rte_errno = EINVAL; 157 return -1; 158 } 159 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN); 160 if (len == 0) { 161 rte_errno = EINVAL; 162 return -1; 163 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) { 164 rte_errno = ENAMETOOLONG; 165 return -1; 166 } 167 rte_rwlock_write_lock(&mem_event_rwlock); 168 169 entry = find_mem_event_callback(name, arg); 170 if (entry != NULL) { 171 rte_errno = EEXIST; 172 ret = -1; 173 goto unlock; 174 } 175 176 entry = malloc(sizeof(*entry)); 177 if (entry == NULL) { 178 rte_errno = ENOMEM; 179 ret = -1; 180 goto unlock; 181 } 182 183 /* callback successfully created and is valid, add it to the list */ 184 entry->clb = clb; 185 entry->arg = arg; 186 strlcpy(entry->name, name, RTE_MEM_EVENT_CALLBACK_NAME_LEN); 187 TAILQ_INSERT_TAIL(&mem_event_callback_list, entry, next); 188 189 ret = 0; 190 191 RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' registered\n", 192 name, arg); 193 194 unlock: 195 rte_rwlock_write_unlock(&mem_event_rwlock); 196 return ret; 197 } 198 199 int 200 eal_memalloc_mem_event_callback_unregister(const char *name, void *arg) 201 { 202 struct mem_event_callback_entry *entry; 203 int ret, len; 204 205 if (name == NULL) { 206 rte_errno = EINVAL; 207 return -1; 208 } 209 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN); 210 if (len == 0) { 211 rte_errno = EINVAL; 212 return -1; 213 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) { 214 rte_errno = ENAMETOOLONG; 215 return -1; 216 } 217 rte_rwlock_write_lock(&mem_event_rwlock); 218 219 entry = find_mem_event_callback(name, arg); 220 if (entry == NULL) { 221 rte_errno = ENOENT; 222 ret = -1; 223 goto unlock; 224 } 225 TAILQ_REMOVE(&mem_event_callback_list, entry, next); 226 free(entry); 227 228 ret = 0; 229 230 RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' unregistered\n", 231 name, arg); 232 233 unlock: 234 rte_rwlock_write_unlock(&mem_event_rwlock); 235 return ret; 236 } 237 238 void 239 eal_memalloc_mem_event_notify(enum rte_mem_event event, const void *start, 240 size_t len) 241 { 242 struct mem_event_callback_entry *entry; 243 244 rte_rwlock_read_lock(&mem_event_rwlock); 245 246 TAILQ_FOREACH(entry, &mem_event_callback_list, next) { 247 RTE_LOG(DEBUG, EAL, "Calling mem event callback '%s:%p'\n", 248 entry->name, entry->arg); 249 entry->clb(event, start, len, entry->arg); 250 } 251 252 rte_rwlock_read_unlock(&mem_event_rwlock); 253 } 254 255 int 256 eal_memalloc_mem_alloc_validator_register(const char *name, 257 rte_mem_alloc_validator_t clb, int socket_id, size_t limit) 258 { 259 struct mem_alloc_validator_entry *entry; 260 int ret, len; 261 if (name == NULL || clb == NULL || socket_id < 0) { 262 rte_errno = EINVAL; 263 return -1; 264 } 265 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN); 266 if (len == 0) { 267 rte_errno = EINVAL; 268 return -1; 269 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) { 270 rte_errno = ENAMETOOLONG; 271 return -1; 272 } 273 rte_rwlock_write_lock(&mem_alloc_validator_rwlock); 274 275 entry = find_mem_alloc_validator(name, socket_id); 276 if (entry != NULL) { 277 rte_errno = EEXIST; 278 ret = -1; 279 goto unlock; 280 } 281 282 entry = malloc(sizeof(*entry)); 283 if (entry == NULL) { 284 rte_errno = ENOMEM; 285 ret = -1; 286 goto unlock; 287 } 288 289 /* callback successfully created and is valid, add it to the list */ 290 entry->clb = clb; 291 entry->socket_id = socket_id; 292 entry->limit = limit; 293 strlcpy(entry->name, name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN); 294 TAILQ_INSERT_TAIL(&mem_alloc_validator_list, entry, next); 295 296 ret = 0; 297 298 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i with limit %zu registered\n", 299 name, socket_id, limit); 300 301 unlock: 302 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock); 303 return ret; 304 } 305 306 int 307 eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id) 308 { 309 struct mem_alloc_validator_entry *entry; 310 int ret, len; 311 312 if (name == NULL || socket_id < 0) { 313 rte_errno = EINVAL; 314 return -1; 315 } 316 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN); 317 if (len == 0) { 318 rte_errno = EINVAL; 319 return -1; 320 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) { 321 rte_errno = ENAMETOOLONG; 322 return -1; 323 } 324 rte_rwlock_write_lock(&mem_alloc_validator_rwlock); 325 326 entry = find_mem_alloc_validator(name, socket_id); 327 if (entry == NULL) { 328 rte_errno = ENOENT; 329 ret = -1; 330 goto unlock; 331 } 332 TAILQ_REMOVE(&mem_alloc_validator_list, entry, next); 333 free(entry); 334 335 ret = 0; 336 337 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i unregistered\n", 338 name, socket_id); 339 340 unlock: 341 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock); 342 return ret; 343 } 344 345 int 346 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len) 347 { 348 struct mem_alloc_validator_entry *entry; 349 int ret = 0; 350 351 rte_rwlock_read_lock(&mem_alloc_validator_rwlock); 352 353 TAILQ_FOREACH(entry, &mem_alloc_validator_list, next) { 354 if (entry->socket_id != socket_id || entry->limit > new_len) 355 continue; 356 RTE_LOG(DEBUG, EAL, "Calling mem alloc validator '%s' on socket %i\n", 357 entry->name, entry->socket_id); 358 if (entry->clb(socket_id, entry->limit, new_len) < 0) 359 ret = -1; 360 } 361 362 rte_rwlock_read_unlock(&mem_alloc_validator_rwlock); 363 364 return ret; 365 } 366