1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/assert.h" 5 #include "jemalloc/internal/ctl.h" 6 #include "jemalloc/internal/extent_dss.h" 7 #include "jemalloc/internal/extent_mmap.h" 8 #include "jemalloc/internal/inspect.h" 9 #include "jemalloc/internal/mutex.h" 10 #include "jemalloc/internal/nstime.h" 11 #include "jemalloc/internal/peak_event.h" 12 #include "jemalloc/internal/prof_data.h" 13 #include "jemalloc/internal/prof_log.h" 14 #include "jemalloc/internal/prof_recent.h" 15 #include "jemalloc/internal/prof_stats.h" 16 #include "jemalloc/internal/prof_sys.h" 17 #include "jemalloc/internal/safety_check.h" 18 #include "jemalloc/internal/sc.h" 19 #include "jemalloc/internal/util.h" 20 21 /******************************************************************************/ 22 /* Data. */ 23 24 /* 25 * ctl_mtx protects the following: 26 * - ctl_stats->* 27 */ 28 static malloc_mutex_t ctl_mtx; 29 static bool ctl_initialized; 30 static ctl_stats_t *ctl_stats; 31 static ctl_arenas_t *ctl_arenas; 32 33 /******************************************************************************/ 34 /* Helpers for named and indexed nodes. */ 35 36 static const ctl_named_node_t * 37 ctl_named_node(const ctl_node_t *node) { 38 return ((node->named) ? (const ctl_named_node_t *)node : NULL); 39 } 40 41 static const ctl_named_node_t * 42 ctl_named_children(const ctl_named_node_t *node, size_t index) { 43 const ctl_named_node_t *children = ctl_named_node(node->children); 44 45 return (children ? &children[index] : NULL); 46 } 47 48 static const ctl_indexed_node_t * 49 ctl_indexed_node(const ctl_node_t *node) { 50 return (!node->named ? (const ctl_indexed_node_t *)node : NULL); 51 } 52 53 /******************************************************************************/ 54 /* Function prototypes for non-inline static functions. */ 55 56 #define CTL_PROTO(n) \ 57 static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 58 void *oldp, size_t *oldlenp, void *newp, size_t newlen); 59 60 #define INDEX_PROTO(n) \ 61 static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ 62 const size_t *mib, size_t miblen, size_t i); 63 64 CTL_PROTO(version) 65 CTL_PROTO(epoch) 66 CTL_PROTO(background_thread) 67 CTL_PROTO(max_background_threads) 68 CTL_PROTO(thread_tcache_enabled) 69 CTL_PROTO(thread_tcache_flush) 70 CTL_PROTO(thread_peak_read) 71 CTL_PROTO(thread_peak_reset) 72 CTL_PROTO(thread_prof_name) 73 CTL_PROTO(thread_prof_active) 74 CTL_PROTO(thread_arena) 75 CTL_PROTO(thread_allocated) 76 CTL_PROTO(thread_allocatedp) 77 CTL_PROTO(thread_deallocated) 78 CTL_PROTO(thread_deallocatedp) 79 CTL_PROTO(thread_idle) 80 CTL_PROTO(config_cache_oblivious) 81 CTL_PROTO(config_debug) 82 CTL_PROTO(config_fill) 83 CTL_PROTO(config_lazy_lock) 84 CTL_PROTO(config_malloc_conf) 85 CTL_PROTO(config_opt_safety_checks) 86 CTL_PROTO(config_prof) 87 CTL_PROTO(config_prof_libgcc) 88 CTL_PROTO(config_prof_libunwind) 89 CTL_PROTO(config_stats) 90 CTL_PROTO(config_utrace) 91 CTL_PROTO(config_xmalloc) 92 CTL_PROTO(opt_abort) 93 CTL_PROTO(opt_abort_conf) 94 CTL_PROTO(opt_cache_oblivious) 95 CTL_PROTO(opt_trust_madvise) 96 CTL_PROTO(opt_confirm_conf) 97 CTL_PROTO(opt_hpa) 98 CTL_PROTO(opt_hpa_slab_max_alloc) 99 CTL_PROTO(opt_hpa_hugification_threshold) 100 CTL_PROTO(opt_hpa_hugify_delay_ms) 101 CTL_PROTO(opt_hpa_min_purge_interval_ms) 102 CTL_PROTO(opt_hpa_dirty_mult) 103 CTL_PROTO(opt_hpa_sec_nshards) 104 CTL_PROTO(opt_hpa_sec_max_alloc) 105 CTL_PROTO(opt_hpa_sec_max_bytes) 106 CTL_PROTO(opt_hpa_sec_bytes_after_flush) 107 CTL_PROTO(opt_hpa_sec_batch_fill_extra) 108 CTL_PROTO(opt_metadata_thp) 109 CTL_PROTO(opt_retain) 110 CTL_PROTO(opt_dss) 111 CTL_PROTO(opt_narenas) 112 CTL_PROTO(opt_percpu_arena) 113 CTL_PROTO(opt_oversize_threshold) 114 CTL_PROTO(opt_background_thread) 115 CTL_PROTO(opt_mutex_max_spin) 116 CTL_PROTO(opt_max_background_threads) 117 CTL_PROTO(opt_dirty_decay_ms) 118 CTL_PROTO(opt_muzzy_decay_ms) 119 CTL_PROTO(opt_stats_print) 120 CTL_PROTO(opt_stats_print_opts) 121 CTL_PROTO(opt_stats_interval) 122 CTL_PROTO(opt_stats_interval_opts) 123 CTL_PROTO(opt_junk) 124 CTL_PROTO(opt_zero) 125 CTL_PROTO(opt_utrace) 126 CTL_PROTO(opt_xmalloc) 127 CTL_PROTO(opt_experimental_infallible_new) 128 CTL_PROTO(opt_tcache) 129 CTL_PROTO(opt_tcache_max) 130 CTL_PROTO(opt_tcache_nslots_small_min) 131 CTL_PROTO(opt_tcache_nslots_small_max) 132 CTL_PROTO(opt_tcache_nslots_large) 133 CTL_PROTO(opt_lg_tcache_nslots_mul) 134 CTL_PROTO(opt_tcache_gc_incr_bytes) 135 CTL_PROTO(opt_tcache_gc_delay_bytes) 136 CTL_PROTO(opt_lg_tcache_flush_small_div) 137 CTL_PROTO(opt_lg_tcache_flush_large_div) 138 CTL_PROTO(opt_thp) 139 CTL_PROTO(opt_lg_extent_max_active_fit) 140 CTL_PROTO(opt_prof) 141 CTL_PROTO(opt_prof_prefix) 142 CTL_PROTO(opt_prof_active) 143 CTL_PROTO(opt_prof_thread_active_init) 144 CTL_PROTO(opt_lg_prof_sample) 145 CTL_PROTO(opt_lg_prof_interval) 146 CTL_PROTO(opt_prof_gdump) 147 CTL_PROTO(opt_prof_final) 148 CTL_PROTO(opt_prof_leak) 149 CTL_PROTO(opt_prof_leak_error) 150 CTL_PROTO(opt_prof_accum) 151 CTL_PROTO(opt_prof_recent_alloc_max) 152 CTL_PROTO(opt_prof_stats) 153 CTL_PROTO(opt_prof_sys_thread_name) 154 CTL_PROTO(opt_prof_time_res) 155 CTL_PROTO(opt_lg_san_uaf_align) 156 CTL_PROTO(opt_zero_realloc) 157 CTL_PROTO(tcache_create) 158 CTL_PROTO(tcache_flush) 159 CTL_PROTO(tcache_destroy) 160 CTL_PROTO(arena_i_initialized) 161 CTL_PROTO(arena_i_decay) 162 CTL_PROTO(arena_i_purge) 163 CTL_PROTO(arena_i_reset) 164 CTL_PROTO(arena_i_destroy) 165 CTL_PROTO(arena_i_dss) 166 CTL_PROTO(arena_i_oversize_threshold) 167 CTL_PROTO(arena_i_dirty_decay_ms) 168 CTL_PROTO(arena_i_muzzy_decay_ms) 169 CTL_PROTO(arena_i_extent_hooks) 170 CTL_PROTO(arena_i_retain_grow_limit) 171 INDEX_PROTO(arena_i) 172 CTL_PROTO(arenas_bin_i_size) 173 CTL_PROTO(arenas_bin_i_nregs) 174 CTL_PROTO(arenas_bin_i_slab_size) 175 CTL_PROTO(arenas_bin_i_nshards) 176 INDEX_PROTO(arenas_bin_i) 177 CTL_PROTO(arenas_lextent_i_size) 178 INDEX_PROTO(arenas_lextent_i) 179 CTL_PROTO(arenas_narenas) 180 CTL_PROTO(arenas_dirty_decay_ms) 181 CTL_PROTO(arenas_muzzy_decay_ms) 182 CTL_PROTO(arenas_quantum) 183 CTL_PROTO(arenas_page) 184 CTL_PROTO(arenas_tcache_max) 185 CTL_PROTO(arenas_nbins) 186 CTL_PROTO(arenas_nhbins) 187 CTL_PROTO(arenas_nlextents) 188 CTL_PROTO(arenas_create) 189 CTL_PROTO(arenas_lookup) 190 CTL_PROTO(prof_thread_active_init) 191 CTL_PROTO(prof_active) 192 CTL_PROTO(prof_dump) 193 CTL_PROTO(prof_gdump) 194 CTL_PROTO(prof_prefix) 195 CTL_PROTO(prof_reset) 196 CTL_PROTO(prof_interval) 197 CTL_PROTO(lg_prof_sample) 198 CTL_PROTO(prof_log_start) 199 CTL_PROTO(prof_log_stop) 200 CTL_PROTO(prof_stats_bins_i_live) 201 CTL_PROTO(prof_stats_bins_i_accum) 202 INDEX_PROTO(prof_stats_bins_i) 203 CTL_PROTO(prof_stats_lextents_i_live) 204 CTL_PROTO(prof_stats_lextents_i_accum) 205 INDEX_PROTO(prof_stats_lextents_i) 206 CTL_PROTO(stats_arenas_i_small_allocated) 207 CTL_PROTO(stats_arenas_i_small_nmalloc) 208 CTL_PROTO(stats_arenas_i_small_ndalloc) 209 CTL_PROTO(stats_arenas_i_small_nrequests) 210 CTL_PROTO(stats_arenas_i_small_nfills) 211 CTL_PROTO(stats_arenas_i_small_nflushes) 212 CTL_PROTO(stats_arenas_i_large_allocated) 213 CTL_PROTO(stats_arenas_i_large_nmalloc) 214 CTL_PROTO(stats_arenas_i_large_ndalloc) 215 CTL_PROTO(stats_arenas_i_large_nrequests) 216 CTL_PROTO(stats_arenas_i_large_nfills) 217 CTL_PROTO(stats_arenas_i_large_nflushes) 218 CTL_PROTO(stats_arenas_i_bins_j_nmalloc) 219 CTL_PROTO(stats_arenas_i_bins_j_ndalloc) 220 CTL_PROTO(stats_arenas_i_bins_j_nrequests) 221 CTL_PROTO(stats_arenas_i_bins_j_curregs) 222 CTL_PROTO(stats_arenas_i_bins_j_nfills) 223 CTL_PROTO(stats_arenas_i_bins_j_nflushes) 224 CTL_PROTO(stats_arenas_i_bins_j_nslabs) 225 CTL_PROTO(stats_arenas_i_bins_j_nreslabs) 226 CTL_PROTO(stats_arenas_i_bins_j_curslabs) 227 CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs) 228 INDEX_PROTO(stats_arenas_i_bins_j) 229 CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) 230 CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) 231 CTL_PROTO(stats_arenas_i_lextents_j_nrequests) 232 CTL_PROTO(stats_arenas_i_lextents_j_curlextents) 233 INDEX_PROTO(stats_arenas_i_lextents_j) 234 CTL_PROTO(stats_arenas_i_extents_j_ndirty) 235 CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) 236 CTL_PROTO(stats_arenas_i_extents_j_nretained) 237 CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) 238 CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) 239 CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) 240 INDEX_PROTO(stats_arenas_i_extents_j) 241 CTL_PROTO(stats_arenas_i_hpa_shard_npurge_passes) 242 CTL_PROTO(stats_arenas_i_hpa_shard_npurges) 243 CTL_PROTO(stats_arenas_i_hpa_shard_nhugifies) 244 CTL_PROTO(stats_arenas_i_hpa_shard_ndehugifies) 245 246 /* We have a set of stats for full slabs. */ 247 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge) 248 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge) 249 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge) 250 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_huge) 251 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge) 252 CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge) 253 254 /* A parallel set for the empty slabs. */ 255 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge) 256 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge) 257 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge) 258 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge) 259 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge) 260 CTL_PROTO(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge) 261 262 /* 263 * And one for the slabs that are neither empty nor full, but indexed by how 264 * full they are. 265 */ 266 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge) 267 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge) 268 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge) 269 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge) 270 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge) 271 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge) 272 273 INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j) 274 CTL_PROTO(stats_arenas_i_nthreads) 275 CTL_PROTO(stats_arenas_i_uptime) 276 CTL_PROTO(stats_arenas_i_dss) 277 CTL_PROTO(stats_arenas_i_dirty_decay_ms) 278 CTL_PROTO(stats_arenas_i_muzzy_decay_ms) 279 CTL_PROTO(stats_arenas_i_pactive) 280 CTL_PROTO(stats_arenas_i_pdirty) 281 CTL_PROTO(stats_arenas_i_pmuzzy) 282 CTL_PROTO(stats_arenas_i_mapped) 283 CTL_PROTO(stats_arenas_i_retained) 284 CTL_PROTO(stats_arenas_i_extent_avail) 285 CTL_PROTO(stats_arenas_i_dirty_npurge) 286 CTL_PROTO(stats_arenas_i_dirty_nmadvise) 287 CTL_PROTO(stats_arenas_i_dirty_purged) 288 CTL_PROTO(stats_arenas_i_muzzy_npurge) 289 CTL_PROTO(stats_arenas_i_muzzy_nmadvise) 290 CTL_PROTO(stats_arenas_i_muzzy_purged) 291 CTL_PROTO(stats_arenas_i_base) 292 CTL_PROTO(stats_arenas_i_internal) 293 CTL_PROTO(stats_arenas_i_metadata_thp) 294 CTL_PROTO(stats_arenas_i_tcache_bytes) 295 CTL_PROTO(stats_arenas_i_tcache_stashed_bytes) 296 CTL_PROTO(stats_arenas_i_resident) 297 CTL_PROTO(stats_arenas_i_abandoned_vm) 298 CTL_PROTO(stats_arenas_i_hpa_sec_bytes) 299 INDEX_PROTO(stats_arenas_i) 300 CTL_PROTO(stats_allocated) 301 CTL_PROTO(stats_active) 302 CTL_PROTO(stats_background_thread_num_threads) 303 CTL_PROTO(stats_background_thread_num_runs) 304 CTL_PROTO(stats_background_thread_run_interval) 305 CTL_PROTO(stats_metadata) 306 CTL_PROTO(stats_metadata_thp) 307 CTL_PROTO(stats_resident) 308 CTL_PROTO(stats_mapped) 309 CTL_PROTO(stats_retained) 310 CTL_PROTO(stats_zero_reallocs) 311 CTL_PROTO(experimental_hooks_install) 312 CTL_PROTO(experimental_hooks_remove) 313 CTL_PROTO(experimental_hooks_prof_backtrace) 314 CTL_PROTO(experimental_hooks_prof_dump) 315 CTL_PROTO(experimental_hooks_safety_check_abort) 316 CTL_PROTO(experimental_thread_activity_callback) 317 CTL_PROTO(experimental_utilization_query) 318 CTL_PROTO(experimental_utilization_batch_query) 319 CTL_PROTO(experimental_arenas_i_pactivep) 320 INDEX_PROTO(experimental_arenas_i) 321 CTL_PROTO(experimental_prof_recent_alloc_max) 322 CTL_PROTO(experimental_prof_recent_alloc_dump) 323 CTL_PROTO(experimental_batch_alloc) 324 CTL_PROTO(experimental_arenas_create_ext) 325 326 #define MUTEX_STATS_CTL_PROTO_GEN(n) \ 327 CTL_PROTO(stats_##n##_num_ops) \ 328 CTL_PROTO(stats_##n##_num_wait) \ 329 CTL_PROTO(stats_##n##_num_spin_acq) \ 330 CTL_PROTO(stats_##n##_num_owner_switch) \ 331 CTL_PROTO(stats_##n##_total_wait_time) \ 332 CTL_PROTO(stats_##n##_max_wait_time) \ 333 CTL_PROTO(stats_##n##_max_num_thds) 334 335 /* Global mutexes. */ 336 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) 337 MUTEX_PROF_GLOBAL_MUTEXES 338 #undef OP 339 340 /* Per arena mutexes. */ 341 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx) 342 MUTEX_PROF_ARENA_MUTEXES 343 #undef OP 344 345 /* Arena bin mutexes. */ 346 MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) 347 #undef MUTEX_STATS_CTL_PROTO_GEN 348 349 CTL_PROTO(stats_mutexes_reset) 350 351 /******************************************************************************/ 352 /* mallctl tree. */ 353 354 #define NAME(n) {true}, n 355 #define CHILD(t, c) \ 356 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ 357 (const ctl_node_t *)c##_node, \ 358 NULL 359 #define CTL(c) 0, NULL, c##_ctl 360 361 /* 362 * Only handles internal indexed nodes, since there are currently no external 363 * ones. 364 */ 365 #define INDEX(i) {false}, i##_index 366 367 static const ctl_named_node_t thread_tcache_node[] = { 368 {NAME("enabled"), CTL(thread_tcache_enabled)}, 369 {NAME("flush"), CTL(thread_tcache_flush)} 370 }; 371 372 static const ctl_named_node_t thread_peak_node[] = { 373 {NAME("read"), CTL(thread_peak_read)}, 374 {NAME("reset"), CTL(thread_peak_reset)}, 375 }; 376 377 static const ctl_named_node_t thread_prof_node[] = { 378 {NAME("name"), CTL(thread_prof_name)}, 379 {NAME("active"), CTL(thread_prof_active)} 380 }; 381 382 static const ctl_named_node_t thread_node[] = { 383 {NAME("arena"), CTL(thread_arena)}, 384 {NAME("allocated"), CTL(thread_allocated)}, 385 {NAME("allocatedp"), CTL(thread_allocatedp)}, 386 {NAME("deallocated"), CTL(thread_deallocated)}, 387 {NAME("deallocatedp"), CTL(thread_deallocatedp)}, 388 {NAME("tcache"), CHILD(named, thread_tcache)}, 389 {NAME("peak"), CHILD(named, thread_peak)}, 390 {NAME("prof"), CHILD(named, thread_prof)}, 391 {NAME("idle"), CTL(thread_idle)} 392 }; 393 394 static const ctl_named_node_t config_node[] = { 395 {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, 396 {NAME("debug"), CTL(config_debug)}, 397 {NAME("fill"), CTL(config_fill)}, 398 {NAME("lazy_lock"), CTL(config_lazy_lock)}, 399 {NAME("malloc_conf"), CTL(config_malloc_conf)}, 400 {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, 401 {NAME("prof"), CTL(config_prof)}, 402 {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, 403 {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, 404 {NAME("stats"), CTL(config_stats)}, 405 {NAME("utrace"), CTL(config_utrace)}, 406 {NAME("xmalloc"), CTL(config_xmalloc)} 407 }; 408 409 static const ctl_named_node_t opt_node[] = { 410 {NAME("abort"), CTL(opt_abort)}, 411 {NAME("abort_conf"), CTL(opt_abort_conf)}, 412 {NAME("cache_oblivious"), CTL(opt_cache_oblivious)}, 413 {NAME("trust_madvise"), CTL(opt_trust_madvise)}, 414 {NAME("confirm_conf"), CTL(opt_confirm_conf)}, 415 {NAME("hpa"), CTL(opt_hpa)}, 416 {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, 417 {NAME("hpa_hugification_threshold"), 418 CTL(opt_hpa_hugification_threshold)}, 419 {NAME("hpa_hugify_delay_ms"), CTL(opt_hpa_hugify_delay_ms)}, 420 {NAME("hpa_min_purge_interval_ms"), CTL(opt_hpa_min_purge_interval_ms)}, 421 {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)}, 422 {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, 423 {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, 424 {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, 425 {NAME("hpa_sec_bytes_after_flush"), 426 CTL(opt_hpa_sec_bytes_after_flush)}, 427 {NAME("hpa_sec_batch_fill_extra"), 428 CTL(opt_hpa_sec_batch_fill_extra)}, 429 {NAME("metadata_thp"), CTL(opt_metadata_thp)}, 430 {NAME("retain"), CTL(opt_retain)}, 431 {NAME("dss"), CTL(opt_dss)}, 432 {NAME("narenas"), CTL(opt_narenas)}, 433 {NAME("percpu_arena"), CTL(opt_percpu_arena)}, 434 {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, 435 {NAME("mutex_max_spin"), CTL(opt_mutex_max_spin)}, 436 {NAME("background_thread"), CTL(opt_background_thread)}, 437 {NAME("max_background_threads"), CTL(opt_max_background_threads)}, 438 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, 439 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, 440 {NAME("stats_print"), CTL(opt_stats_print)}, 441 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, 442 {NAME("stats_interval"), CTL(opt_stats_interval)}, 443 {NAME("stats_interval_opts"), CTL(opt_stats_interval_opts)}, 444 {NAME("junk"), CTL(opt_junk)}, 445 {NAME("zero"), CTL(opt_zero)}, 446 {NAME("utrace"), CTL(opt_utrace)}, 447 {NAME("xmalloc"), CTL(opt_xmalloc)}, 448 {NAME("experimental_infallible_new"), 449 CTL(opt_experimental_infallible_new)}, 450 {NAME("tcache"), CTL(opt_tcache)}, 451 {NAME("tcache_max"), CTL(opt_tcache_max)}, 452 {NAME("tcache_nslots_small_min"), 453 CTL(opt_tcache_nslots_small_min)}, 454 {NAME("tcache_nslots_small_max"), 455 CTL(opt_tcache_nslots_small_max)}, 456 {NAME("tcache_nslots_large"), CTL(opt_tcache_nslots_large)}, 457 {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)}, 458 {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)}, 459 {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)}, 460 {NAME("lg_tcache_flush_small_div"), 461 CTL(opt_lg_tcache_flush_small_div)}, 462 {NAME("lg_tcache_flush_large_div"), 463 CTL(opt_lg_tcache_flush_large_div)}, 464 {NAME("thp"), CTL(opt_thp)}, 465 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, 466 {NAME("prof"), CTL(opt_prof)}, 467 {NAME("prof_prefix"), CTL(opt_prof_prefix)}, 468 {NAME("prof_active"), CTL(opt_prof_active)}, 469 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, 470 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, 471 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, 472 {NAME("prof_gdump"), CTL(opt_prof_gdump)}, 473 {NAME("prof_final"), CTL(opt_prof_final)}, 474 {NAME("prof_leak"), CTL(opt_prof_leak)}, 475 {NAME("prof_leak_error"), CTL(opt_prof_leak_error)}, 476 {NAME("prof_accum"), CTL(opt_prof_accum)}, 477 {NAME("prof_recent_alloc_max"), CTL(opt_prof_recent_alloc_max)}, 478 {NAME("prof_stats"), CTL(opt_prof_stats)}, 479 {NAME("prof_sys_thread_name"), CTL(opt_prof_sys_thread_name)}, 480 {NAME("prof_time_resolution"), CTL(opt_prof_time_res)}, 481 {NAME("lg_san_uaf_align"), CTL(opt_lg_san_uaf_align)}, 482 {NAME("zero_realloc"), CTL(opt_zero_realloc)} 483 }; 484 485 static const ctl_named_node_t tcache_node[] = { 486 {NAME("create"), CTL(tcache_create)}, 487 {NAME("flush"), CTL(tcache_flush)}, 488 {NAME("destroy"), CTL(tcache_destroy)} 489 }; 490 491 static const ctl_named_node_t arena_i_node[] = { 492 {NAME("initialized"), CTL(arena_i_initialized)}, 493 {NAME("decay"), CTL(arena_i_decay)}, 494 {NAME("purge"), CTL(arena_i_purge)}, 495 {NAME("reset"), CTL(arena_i_reset)}, 496 {NAME("destroy"), CTL(arena_i_destroy)}, 497 {NAME("dss"), CTL(arena_i_dss)}, 498 /* 499 * Undocumented for now, since we anticipate an arena API in flux after 500 * we cut the last 5-series release. 501 */ 502 {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)}, 503 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, 504 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, 505 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, 506 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)} 507 }; 508 static const ctl_named_node_t super_arena_i_node[] = { 509 {NAME(""), CHILD(named, arena_i)} 510 }; 511 512 static const ctl_indexed_node_t arena_node[] = { 513 {INDEX(arena_i)} 514 }; 515 516 static const ctl_named_node_t arenas_bin_i_node[] = { 517 {NAME("size"), CTL(arenas_bin_i_size)}, 518 {NAME("nregs"), CTL(arenas_bin_i_nregs)}, 519 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, 520 {NAME("nshards"), CTL(arenas_bin_i_nshards)} 521 }; 522 static const ctl_named_node_t super_arenas_bin_i_node[] = { 523 {NAME(""), CHILD(named, arenas_bin_i)} 524 }; 525 526 static const ctl_indexed_node_t arenas_bin_node[] = { 527 {INDEX(arenas_bin_i)} 528 }; 529 530 static const ctl_named_node_t arenas_lextent_i_node[] = { 531 {NAME("size"), CTL(arenas_lextent_i_size)} 532 }; 533 static const ctl_named_node_t super_arenas_lextent_i_node[] = { 534 {NAME(""), CHILD(named, arenas_lextent_i)} 535 }; 536 537 static const ctl_indexed_node_t arenas_lextent_node[] = { 538 {INDEX(arenas_lextent_i)} 539 }; 540 541 static const ctl_named_node_t arenas_node[] = { 542 {NAME("narenas"), CTL(arenas_narenas)}, 543 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, 544 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, 545 {NAME("quantum"), CTL(arenas_quantum)}, 546 {NAME("page"), CTL(arenas_page)}, 547 {NAME("tcache_max"), CTL(arenas_tcache_max)}, 548 {NAME("nbins"), CTL(arenas_nbins)}, 549 {NAME("nhbins"), CTL(arenas_nhbins)}, 550 {NAME("bin"), CHILD(indexed, arenas_bin)}, 551 {NAME("nlextents"), CTL(arenas_nlextents)}, 552 {NAME("lextent"), CHILD(indexed, arenas_lextent)}, 553 {NAME("create"), CTL(arenas_create)}, 554 {NAME("lookup"), CTL(arenas_lookup)} 555 }; 556 557 static const ctl_named_node_t prof_stats_bins_i_node[] = { 558 {NAME("live"), CTL(prof_stats_bins_i_live)}, 559 {NAME("accum"), CTL(prof_stats_bins_i_accum)} 560 }; 561 562 static const ctl_named_node_t super_prof_stats_bins_i_node[] = { 563 {NAME(""), CHILD(named, prof_stats_bins_i)} 564 }; 565 566 static const ctl_indexed_node_t prof_stats_bins_node[] = { 567 {INDEX(prof_stats_bins_i)} 568 }; 569 570 static const ctl_named_node_t prof_stats_lextents_i_node[] = { 571 {NAME("live"), CTL(prof_stats_lextents_i_live)}, 572 {NAME("accum"), CTL(prof_stats_lextents_i_accum)} 573 }; 574 575 static const ctl_named_node_t super_prof_stats_lextents_i_node[] = { 576 {NAME(""), CHILD(named, prof_stats_lextents_i)} 577 }; 578 579 static const ctl_indexed_node_t prof_stats_lextents_node[] = { 580 {INDEX(prof_stats_lextents_i)} 581 }; 582 583 static const ctl_named_node_t prof_stats_node[] = { 584 {NAME("bins"), CHILD(indexed, prof_stats_bins)}, 585 {NAME("lextents"), CHILD(indexed, prof_stats_lextents)}, 586 }; 587 588 static const ctl_named_node_t prof_node[] = { 589 {NAME("thread_active_init"), CTL(prof_thread_active_init)}, 590 {NAME("active"), CTL(prof_active)}, 591 {NAME("dump"), CTL(prof_dump)}, 592 {NAME("gdump"), CTL(prof_gdump)}, 593 {NAME("prefix"), CTL(prof_prefix)}, 594 {NAME("reset"), CTL(prof_reset)}, 595 {NAME("interval"), CTL(prof_interval)}, 596 {NAME("lg_sample"), CTL(lg_prof_sample)}, 597 {NAME("log_start"), CTL(prof_log_start)}, 598 {NAME("log_stop"), CTL(prof_log_stop)}, 599 {NAME("stats"), CHILD(named, prof_stats)} 600 }; 601 602 static const ctl_named_node_t stats_arenas_i_small_node[] = { 603 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, 604 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, 605 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, 606 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, 607 {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, 608 {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)} 609 }; 610 611 static const ctl_named_node_t stats_arenas_i_large_node[] = { 612 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, 613 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, 614 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, 615 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, 616 {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, 617 {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)} 618 }; 619 620 #define MUTEX_PROF_DATA_NODE(prefix) \ 621 static const ctl_named_node_t stats_##prefix##_node[] = { \ 622 {NAME("num_ops"), \ 623 CTL(stats_##prefix##_num_ops)}, \ 624 {NAME("num_wait"), \ 625 CTL(stats_##prefix##_num_wait)}, \ 626 {NAME("num_spin_acq"), \ 627 CTL(stats_##prefix##_num_spin_acq)}, \ 628 {NAME("num_owner_switch"), \ 629 CTL(stats_##prefix##_num_owner_switch)}, \ 630 {NAME("total_wait_time"), \ 631 CTL(stats_##prefix##_total_wait_time)}, \ 632 {NAME("max_wait_time"), \ 633 CTL(stats_##prefix##_max_wait_time)}, \ 634 {NAME("max_num_thds"), \ 635 CTL(stats_##prefix##_max_num_thds)} \ 636 /* Note that # of current waiting thread not provided. */ \ 637 }; 638 639 MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) 640 641 static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { 642 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, 643 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, 644 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, 645 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, 646 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, 647 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, 648 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, 649 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, 650 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, 651 {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, 652 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} 653 }; 654 655 static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { 656 {NAME(""), CHILD(named, stats_arenas_i_bins_j)} 657 }; 658 659 static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { 660 {INDEX(stats_arenas_i_bins_j)} 661 }; 662 663 static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = { 664 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, 665 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, 666 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, 667 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)} 668 }; 669 static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = { 670 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)} 671 }; 672 673 static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { 674 {INDEX(stats_arenas_i_lextents_j)} 675 }; 676 677 static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { 678 {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, 679 {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, 680 {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, 681 {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, 682 {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, 683 {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} 684 }; 685 686 static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { 687 {NAME(""), CHILD(named, stats_arenas_i_extents_j)} 688 }; 689 690 static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { 691 {INDEX(stats_arenas_i_extents_j)} 692 }; 693 694 #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) 695 MUTEX_PROF_ARENA_MUTEXES 696 #undef OP 697 698 static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { 699 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, 700 MUTEX_PROF_ARENA_MUTEXES 701 #undef OP 702 }; 703 704 static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = { 705 {NAME("npageslabs_nonhuge"), 706 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)}, 707 {NAME("npageslabs_huge"), 708 CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)}, 709 {NAME("nactive_nonhuge"), 710 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)}, 711 {NAME("nactive_huge"), 712 CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)}, 713 {NAME("ndirty_nonhuge"), 714 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)}, 715 {NAME("ndirty_huge"), 716 CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)} 717 }; 718 719 static const ctl_named_node_t stats_arenas_i_hpa_shard_empty_slabs_node[] = { 720 {NAME("npageslabs_nonhuge"), 721 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)}, 722 {NAME("npageslabs_huge"), 723 CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)}, 724 {NAME("nactive_nonhuge"), 725 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)}, 726 {NAME("nactive_huge"), 727 CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)}, 728 {NAME("ndirty_nonhuge"), 729 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)}, 730 {NAME("ndirty_huge"), 731 CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)} 732 }; 733 734 static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { 735 {NAME("npageslabs_nonhuge"), 736 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)}, 737 {NAME("npageslabs_huge"), 738 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)}, 739 {NAME("nactive_nonhuge"), 740 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)}, 741 {NAME("nactive_huge"), 742 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)}, 743 {NAME("ndirty_nonhuge"), 744 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)}, 745 {NAME("ndirty_huge"), 746 CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)} 747 }; 748 749 static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { 750 {NAME(""), 751 CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)} 752 }; 753 754 static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] = 755 { 756 {INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)} 757 }; 758 759 static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = { 760 {NAME("full_slabs"), CHILD(named, 761 stats_arenas_i_hpa_shard_full_slabs)}, 762 {NAME("empty_slabs"), CHILD(named, 763 stats_arenas_i_hpa_shard_empty_slabs)}, 764 {NAME("nonfull_slabs"), CHILD(indexed, 765 stats_arenas_i_hpa_shard_nonfull_slabs)}, 766 767 {NAME("npurge_passes"), CTL(stats_arenas_i_hpa_shard_npurge_passes)}, 768 {NAME("npurges"), CTL(stats_arenas_i_hpa_shard_npurges)}, 769 {NAME("nhugifies"), CTL(stats_arenas_i_hpa_shard_nhugifies)}, 770 {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)} 771 }; 772 773 static const ctl_named_node_t stats_arenas_i_node[] = { 774 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, 775 {NAME("uptime"), CTL(stats_arenas_i_uptime)}, 776 {NAME("dss"), CTL(stats_arenas_i_dss)}, 777 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, 778 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, 779 {NAME("pactive"), CTL(stats_arenas_i_pactive)}, 780 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, 781 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, 782 {NAME("mapped"), CTL(stats_arenas_i_mapped)}, 783 {NAME("retained"), CTL(stats_arenas_i_retained)}, 784 {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, 785 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, 786 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, 787 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, 788 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, 789 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, 790 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, 791 {NAME("base"), CTL(stats_arenas_i_base)}, 792 {NAME("internal"), CTL(stats_arenas_i_internal)}, 793 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, 794 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, 795 {NAME("tcache_stashed_bytes"), 796 CTL(stats_arenas_i_tcache_stashed_bytes)}, 797 {NAME("resident"), CTL(stats_arenas_i_resident)}, 798 {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, 799 {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)}, 800 {NAME("small"), CHILD(named, stats_arenas_i_small)}, 801 {NAME("large"), CHILD(named, stats_arenas_i_large)}, 802 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, 803 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, 804 {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, 805 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}, 806 {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)} 807 }; 808 static const ctl_named_node_t super_stats_arenas_i_node[] = { 809 {NAME(""), CHILD(named, stats_arenas_i)} 810 }; 811 812 static const ctl_indexed_node_t stats_arenas_node[] = { 813 {INDEX(stats_arenas_i)} 814 }; 815 816 static const ctl_named_node_t stats_background_thread_node[] = { 817 {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, 818 {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, 819 {NAME("run_interval"), CTL(stats_background_thread_run_interval)} 820 }; 821 822 #define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) 823 MUTEX_PROF_GLOBAL_MUTEXES 824 #undef OP 825 826 static const ctl_named_node_t stats_mutexes_node[] = { 827 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, 828 MUTEX_PROF_GLOBAL_MUTEXES 829 #undef OP 830 {NAME("reset"), CTL(stats_mutexes_reset)} 831 }; 832 #undef MUTEX_PROF_DATA_NODE 833 834 static const ctl_named_node_t stats_node[] = { 835 {NAME("allocated"), CTL(stats_allocated)}, 836 {NAME("active"), CTL(stats_active)}, 837 {NAME("metadata"), CTL(stats_metadata)}, 838 {NAME("metadata_thp"), CTL(stats_metadata_thp)}, 839 {NAME("resident"), CTL(stats_resident)}, 840 {NAME("mapped"), CTL(stats_mapped)}, 841 {NAME("retained"), CTL(stats_retained)}, 842 {NAME("background_thread"), 843 CHILD(named, stats_background_thread)}, 844 {NAME("mutexes"), CHILD(named, stats_mutexes)}, 845 {NAME("arenas"), CHILD(indexed, stats_arenas)}, 846 {NAME("zero_reallocs"), CTL(stats_zero_reallocs)}, 847 }; 848 849 static const ctl_named_node_t experimental_hooks_node[] = { 850 {NAME("install"), CTL(experimental_hooks_install)}, 851 {NAME("remove"), CTL(experimental_hooks_remove)}, 852 {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)}, 853 {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)}, 854 {NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)}, 855 }; 856 857 static const ctl_named_node_t experimental_thread_node[] = { 858 {NAME("activity_callback"), 859 CTL(experimental_thread_activity_callback)} 860 }; 861 862 static const ctl_named_node_t experimental_utilization_node[] = { 863 {NAME("query"), CTL(experimental_utilization_query)}, 864 {NAME("batch_query"), CTL(experimental_utilization_batch_query)} 865 }; 866 867 static const ctl_named_node_t experimental_arenas_i_node[] = { 868 {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)} 869 }; 870 static const ctl_named_node_t super_experimental_arenas_i_node[] = { 871 {NAME(""), CHILD(named, experimental_arenas_i)} 872 }; 873 874 static const ctl_indexed_node_t experimental_arenas_node[] = { 875 {INDEX(experimental_arenas_i)} 876 }; 877 878 static const ctl_named_node_t experimental_prof_recent_node[] = { 879 {NAME("alloc_max"), CTL(experimental_prof_recent_alloc_max)}, 880 {NAME("alloc_dump"), CTL(experimental_prof_recent_alloc_dump)}, 881 }; 882 883 static const ctl_named_node_t experimental_node[] = { 884 {NAME("hooks"), CHILD(named, experimental_hooks)}, 885 {NAME("utilization"), CHILD(named, experimental_utilization)}, 886 {NAME("arenas"), CHILD(indexed, experimental_arenas)}, 887 {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)}, 888 {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}, 889 {NAME("batch_alloc"), CTL(experimental_batch_alloc)}, 890 {NAME("thread"), CHILD(named, experimental_thread)} 891 }; 892 893 static const ctl_named_node_t root_node[] = { 894 {NAME("version"), CTL(version)}, 895 {NAME("epoch"), CTL(epoch)}, 896 {NAME("background_thread"), CTL(background_thread)}, 897 {NAME("max_background_threads"), CTL(max_background_threads)}, 898 {NAME("thread"), CHILD(named, thread)}, 899 {NAME("config"), CHILD(named, config)}, 900 {NAME("opt"), CHILD(named, opt)}, 901 {NAME("tcache"), CHILD(named, tcache)}, 902 {NAME("arena"), CHILD(indexed, arena)}, 903 {NAME("arenas"), CHILD(named, arenas)}, 904 {NAME("prof"), CHILD(named, prof)}, 905 {NAME("stats"), CHILD(named, stats)}, 906 {NAME("experimental"), CHILD(named, experimental)} 907 }; 908 static const ctl_named_node_t super_root_node[] = { 909 {NAME(""), CHILD(named, root)} 910 }; 911 912 #undef NAME 913 #undef CHILD 914 #undef CTL 915 #undef INDEX 916 917 /******************************************************************************/ 918 919 /* 920 * Sets *dst + *src non-atomically. This is safe, since everything is 921 * synchronized by the ctl mutex. 922 */ 923 static void 924 ctl_accum_locked_u64(locked_u64_t *dst, locked_u64_t *src) { 925 locked_inc_u64_unsynchronized(dst, 926 locked_read_u64_unsynchronized(src)); 927 } 928 929 static void 930 ctl_accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { 931 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); 932 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED); 933 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED); 934 } 935 936 /******************************************************************************/ 937 938 static unsigned 939 arenas_i2a_impl(size_t i, bool compat, bool validate) { 940 unsigned a; 941 942 switch (i) { 943 case MALLCTL_ARENAS_ALL: 944 a = 0; 945 break; 946 case MALLCTL_ARENAS_DESTROYED: 947 a = 1; 948 break; 949 default: 950 if (compat && i == ctl_arenas->narenas) { 951 /* 952 * Provide deprecated backward compatibility for 953 * accessing the merged stats at index narenas rather 954 * than via MALLCTL_ARENAS_ALL. This is scheduled for 955 * removal in 6.0.0. 956 */ 957 a = 0; 958 } else if (validate && i >= ctl_arenas->narenas) { 959 a = UINT_MAX; 960 } else { 961 /* 962 * This function should never be called for an index 963 * more than one past the range of indices that have 964 * initialized ctl data. 965 */ 966 assert(i < ctl_arenas->narenas || (!validate && i == 967 ctl_arenas->narenas)); 968 a = (unsigned)i + 2; 969 } 970 break; 971 } 972 973 return a; 974 } 975 976 static unsigned 977 arenas_i2a(size_t i) { 978 return arenas_i2a_impl(i, true, false); 979 } 980 981 static ctl_arena_t * 982 arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { 983 ctl_arena_t *ret; 984 985 assert(!compat || !init); 986 987 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)]; 988 if (init && ret == NULL) { 989 if (config_stats) { 990 struct container_s { 991 ctl_arena_t ctl_arena; 992 ctl_arena_stats_t astats; 993 }; 994 struct container_s *cont = 995 (struct container_s *)base_alloc(tsd_tsdn(tsd), 996 b0get(), sizeof(struct container_s), QUANTUM); 997 if (cont == NULL) { 998 return NULL; 999 } 1000 ret = &cont->ctl_arena; 1001 ret->astats = &cont->astats; 1002 } else { 1003 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(), 1004 sizeof(ctl_arena_t), QUANTUM); 1005 if (ret == NULL) { 1006 return NULL; 1007 } 1008 } 1009 ret->arena_ind = (unsigned)i; 1010 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret; 1011 } 1012 1013 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i)); 1014 return ret; 1015 } 1016 1017 static ctl_arena_t * 1018 arenas_i(size_t i) { 1019 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false); 1020 assert(ret != NULL); 1021 return ret; 1022 } 1023 1024 static void 1025 ctl_arena_clear(ctl_arena_t *ctl_arena) { 1026 ctl_arena->nthreads = 0; 1027 ctl_arena->dss = dss_prec_names[dss_prec_limit]; 1028 ctl_arena->dirty_decay_ms = -1; 1029 ctl_arena->muzzy_decay_ms = -1; 1030 ctl_arena->pactive = 0; 1031 ctl_arena->pdirty = 0; 1032 ctl_arena->pmuzzy = 0; 1033 if (config_stats) { 1034 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); 1035 ctl_arena->astats->allocated_small = 0; 1036 ctl_arena->astats->nmalloc_small = 0; 1037 ctl_arena->astats->ndalloc_small = 0; 1038 ctl_arena->astats->nrequests_small = 0; 1039 ctl_arena->astats->nfills_small = 0; 1040 ctl_arena->astats->nflushes_small = 0; 1041 memset(ctl_arena->astats->bstats, 0, SC_NBINS * 1042 sizeof(bin_stats_data_t)); 1043 memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * 1044 sizeof(arena_stats_large_t)); 1045 memset(ctl_arena->astats->estats, 0, SC_NPSIZES * 1046 sizeof(pac_estats_t)); 1047 memset(&ctl_arena->astats->hpastats, 0, 1048 sizeof(hpa_shard_stats_t)); 1049 memset(&ctl_arena->astats->secstats, 0, 1050 sizeof(sec_stats_t)); 1051 } 1052 } 1053 1054 static void 1055 ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { 1056 unsigned i; 1057 1058 if (config_stats) { 1059 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, 1060 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 1061 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 1062 &ctl_arena->pdirty, &ctl_arena->pmuzzy, 1063 &ctl_arena->astats->astats, ctl_arena->astats->bstats, 1064 ctl_arena->astats->lstats, ctl_arena->astats->estats, 1065 &ctl_arena->astats->hpastats, &ctl_arena->astats->secstats); 1066 1067 for (i = 0; i < SC_NBINS; i++) { 1068 bin_stats_t *bstats = 1069 &ctl_arena->astats->bstats[i].stats_data; 1070 ctl_arena->astats->allocated_small += bstats->curregs * 1071 sz_index2size(i); 1072 ctl_arena->astats->nmalloc_small += bstats->nmalloc; 1073 ctl_arena->astats->ndalloc_small += bstats->ndalloc; 1074 ctl_arena->astats->nrequests_small += bstats->nrequests; 1075 ctl_arena->astats->nfills_small += bstats->nfills; 1076 ctl_arena->astats->nflushes_small += bstats->nflushes; 1077 } 1078 } else { 1079 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, 1080 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 1081 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 1082 &ctl_arena->pdirty, &ctl_arena->pmuzzy); 1083 } 1084 } 1085 1086 static void 1087 ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, 1088 bool destroyed) { 1089 unsigned i; 1090 1091 if (!destroyed) { 1092 ctl_sdarena->nthreads += ctl_arena->nthreads; 1093 ctl_sdarena->pactive += ctl_arena->pactive; 1094 ctl_sdarena->pdirty += ctl_arena->pdirty; 1095 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy; 1096 } else { 1097 assert(ctl_arena->nthreads == 0); 1098 assert(ctl_arena->pactive == 0); 1099 assert(ctl_arena->pdirty == 0); 1100 assert(ctl_arena->pmuzzy == 0); 1101 } 1102 1103 if (config_stats) { 1104 ctl_arena_stats_t *sdstats = ctl_sdarena->astats; 1105 ctl_arena_stats_t *astats = ctl_arena->astats; 1106 1107 if (!destroyed) { 1108 sdstats->astats.mapped += astats->astats.mapped; 1109 sdstats->astats.pa_shard_stats.pac_stats.retained 1110 += astats->astats.pa_shard_stats.pac_stats.retained; 1111 sdstats->astats.pa_shard_stats.edata_avail 1112 += astats->astats.pa_shard_stats.edata_avail; 1113 } 1114 1115 ctl_accum_locked_u64( 1116 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge, 1117 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge); 1118 ctl_accum_locked_u64( 1119 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise, 1120 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise); 1121 ctl_accum_locked_u64( 1122 &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.purged, 1123 &astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged); 1124 1125 ctl_accum_locked_u64( 1126 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge, 1127 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge); 1128 ctl_accum_locked_u64( 1129 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise, 1130 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise); 1131 ctl_accum_locked_u64( 1132 &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged, 1133 &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged); 1134 1135 #define OP(mtx) malloc_mutex_prof_merge( \ 1136 &(sdstats->astats.mutex_prof_data[ \ 1137 arena_prof_mutex_##mtx]), \ 1138 &(astats->astats.mutex_prof_data[ \ 1139 arena_prof_mutex_##mtx])); 1140 MUTEX_PROF_ARENA_MUTEXES 1141 #undef OP 1142 if (!destroyed) { 1143 sdstats->astats.base += astats->astats.base; 1144 sdstats->astats.resident += astats->astats.resident; 1145 sdstats->astats.metadata_thp += astats->astats.metadata_thp; 1146 ctl_accum_atomic_zu(&sdstats->astats.internal, 1147 &astats->astats.internal); 1148 } else { 1149 assert(atomic_load_zu( 1150 &astats->astats.internal, ATOMIC_RELAXED) == 0); 1151 } 1152 1153 if (!destroyed) { 1154 sdstats->allocated_small += astats->allocated_small; 1155 } else { 1156 assert(astats->allocated_small == 0); 1157 } 1158 sdstats->nmalloc_small += astats->nmalloc_small; 1159 sdstats->ndalloc_small += astats->ndalloc_small; 1160 sdstats->nrequests_small += astats->nrequests_small; 1161 sdstats->nfills_small += astats->nfills_small; 1162 sdstats->nflushes_small += astats->nflushes_small; 1163 1164 if (!destroyed) { 1165 sdstats->astats.allocated_large += 1166 astats->astats.allocated_large; 1167 } else { 1168 assert(astats->astats.allocated_large == 0); 1169 } 1170 sdstats->astats.nmalloc_large += astats->astats.nmalloc_large; 1171 sdstats->astats.ndalloc_large += astats->astats.ndalloc_large; 1172 sdstats->astats.nrequests_large 1173 += astats->astats.nrequests_large; 1174 sdstats->astats.nflushes_large += astats->astats.nflushes_large; 1175 ctl_accum_atomic_zu( 1176 &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm, 1177 &astats->astats.pa_shard_stats.pac_stats.abandoned_vm); 1178 1179 sdstats->astats.tcache_bytes += astats->astats.tcache_bytes; 1180 sdstats->astats.tcache_stashed_bytes += 1181 astats->astats.tcache_stashed_bytes; 1182 1183 if (ctl_arena->arena_ind == 0) { 1184 sdstats->astats.uptime = astats->astats.uptime; 1185 } 1186 1187 /* Merge bin stats. */ 1188 for (i = 0; i < SC_NBINS; i++) { 1189 bin_stats_t *bstats = &astats->bstats[i].stats_data; 1190 bin_stats_t *merged = &sdstats->bstats[i].stats_data; 1191 merged->nmalloc += bstats->nmalloc; 1192 merged->ndalloc += bstats->ndalloc; 1193 merged->nrequests += bstats->nrequests; 1194 if (!destroyed) { 1195 merged->curregs += bstats->curregs; 1196 } else { 1197 assert(bstats->curregs == 0); 1198 } 1199 merged->nfills += bstats->nfills; 1200 merged->nflushes += bstats->nflushes; 1201 merged->nslabs += bstats->nslabs; 1202 merged->reslabs += bstats->reslabs; 1203 if (!destroyed) { 1204 merged->curslabs += bstats->curslabs; 1205 merged->nonfull_slabs += bstats->nonfull_slabs; 1206 } else { 1207 assert(bstats->curslabs == 0); 1208 assert(bstats->nonfull_slabs == 0); 1209 } 1210 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, 1211 &astats->bstats[i].mutex_data); 1212 } 1213 1214 /* Merge stats for large allocations. */ 1215 for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { 1216 ctl_accum_locked_u64(&sdstats->lstats[i].nmalloc, 1217 &astats->lstats[i].nmalloc); 1218 ctl_accum_locked_u64(&sdstats->lstats[i].ndalloc, 1219 &astats->lstats[i].ndalloc); 1220 ctl_accum_locked_u64(&sdstats->lstats[i].nrequests, 1221 &astats->lstats[i].nrequests); 1222 if (!destroyed) { 1223 sdstats->lstats[i].curlextents += 1224 astats->lstats[i].curlextents; 1225 } else { 1226 assert(astats->lstats[i].curlextents == 0); 1227 } 1228 } 1229 1230 /* Merge extents stats. */ 1231 for (i = 0; i < SC_NPSIZES; i++) { 1232 sdstats->estats[i].ndirty += astats->estats[i].ndirty; 1233 sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy; 1234 sdstats->estats[i].nretained 1235 += astats->estats[i].nretained; 1236 sdstats->estats[i].dirty_bytes 1237 += astats->estats[i].dirty_bytes; 1238 sdstats->estats[i].muzzy_bytes 1239 += astats->estats[i].muzzy_bytes; 1240 sdstats->estats[i].retained_bytes 1241 += astats->estats[i].retained_bytes; 1242 } 1243 1244 /* Merge HPA stats. */ 1245 hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats); 1246 sec_stats_accum(&sdstats->secstats, &astats->secstats); 1247 } 1248 } 1249 1250 static void 1251 ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena, 1252 unsigned i, bool destroyed) { 1253 ctl_arena_t *ctl_arena = arenas_i(i); 1254 1255 ctl_arena_clear(ctl_arena); 1256 ctl_arena_stats_amerge(tsdn, ctl_arena, arena); 1257 /* Merge into sum stats as well. */ 1258 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed); 1259 } 1260 1261 static unsigned 1262 ctl_arena_init(tsd_t *tsd, const arena_config_t *config) { 1263 unsigned arena_ind; 1264 ctl_arena_t *ctl_arena; 1265 1266 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) != 1267 NULL) { 1268 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 1269 arena_ind = ctl_arena->arena_ind; 1270 } else { 1271 arena_ind = ctl_arenas->narenas; 1272 } 1273 1274 /* Trigger stats allocation. */ 1275 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) { 1276 return UINT_MAX; 1277 } 1278 1279 /* Initialize new arena. */ 1280 if (arena_init(tsd_tsdn(tsd), arena_ind, config) == NULL) { 1281 return UINT_MAX; 1282 } 1283 1284 if (arena_ind == ctl_arenas->narenas) { 1285 ctl_arenas->narenas++; 1286 } 1287 1288 return arena_ind; 1289 } 1290 1291 static void 1292 ctl_background_thread_stats_read(tsdn_t *tsdn) { 1293 background_thread_stats_t *stats = &ctl_stats->background_thread; 1294 if (!have_background_thread || 1295 background_thread_stats_read(tsdn, stats)) { 1296 memset(stats, 0, sizeof(background_thread_stats_t)); 1297 nstime_init_zero(&stats->run_interval); 1298 } 1299 malloc_mutex_prof_copy( 1300 &ctl_stats->mutex_prof_data[global_prof_mutex_max_per_bg_thd], 1301 &stats->max_counter_per_bg_thd); 1302 } 1303 1304 static void 1305 ctl_refresh(tsdn_t *tsdn) { 1306 unsigned i; 1307 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL); 1308 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas); 1309 1310 /* 1311 * Clear sum stats, since they will be merged into by 1312 * ctl_arena_refresh(). 1313 */ 1314 ctl_arena_clear(ctl_sarena); 1315 1316 for (i = 0; i < ctl_arenas->narenas; i++) { 1317 tarenas[i] = arena_get(tsdn, i, false); 1318 } 1319 1320 for (i = 0; i < ctl_arenas->narenas; i++) { 1321 ctl_arena_t *ctl_arena = arenas_i(i); 1322 bool initialized = (tarenas[i] != NULL); 1323 1324 ctl_arena->initialized = initialized; 1325 if (initialized) { 1326 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i, 1327 false); 1328 } 1329 } 1330 1331 if (config_stats) { 1332 ctl_stats->allocated = ctl_sarena->astats->allocated_small + 1333 ctl_sarena->astats->astats.allocated_large; 1334 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE); 1335 ctl_stats->metadata = ctl_sarena->astats->astats.base + 1336 atomic_load_zu(&ctl_sarena->astats->astats.internal, 1337 ATOMIC_RELAXED); 1338 ctl_stats->resident = ctl_sarena->astats->astats.resident; 1339 ctl_stats->metadata_thp = 1340 ctl_sarena->astats->astats.metadata_thp; 1341 ctl_stats->mapped = ctl_sarena->astats->astats.mapped; 1342 ctl_stats->retained = ctl_sarena->astats->astats 1343 .pa_shard_stats.pac_stats.retained; 1344 1345 ctl_background_thread_stats_read(tsdn); 1346 1347 #define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ 1348 malloc_mutex_lock(tsdn, &mtx); \ 1349 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ 1350 malloc_mutex_unlock(tsdn, &mtx); 1351 1352 if (config_prof && opt_prof) { 1353 READ_GLOBAL_MUTEX_PROF_DATA( 1354 global_prof_mutex_prof, bt2gctx_mtx); 1355 READ_GLOBAL_MUTEX_PROF_DATA( 1356 global_prof_mutex_prof_thds_data, tdatas_mtx); 1357 READ_GLOBAL_MUTEX_PROF_DATA( 1358 global_prof_mutex_prof_dump, prof_dump_mtx); 1359 READ_GLOBAL_MUTEX_PROF_DATA( 1360 global_prof_mutex_prof_recent_alloc, 1361 prof_recent_alloc_mtx); 1362 READ_GLOBAL_MUTEX_PROF_DATA( 1363 global_prof_mutex_prof_recent_dump, 1364 prof_recent_dump_mtx); 1365 READ_GLOBAL_MUTEX_PROF_DATA( 1366 global_prof_mutex_prof_stats, prof_stats_mtx); 1367 } 1368 if (have_background_thread) { 1369 READ_GLOBAL_MUTEX_PROF_DATA( 1370 global_prof_mutex_background_thread, 1371 background_thread_lock); 1372 } else { 1373 memset(&ctl_stats->mutex_prof_data[ 1374 global_prof_mutex_background_thread], 0, 1375 sizeof(mutex_prof_data_t)); 1376 } 1377 /* We own ctl mutex already. */ 1378 malloc_mutex_prof_read(tsdn, 1379 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl], 1380 &ctl_mtx); 1381 #undef READ_GLOBAL_MUTEX_PROF_DATA 1382 } 1383 ctl_arenas->epoch++; 1384 } 1385 1386 static bool 1387 ctl_init(tsd_t *tsd) { 1388 bool ret; 1389 tsdn_t *tsdn = tsd_tsdn(tsd); 1390 1391 malloc_mutex_lock(tsdn, &ctl_mtx); 1392 if (!ctl_initialized) { 1393 ctl_arena_t *ctl_sarena, *ctl_darena; 1394 unsigned i; 1395 1396 /* 1397 * Allocate demand-zeroed space for pointers to the full 1398 * range of supported arena indices. 1399 */ 1400 if (ctl_arenas == NULL) { 1401 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn, 1402 b0get(), sizeof(ctl_arenas_t), QUANTUM); 1403 if (ctl_arenas == NULL) { 1404 ret = true; 1405 goto label_return; 1406 } 1407 } 1408 1409 if (config_stats && ctl_stats == NULL) { 1410 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(), 1411 sizeof(ctl_stats_t), QUANTUM); 1412 if (ctl_stats == NULL) { 1413 ret = true; 1414 goto label_return; 1415 } 1416 } 1417 1418 /* 1419 * Allocate space for the current full range of arenas 1420 * here rather than doing it lazily elsewhere, in order 1421 * to limit when OOM-caused errors can occur. 1422 */ 1423 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, 1424 true)) == NULL) { 1425 ret = true; 1426 goto label_return; 1427 } 1428 ctl_sarena->initialized = true; 1429 1430 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED, 1431 false, true)) == NULL) { 1432 ret = true; 1433 goto label_return; 1434 } 1435 ctl_arena_clear(ctl_darena); 1436 /* 1437 * Don't toggle ctl_darena to initialized until an arena is 1438 * actually destroyed, so that arena.<i>.initialized can be used 1439 * to query whether the stats are relevant. 1440 */ 1441 1442 ctl_arenas->narenas = narenas_total_get(); 1443 for (i = 0; i < ctl_arenas->narenas; i++) { 1444 if (arenas_i_impl(tsd, i, false, true) == NULL) { 1445 ret = true; 1446 goto label_return; 1447 } 1448 } 1449 1450 ql_new(&ctl_arenas->destroyed); 1451 ctl_refresh(tsdn); 1452 1453 ctl_initialized = true; 1454 } 1455 1456 ret = false; 1457 label_return: 1458 malloc_mutex_unlock(tsdn, &ctl_mtx); 1459 return ret; 1460 } 1461 1462 static int 1463 ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node, 1464 const char *name, const ctl_named_node_t **ending_nodep, size_t *mibp, 1465 size_t *depthp) { 1466 int ret; 1467 const char *elm, *tdot, *dot; 1468 size_t elen, i, j; 1469 const ctl_named_node_t *node; 1470 1471 elm = name; 1472 /* Equivalent to strchrnul(). */ 1473 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); 1474 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1475 if (elen == 0) { 1476 ret = ENOENT; 1477 goto label_return; 1478 } 1479 node = starting_node; 1480 for (i = 0; i < *depthp; i++) { 1481 assert(node); 1482 assert(node->nchildren > 0); 1483 if (ctl_named_node(node->children) != NULL) { 1484 const ctl_named_node_t *pnode = node; 1485 1486 /* Children are named. */ 1487 for (j = 0; j < node->nchildren; j++) { 1488 const ctl_named_node_t *child = 1489 ctl_named_children(node, j); 1490 if (strlen(child->name) == elen && 1491 strncmp(elm, child->name, elen) == 0) { 1492 node = child; 1493 mibp[i] = j; 1494 break; 1495 } 1496 } 1497 if (node == pnode) { 1498 ret = ENOENT; 1499 goto label_return; 1500 } 1501 } else { 1502 uintmax_t index; 1503 const ctl_indexed_node_t *inode; 1504 1505 /* Children are indexed. */ 1506 index = malloc_strtoumax(elm, NULL, 10); 1507 if (index == UINTMAX_MAX || index > SIZE_T_MAX) { 1508 ret = ENOENT; 1509 goto label_return; 1510 } 1511 1512 inode = ctl_indexed_node(node->children); 1513 node = inode->index(tsdn, mibp, *depthp, (size_t)index); 1514 if (node == NULL) { 1515 ret = ENOENT; 1516 goto label_return; 1517 } 1518 1519 mibp[i] = (size_t)index; 1520 } 1521 1522 /* Reached the end? */ 1523 if (node->ctl != NULL || *dot == '\0') { 1524 /* Terminal node. */ 1525 if (*dot != '\0') { 1526 /* 1527 * The name contains more elements than are 1528 * in this path through the tree. 1529 */ 1530 ret = ENOENT; 1531 goto label_return; 1532 } 1533 /* Complete lookup successful. */ 1534 *depthp = i + 1; 1535 break; 1536 } 1537 1538 /* Update elm. */ 1539 elm = &dot[1]; 1540 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : 1541 strchr(elm, '\0'); 1542 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1543 } 1544 if (ending_nodep != NULL) { 1545 *ending_nodep = node; 1546 } 1547 1548 ret = 0; 1549 label_return: 1550 return ret; 1551 } 1552 1553 int 1554 ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, 1555 void *newp, size_t newlen) { 1556 int ret; 1557 size_t depth; 1558 size_t mib[CTL_MAX_DEPTH]; 1559 const ctl_named_node_t *node; 1560 1561 if (!ctl_initialized && ctl_init(tsd)) { 1562 ret = EAGAIN; 1563 goto label_return; 1564 } 1565 1566 depth = CTL_MAX_DEPTH; 1567 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, &node, mib, 1568 &depth); 1569 if (ret != 0) { 1570 goto label_return; 1571 } 1572 1573 if (node != NULL && node->ctl) { 1574 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen); 1575 } else { 1576 /* The name refers to a partial path through the ctl tree. */ 1577 ret = ENOENT; 1578 } 1579 1580 label_return: 1581 return(ret); 1582 } 1583 1584 int 1585 ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) { 1586 int ret; 1587 1588 if (!ctl_initialized && ctl_init(tsd)) { 1589 ret = EAGAIN; 1590 goto label_return; 1591 } 1592 1593 ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, NULL, mibp, 1594 miblenp); 1595 label_return: 1596 return(ret); 1597 } 1598 1599 static int 1600 ctl_lookupbymib(tsdn_t *tsdn, const ctl_named_node_t **ending_nodep, 1601 const size_t *mib, size_t miblen) { 1602 int ret; 1603 1604 const ctl_named_node_t *node = super_root_node; 1605 for (size_t i = 0; i < miblen; i++) { 1606 assert(node); 1607 assert(node->nchildren > 0); 1608 if (ctl_named_node(node->children) != NULL) { 1609 /* Children are named. */ 1610 if (node->nchildren <= mib[i]) { 1611 ret = ENOENT; 1612 goto label_return; 1613 } 1614 node = ctl_named_children(node, mib[i]); 1615 } else { 1616 const ctl_indexed_node_t *inode; 1617 1618 /* Indexed element. */ 1619 inode = ctl_indexed_node(node->children); 1620 node = inode->index(tsdn, mib, miblen, mib[i]); 1621 if (node == NULL) { 1622 ret = ENOENT; 1623 goto label_return; 1624 } 1625 } 1626 } 1627 assert(ending_nodep != NULL); 1628 *ending_nodep = node; 1629 ret = 0; 1630 1631 label_return: 1632 return(ret); 1633 } 1634 1635 int 1636 ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1637 size_t *oldlenp, void *newp, size_t newlen) { 1638 int ret; 1639 const ctl_named_node_t *node; 1640 1641 if (!ctl_initialized && ctl_init(tsd)) { 1642 ret = EAGAIN; 1643 goto label_return; 1644 } 1645 1646 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen); 1647 if (ret != 0) { 1648 goto label_return; 1649 } 1650 1651 /* Call the ctl function. */ 1652 if (node && node->ctl) { 1653 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 1654 } else { 1655 /* Partial MIB. */ 1656 ret = ENOENT; 1657 } 1658 1659 label_return: 1660 return(ret); 1661 } 1662 1663 int 1664 ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, 1665 size_t *miblenp) { 1666 int ret; 1667 const ctl_named_node_t *node; 1668 1669 if (!ctl_initialized && ctl_init(tsd)) { 1670 ret = EAGAIN; 1671 goto label_return; 1672 } 1673 1674 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen); 1675 if (ret != 0) { 1676 goto label_return; 1677 } 1678 if (node == NULL || node->ctl != NULL) { 1679 ret = ENOENT; 1680 goto label_return; 1681 } 1682 1683 assert(miblenp != NULL); 1684 assert(*miblenp >= miblen); 1685 *miblenp -= miblen; 1686 ret = ctl_lookup(tsd_tsdn(tsd), node, name, NULL, mib + miblen, 1687 miblenp); 1688 *miblenp += miblen; 1689 label_return: 1690 return(ret); 1691 } 1692 1693 int 1694 ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, 1695 size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1696 int ret; 1697 const ctl_named_node_t *node; 1698 1699 if (!ctl_initialized && ctl_init(tsd)) { 1700 ret = EAGAIN; 1701 goto label_return; 1702 } 1703 1704 ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen); 1705 if (ret != 0) { 1706 goto label_return; 1707 } 1708 if (node == NULL || node->ctl != NULL) { 1709 ret = ENOENT; 1710 goto label_return; 1711 } 1712 1713 assert(miblenp != NULL); 1714 assert(*miblenp >= miblen); 1715 *miblenp -= miblen; 1716 /* 1717 * The same node supplies the starting node and stores the ending node. 1718 */ 1719 ret = ctl_lookup(tsd_tsdn(tsd), node, name, &node, mib + miblen, 1720 miblenp); 1721 *miblenp += miblen; 1722 if (ret != 0) { 1723 goto label_return; 1724 } 1725 1726 if (node != NULL && node->ctl) { 1727 ret = node->ctl(tsd, mib, *miblenp, oldp, oldlenp, newp, 1728 newlen); 1729 } else { 1730 /* The name refers to a partial path through the ctl tree. */ 1731 ret = ENOENT; 1732 } 1733 1734 label_return: 1735 return(ret); 1736 } 1737 1738 bool 1739 ctl_boot(void) { 1740 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, 1741 malloc_mutex_rank_exclusive)) { 1742 return true; 1743 } 1744 1745 ctl_initialized = false; 1746 1747 return false; 1748 } 1749 1750 void 1751 ctl_prefork(tsdn_t *tsdn) { 1752 malloc_mutex_prefork(tsdn, &ctl_mtx); 1753 } 1754 1755 void 1756 ctl_postfork_parent(tsdn_t *tsdn) { 1757 malloc_mutex_postfork_parent(tsdn, &ctl_mtx); 1758 } 1759 1760 void 1761 ctl_postfork_child(tsdn_t *tsdn) { 1762 malloc_mutex_postfork_child(tsdn, &ctl_mtx); 1763 } 1764 1765 void 1766 ctl_mtx_assert_held(tsdn_t *tsdn) { 1767 malloc_mutex_assert_owner(tsdn, &ctl_mtx); 1768 } 1769 1770 /******************************************************************************/ 1771 /* *_ctl() functions. */ 1772 1773 #define READONLY() do { \ 1774 if (newp != NULL || newlen != 0) { \ 1775 ret = EPERM; \ 1776 goto label_return; \ 1777 } \ 1778 } while (0) 1779 1780 #define WRITEONLY() do { \ 1781 if (oldp != NULL || oldlenp != NULL) { \ 1782 ret = EPERM; \ 1783 goto label_return; \ 1784 } \ 1785 } while (0) 1786 1787 /* Can read or write, but not both. */ 1788 #define READ_XOR_WRITE() do { \ 1789 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ 1790 newlen != 0)) { \ 1791 ret = EPERM; \ 1792 goto label_return; \ 1793 } \ 1794 } while (0) 1795 1796 /* Can neither read nor write. */ 1797 #define NEITHER_READ_NOR_WRITE() do { \ 1798 if (oldp != NULL || oldlenp != NULL || newp != NULL || \ 1799 newlen != 0) { \ 1800 ret = EPERM; \ 1801 goto label_return; \ 1802 } \ 1803 } while (0) 1804 1805 /* Verify that the space provided is enough. */ 1806 #define VERIFY_READ(t) do { \ 1807 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(t)) { \ 1808 *oldlenp = 0; \ 1809 ret = EINVAL; \ 1810 goto label_return; \ 1811 } \ 1812 } while (0) 1813 1814 #define READ(v, t) do { \ 1815 if (oldp != NULL && oldlenp != NULL) { \ 1816 if (*oldlenp != sizeof(t)) { \ 1817 size_t copylen = (sizeof(t) <= *oldlenp) \ 1818 ? sizeof(t) : *oldlenp; \ 1819 memcpy(oldp, (void *)&(v), copylen); \ 1820 *oldlenp = copylen; \ 1821 ret = EINVAL; \ 1822 goto label_return; \ 1823 } \ 1824 *(t *)oldp = (v); \ 1825 } \ 1826 } while (0) 1827 1828 #define WRITE(v, t) do { \ 1829 if (newp != NULL) { \ 1830 if (newlen != sizeof(t)) { \ 1831 ret = EINVAL; \ 1832 goto label_return; \ 1833 } \ 1834 (v) = *(t *)newp; \ 1835 } \ 1836 } while (0) 1837 1838 #define ASSURED_WRITE(v, t) do { \ 1839 if (newp == NULL || newlen != sizeof(t)) { \ 1840 ret = EINVAL; \ 1841 goto label_return; \ 1842 } \ 1843 (v) = *(t *)newp; \ 1844 } while (0) 1845 1846 #define MIB_UNSIGNED(v, i) do { \ 1847 if (mib[i] > UINT_MAX) { \ 1848 ret = EFAULT; \ 1849 goto label_return; \ 1850 } \ 1851 v = (unsigned)mib[i]; \ 1852 } while (0) 1853 1854 /* 1855 * There's a lot of code duplication in the following macros due to limitations 1856 * in how nested cpp macros are expanded. 1857 */ 1858 #define CTL_RO_CLGEN(c, l, n, v, t) \ 1859 static int \ 1860 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1861 size_t *oldlenp, void *newp, size_t newlen) { \ 1862 int ret; \ 1863 t oldval; \ 1864 \ 1865 if (!(c)) { \ 1866 return ENOENT; \ 1867 } \ 1868 if (l) { \ 1869 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1870 } \ 1871 READONLY(); \ 1872 oldval = (v); \ 1873 READ(oldval, t); \ 1874 \ 1875 ret = 0; \ 1876 label_return: \ 1877 if (l) { \ 1878 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1879 } \ 1880 return ret; \ 1881 } 1882 1883 #define CTL_RO_CGEN(c, n, v, t) \ 1884 static int \ 1885 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1886 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1887 int ret; \ 1888 t oldval; \ 1889 \ 1890 if (!(c)) { \ 1891 return ENOENT; \ 1892 } \ 1893 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1894 READONLY(); \ 1895 oldval = (v); \ 1896 READ(oldval, t); \ 1897 \ 1898 ret = 0; \ 1899 label_return: \ 1900 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1901 return ret; \ 1902 } 1903 1904 #define CTL_RO_GEN(n, v, t) \ 1905 static int \ 1906 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1907 size_t *oldlenp, void *newp, size_t newlen) { \ 1908 int ret; \ 1909 t oldval; \ 1910 \ 1911 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1912 READONLY(); \ 1913 oldval = (v); \ 1914 READ(oldval, t); \ 1915 \ 1916 ret = 0; \ 1917 label_return: \ 1918 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1919 return ret; \ 1920 } 1921 1922 /* 1923 * ctl_mtx is not acquired, under the assumption that no pertinent data will 1924 * mutate during the call. 1925 */ 1926 #define CTL_RO_NL_CGEN(c, n, v, t) \ 1927 static int \ 1928 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1929 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1930 int ret; \ 1931 t oldval; \ 1932 \ 1933 if (!(c)) { \ 1934 return ENOENT; \ 1935 } \ 1936 READONLY(); \ 1937 oldval = (v); \ 1938 READ(oldval, t); \ 1939 \ 1940 ret = 0; \ 1941 label_return: \ 1942 return ret; \ 1943 } 1944 1945 #define CTL_RO_NL_GEN(n, v, t) \ 1946 static int \ 1947 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1948 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1949 int ret; \ 1950 t oldval; \ 1951 \ 1952 READONLY(); \ 1953 oldval = (v); \ 1954 READ(oldval, t); \ 1955 \ 1956 ret = 0; \ 1957 label_return: \ 1958 return ret; \ 1959 } 1960 1961 #define CTL_RO_CONFIG_GEN(n, t) \ 1962 static int \ 1963 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1964 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1965 int ret; \ 1966 t oldval; \ 1967 \ 1968 READONLY(); \ 1969 oldval = n; \ 1970 READ(oldval, t); \ 1971 \ 1972 ret = 0; \ 1973 label_return: \ 1974 return ret; \ 1975 } 1976 1977 /******************************************************************************/ 1978 1979 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) 1980 1981 static int 1982 epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1983 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1984 int ret; 1985 UNUSED uint64_t newval; 1986 1987 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1988 WRITE(newval, uint64_t); 1989 if (newp != NULL) { 1990 ctl_refresh(tsd_tsdn(tsd)); 1991 } 1992 READ(ctl_arenas->epoch, uint64_t); 1993 1994 ret = 0; 1995 label_return: 1996 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1997 return ret; 1998 } 1999 2000 static int 2001 background_thread_ctl(tsd_t *tsd, const size_t *mib, 2002 size_t miblen, void *oldp, size_t *oldlenp, 2003 void *newp, size_t newlen) { 2004 int ret; 2005 bool oldval; 2006 2007 if (!have_background_thread) { 2008 return ENOENT; 2009 } 2010 background_thread_ctl_init(tsd_tsdn(tsd)); 2011 2012 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2013 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 2014 if (newp == NULL) { 2015 oldval = background_thread_enabled(); 2016 READ(oldval, bool); 2017 } else { 2018 if (newlen != sizeof(bool)) { 2019 ret = EINVAL; 2020 goto label_return; 2021 } 2022 oldval = background_thread_enabled(); 2023 READ(oldval, bool); 2024 2025 bool newval = *(bool *)newp; 2026 if (newval == oldval) { 2027 ret = 0; 2028 goto label_return; 2029 } 2030 2031 background_thread_enabled_set(tsd_tsdn(tsd), newval); 2032 if (newval) { 2033 if (background_threads_enable(tsd)) { 2034 ret = EFAULT; 2035 goto label_return; 2036 } 2037 } else { 2038 if (background_threads_disable(tsd)) { 2039 ret = EFAULT; 2040 goto label_return; 2041 } 2042 } 2043 } 2044 ret = 0; 2045 label_return: 2046 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 2047 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2048 2049 return ret; 2050 } 2051 2052 static int 2053 max_background_threads_ctl(tsd_t *tsd, const size_t *mib, 2054 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2055 size_t newlen) { 2056 int ret; 2057 size_t oldval; 2058 2059 if (!have_background_thread) { 2060 return ENOENT; 2061 } 2062 background_thread_ctl_init(tsd_tsdn(tsd)); 2063 2064 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2065 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 2066 if (newp == NULL) { 2067 oldval = max_background_threads; 2068 READ(oldval, size_t); 2069 } else { 2070 if (newlen != sizeof(size_t)) { 2071 ret = EINVAL; 2072 goto label_return; 2073 } 2074 oldval = max_background_threads; 2075 READ(oldval, size_t); 2076 2077 size_t newval = *(size_t *)newp; 2078 if (newval == oldval) { 2079 ret = 0; 2080 goto label_return; 2081 } 2082 if (newval > opt_max_background_threads) { 2083 ret = EINVAL; 2084 goto label_return; 2085 } 2086 2087 if (background_thread_enabled()) { 2088 background_thread_enabled_set(tsd_tsdn(tsd), false); 2089 if (background_threads_disable(tsd)) { 2090 ret = EFAULT; 2091 goto label_return; 2092 } 2093 max_background_threads = newval; 2094 background_thread_enabled_set(tsd_tsdn(tsd), true); 2095 if (background_threads_enable(tsd)) { 2096 ret = EFAULT; 2097 goto label_return; 2098 } 2099 } else { 2100 max_background_threads = newval; 2101 } 2102 } 2103 ret = 0; 2104 label_return: 2105 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 2106 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2107 2108 return ret; 2109 } 2110 2111 /******************************************************************************/ 2112 2113 CTL_RO_CONFIG_GEN(config_cache_oblivious, bool) 2114 CTL_RO_CONFIG_GEN(config_debug, bool) 2115 CTL_RO_CONFIG_GEN(config_fill, bool) 2116 CTL_RO_CONFIG_GEN(config_lazy_lock, bool) 2117 CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) 2118 CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool) 2119 CTL_RO_CONFIG_GEN(config_prof, bool) 2120 CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) 2121 CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) 2122 CTL_RO_CONFIG_GEN(config_stats, bool) 2123 CTL_RO_CONFIG_GEN(config_utrace, bool) 2124 CTL_RO_CONFIG_GEN(config_xmalloc, bool) 2125 2126 /******************************************************************************/ 2127 2128 CTL_RO_NL_GEN(opt_abort, opt_abort, bool) 2129 CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) 2130 CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool) 2131 CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) 2132 CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) 2133 2134 /* HPA options. */ 2135 CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) 2136 CTL_RO_NL_GEN(opt_hpa_hugification_threshold, 2137 opt_hpa_opts.hugification_threshold, size_t) 2138 CTL_RO_NL_GEN(opt_hpa_hugify_delay_ms, opt_hpa_opts.hugify_delay_ms, uint64_t) 2139 CTL_RO_NL_GEN(opt_hpa_min_purge_interval_ms, opt_hpa_opts.min_purge_interval_ms, 2140 uint64_t) 2141 2142 /* 2143 * This will have to change before we publicly document this option; fxp_t and 2144 * its representation are internal implementation details. 2145 */ 2146 CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t) 2147 CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) 2148 2149 /* HPA SEC options */ 2150 CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t) 2151 CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t) 2152 CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t) 2153 CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, 2154 size_t) 2155 CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, 2156 size_t) 2157 2158 CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], 2159 const char *) 2160 CTL_RO_NL_GEN(opt_retain, opt_retain, bool) 2161 CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) 2162 CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) 2163 CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], 2164 const char *) 2165 CTL_RO_NL_GEN(opt_mutex_max_spin, opt_mutex_max_spin, int64_t) 2166 CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) 2167 CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) 2168 CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) 2169 CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) 2170 CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t) 2171 CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) 2172 CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *) 2173 CTL_RO_NL_GEN(opt_stats_interval, opt_stats_interval, int64_t) 2174 CTL_RO_NL_GEN(opt_stats_interval_opts, opt_stats_interval_opts, const char *) 2175 CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) 2176 CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) 2177 CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) 2178 CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) 2179 CTL_RO_NL_CGEN(config_enable_cxx, opt_experimental_infallible_new, 2180 opt_experimental_infallible_new, bool) 2181 CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) 2182 CTL_RO_NL_GEN(opt_tcache_max, opt_tcache_max, size_t) 2183 CTL_RO_NL_GEN(opt_tcache_nslots_small_min, opt_tcache_nslots_small_min, 2184 unsigned) 2185 CTL_RO_NL_GEN(opt_tcache_nslots_small_max, opt_tcache_nslots_small_max, 2186 unsigned) 2187 CTL_RO_NL_GEN(opt_tcache_nslots_large, opt_tcache_nslots_large, unsigned) 2188 CTL_RO_NL_GEN(opt_lg_tcache_nslots_mul, opt_lg_tcache_nslots_mul, ssize_t) 2189 CTL_RO_NL_GEN(opt_tcache_gc_incr_bytes, opt_tcache_gc_incr_bytes, size_t) 2190 CTL_RO_NL_GEN(opt_tcache_gc_delay_bytes, opt_tcache_gc_delay_bytes, size_t) 2191 CTL_RO_NL_GEN(opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div, 2192 unsigned) 2193 CTL_RO_NL_GEN(opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div, 2194 unsigned) 2195 CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) 2196 CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, 2197 size_t) 2198 CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) 2199 CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) 2200 CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) 2201 CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, 2202 opt_prof_thread_active_init, bool) 2203 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) 2204 CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) 2205 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) 2206 CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) 2207 CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) 2208 CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) 2209 CTL_RO_NL_CGEN(config_prof, opt_prof_leak_error, opt_prof_leak_error, bool) 2210 CTL_RO_NL_CGEN(config_prof, opt_prof_recent_alloc_max, 2211 opt_prof_recent_alloc_max, ssize_t) 2212 CTL_RO_NL_CGEN(config_prof, opt_prof_stats, opt_prof_stats, bool) 2213 CTL_RO_NL_CGEN(config_prof, opt_prof_sys_thread_name, opt_prof_sys_thread_name, 2214 bool) 2215 CTL_RO_NL_CGEN(config_prof, opt_prof_time_res, 2216 prof_time_res_mode_names[opt_prof_time_res], const char *) 2217 CTL_RO_NL_CGEN(config_uaf_detection, opt_lg_san_uaf_align, 2218 opt_lg_san_uaf_align, ssize_t) 2219 CTL_RO_NL_GEN(opt_zero_realloc, 2220 zero_realloc_mode_names[opt_zero_realloc_action], const char *) 2221 2222 /******************************************************************************/ 2223 2224 static int 2225 thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2226 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2227 int ret; 2228 arena_t *oldarena; 2229 unsigned newind, oldind; 2230 2231 oldarena = arena_choose(tsd, NULL); 2232 if (oldarena == NULL) { 2233 return EAGAIN; 2234 } 2235 newind = oldind = arena_ind_get(oldarena); 2236 WRITE(newind, unsigned); 2237 READ(oldind, unsigned); 2238 2239 if (newind != oldind) { 2240 arena_t *newarena; 2241 2242 if (newind >= narenas_total_get()) { 2243 /* New arena index is out of range. */ 2244 ret = EFAULT; 2245 goto label_return; 2246 } 2247 2248 if (have_percpu_arena && 2249 PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 2250 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) { 2251 /* 2252 * If perCPU arena is enabled, thread_arena 2253 * control is not allowed for the auto arena 2254 * range. 2255 */ 2256 ret = EPERM; 2257 goto label_return; 2258 } 2259 } 2260 2261 /* Initialize arena if necessary. */ 2262 newarena = arena_get(tsd_tsdn(tsd), newind, true); 2263 if (newarena == NULL) { 2264 ret = EAGAIN; 2265 goto label_return; 2266 } 2267 /* Set new arena/tcache associations. */ 2268 arena_migrate(tsd, oldarena, newarena); 2269 if (tcache_available(tsd)) { 2270 tcache_arena_reassociate(tsd_tsdn(tsd), 2271 tsd_tcache_slowp_get(tsd), tsd_tcachep_get(tsd), 2272 newarena); 2273 } 2274 } 2275 2276 ret = 0; 2277 label_return: 2278 return ret; 2279 } 2280 2281 CTL_RO_NL_GEN(thread_allocated, tsd_thread_allocated_get(tsd), uint64_t) 2282 CTL_RO_NL_GEN(thread_allocatedp, tsd_thread_allocatedp_get(tsd), uint64_t *) 2283 CTL_RO_NL_GEN(thread_deallocated, tsd_thread_deallocated_get(tsd), uint64_t) 2284 CTL_RO_NL_GEN(thread_deallocatedp, tsd_thread_deallocatedp_get(tsd), uint64_t *) 2285 2286 static int 2287 thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, 2288 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2289 size_t newlen) { 2290 int ret; 2291 bool oldval; 2292 2293 oldval = tcache_enabled_get(tsd); 2294 if (newp != NULL) { 2295 if (newlen != sizeof(bool)) { 2296 ret = EINVAL; 2297 goto label_return; 2298 } 2299 tcache_enabled_set(tsd, *(bool *)newp); 2300 } 2301 READ(oldval, bool); 2302 2303 ret = 0; 2304 label_return: 2305 return ret; 2306 } 2307 2308 static int 2309 thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, 2310 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2311 size_t newlen) { 2312 int ret; 2313 2314 if (!tcache_available(tsd)) { 2315 ret = EFAULT; 2316 goto label_return; 2317 } 2318 2319 NEITHER_READ_NOR_WRITE(); 2320 2321 tcache_flush(tsd); 2322 2323 ret = 0; 2324 label_return: 2325 return ret; 2326 } 2327 2328 static int 2329 thread_peak_read_ctl(tsd_t *tsd, const size_t *mib, 2330 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2331 size_t newlen) { 2332 int ret; 2333 if (!config_stats) { 2334 return ENOENT; 2335 } 2336 READONLY(); 2337 peak_event_update(tsd); 2338 uint64_t result = peak_event_max(tsd); 2339 READ(result, uint64_t); 2340 ret = 0; 2341 label_return: 2342 return ret; 2343 } 2344 2345 static int 2346 thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib, 2347 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2348 size_t newlen) { 2349 int ret; 2350 if (!config_stats) { 2351 return ENOENT; 2352 } 2353 NEITHER_READ_NOR_WRITE(); 2354 peak_event_zero(tsd); 2355 ret = 0; 2356 label_return: 2357 return ret; 2358 } 2359 2360 static int 2361 thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, 2362 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2363 size_t newlen) { 2364 int ret; 2365 2366 if (!config_prof || !opt_prof) { 2367 return ENOENT; 2368 } 2369 2370 READ_XOR_WRITE(); 2371 2372 if (newp != NULL) { 2373 if (newlen != sizeof(const char *)) { 2374 ret = EINVAL; 2375 goto label_return; 2376 } 2377 2378 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 2379 0) { 2380 goto label_return; 2381 } 2382 } else { 2383 const char *oldname = prof_thread_name_get(tsd); 2384 READ(oldname, const char *); 2385 } 2386 2387 ret = 0; 2388 label_return: 2389 return ret; 2390 } 2391 2392 static int 2393 thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, 2394 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2395 size_t newlen) { 2396 int ret; 2397 bool oldval; 2398 2399 if (!config_prof) { 2400 return ENOENT; 2401 } 2402 2403 oldval = opt_prof ? prof_thread_active_get(tsd) : false; 2404 if (newp != NULL) { 2405 if (!opt_prof) { 2406 ret = ENOENT; 2407 goto label_return; 2408 } 2409 if (newlen != sizeof(bool)) { 2410 ret = EINVAL; 2411 goto label_return; 2412 } 2413 if (prof_thread_active_set(tsd, *(bool *)newp)) { 2414 ret = EAGAIN; 2415 goto label_return; 2416 } 2417 } 2418 READ(oldval, bool); 2419 2420 ret = 0; 2421 label_return: 2422 return ret; 2423 } 2424 2425 static int 2426 thread_idle_ctl(tsd_t *tsd, const size_t *mib, 2427 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2428 size_t newlen) { 2429 int ret; 2430 2431 NEITHER_READ_NOR_WRITE(); 2432 2433 if (tcache_available(tsd)) { 2434 tcache_flush(tsd); 2435 } 2436 /* 2437 * This heuristic is perhaps not the most well-considered. But it 2438 * matches the only idling policy we have experience with in the status 2439 * quo. Over time we should investigate more principled approaches. 2440 */ 2441 if (opt_narenas > ncpus * 2) { 2442 arena_t *arena = arena_choose(tsd, NULL); 2443 if (arena != NULL) { 2444 arena_decay(tsd_tsdn(tsd), arena, false, true); 2445 } 2446 /* 2447 * The missing arena case is not actually an error; a thread 2448 * might be idle before it associates itself to one. This is 2449 * unusual, but not wrong. 2450 */ 2451 } 2452 2453 ret = 0; 2454 label_return: 2455 return ret; 2456 } 2457 2458 /******************************************************************************/ 2459 2460 static int 2461 tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2462 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2463 int ret; 2464 unsigned tcache_ind; 2465 2466 READONLY(); 2467 VERIFY_READ(unsigned); 2468 if (tcaches_create(tsd, b0get(), &tcache_ind)) { 2469 ret = EFAULT; 2470 goto label_return; 2471 } 2472 READ(tcache_ind, unsigned); 2473 2474 ret = 0; 2475 label_return: 2476 return ret; 2477 } 2478 2479 static int 2480 tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2481 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2482 int ret; 2483 unsigned tcache_ind; 2484 2485 WRITEONLY(); 2486 ASSURED_WRITE(tcache_ind, unsigned); 2487 tcaches_flush(tsd, tcache_ind); 2488 2489 ret = 0; 2490 label_return: 2491 return ret; 2492 } 2493 2494 static int 2495 tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2496 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2497 int ret; 2498 unsigned tcache_ind; 2499 2500 WRITEONLY(); 2501 ASSURED_WRITE(tcache_ind, unsigned); 2502 tcaches_destroy(tsd, tcache_ind); 2503 2504 ret = 0; 2505 label_return: 2506 return ret; 2507 } 2508 2509 /******************************************************************************/ 2510 2511 static int 2512 arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2513 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2514 int ret; 2515 tsdn_t *tsdn = tsd_tsdn(tsd); 2516 unsigned arena_ind; 2517 bool initialized; 2518 2519 READONLY(); 2520 MIB_UNSIGNED(arena_ind, 1); 2521 2522 malloc_mutex_lock(tsdn, &ctl_mtx); 2523 initialized = arenas_i(arena_ind)->initialized; 2524 malloc_mutex_unlock(tsdn, &ctl_mtx); 2525 2526 READ(initialized, bool); 2527 2528 ret = 0; 2529 label_return: 2530 return ret; 2531 } 2532 2533 static void 2534 arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { 2535 malloc_mutex_lock(tsdn, &ctl_mtx); 2536 { 2537 unsigned narenas = ctl_arenas->narenas; 2538 2539 /* 2540 * Access via index narenas is deprecated, and scheduled for 2541 * removal in 6.0.0. 2542 */ 2543 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) { 2544 unsigned i; 2545 VARIABLE_ARRAY(arena_t *, tarenas, narenas); 2546 2547 for (i = 0; i < narenas; i++) { 2548 tarenas[i] = arena_get(tsdn, i, false); 2549 } 2550 2551 /* 2552 * No further need to hold ctl_mtx, since narenas and 2553 * tarenas contain everything needed below. 2554 */ 2555 malloc_mutex_unlock(tsdn, &ctl_mtx); 2556 2557 for (i = 0; i < narenas; i++) { 2558 if (tarenas[i] != NULL) { 2559 arena_decay(tsdn, tarenas[i], false, 2560 all); 2561 } 2562 } 2563 } else { 2564 arena_t *tarena; 2565 2566 assert(arena_ind < narenas); 2567 2568 tarena = arena_get(tsdn, arena_ind, false); 2569 2570 /* No further need to hold ctl_mtx. */ 2571 malloc_mutex_unlock(tsdn, &ctl_mtx); 2572 2573 if (tarena != NULL) { 2574 arena_decay(tsdn, tarena, false, all); 2575 } 2576 } 2577 } 2578 } 2579 2580 static int 2581 arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2582 size_t *oldlenp, void *newp, size_t newlen) { 2583 int ret; 2584 unsigned arena_ind; 2585 2586 NEITHER_READ_NOR_WRITE(); 2587 MIB_UNSIGNED(arena_ind, 1); 2588 arena_i_decay(tsd_tsdn(tsd), arena_ind, false); 2589 2590 ret = 0; 2591 label_return: 2592 return ret; 2593 } 2594 2595 static int 2596 arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2597 size_t *oldlenp, void *newp, size_t newlen) { 2598 int ret; 2599 unsigned arena_ind; 2600 2601 NEITHER_READ_NOR_WRITE(); 2602 MIB_UNSIGNED(arena_ind, 1); 2603 arena_i_decay(tsd_tsdn(tsd), arena_ind, true); 2604 2605 ret = 0; 2606 label_return: 2607 return ret; 2608 } 2609 2610 static int 2611 arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen, 2612 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind, 2613 arena_t **arena) { 2614 int ret; 2615 2616 NEITHER_READ_NOR_WRITE(); 2617 MIB_UNSIGNED(*arena_ind, 1); 2618 2619 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false); 2620 if (*arena == NULL || arena_is_auto(*arena)) { 2621 ret = EFAULT; 2622 goto label_return; 2623 } 2624 2625 ret = 0; 2626 label_return: 2627 return ret; 2628 } 2629 2630 static void 2631 arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { 2632 /* Temporarily disable the background thread during arena reset. */ 2633 if (have_background_thread) { 2634 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 2635 if (background_thread_enabled()) { 2636 background_thread_info_t *info = 2637 background_thread_info_get(arena_ind); 2638 assert(info->state == background_thread_started); 2639 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2640 info->state = background_thread_paused; 2641 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2642 } 2643 } 2644 } 2645 2646 static void 2647 arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { 2648 if (have_background_thread) { 2649 if (background_thread_enabled()) { 2650 background_thread_info_t *info = 2651 background_thread_info_get(arena_ind); 2652 assert(info->state == background_thread_paused); 2653 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2654 info->state = background_thread_started; 2655 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2656 } 2657 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 2658 } 2659 } 2660 2661 static int 2662 arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2663 size_t *oldlenp, void *newp, size_t newlen) { 2664 int ret; 2665 unsigned arena_ind; 2666 arena_t *arena; 2667 2668 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2669 newp, newlen, &arena_ind, &arena); 2670 if (ret != 0) { 2671 return ret; 2672 } 2673 2674 arena_reset_prepare_background_thread(tsd, arena_ind); 2675 arena_reset(tsd, arena); 2676 arena_reset_finish_background_thread(tsd, arena_ind); 2677 2678 return ret; 2679 } 2680 2681 static int 2682 arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2683 size_t *oldlenp, void *newp, size_t newlen) { 2684 int ret; 2685 unsigned arena_ind; 2686 arena_t *arena; 2687 ctl_arena_t *ctl_darena, *ctl_arena; 2688 2689 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2690 2691 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2692 newp, newlen, &arena_ind, &arena); 2693 if (ret != 0) { 2694 goto label_return; 2695 } 2696 2697 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena, 2698 true) != 0) { 2699 ret = EFAULT; 2700 goto label_return; 2701 } 2702 2703 arena_reset_prepare_background_thread(tsd, arena_ind); 2704 /* Merge stats after resetting and purging arena. */ 2705 arena_reset(tsd, arena); 2706 arena_decay(tsd_tsdn(tsd), arena, false, true); 2707 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); 2708 ctl_darena->initialized = true; 2709 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); 2710 /* Destroy arena. */ 2711 arena_destroy(tsd, arena); 2712 ctl_arena = arenas_i(arena_ind); 2713 ctl_arena->initialized = false; 2714 /* Record arena index for later recycling via arenas.create. */ 2715 ql_elm_new(ctl_arena, destroyed_link); 2716 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 2717 arena_reset_finish_background_thread(tsd, arena_ind); 2718 2719 assert(ret == 0); 2720 label_return: 2721 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2722 2723 return ret; 2724 } 2725 2726 static int 2727 arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2728 size_t *oldlenp, void *newp, size_t newlen) { 2729 int ret; 2730 const char *dss = NULL; 2731 unsigned arena_ind; 2732 dss_prec_t dss_prec_old = dss_prec_limit; 2733 dss_prec_t dss_prec = dss_prec_limit; 2734 2735 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2736 WRITE(dss, const char *); 2737 MIB_UNSIGNED(arena_ind, 1); 2738 if (dss != NULL) { 2739 int i; 2740 bool match = false; 2741 2742 for (i = 0; i < dss_prec_limit; i++) { 2743 if (strcmp(dss_prec_names[i], dss) == 0) { 2744 dss_prec = i; 2745 match = true; 2746 break; 2747 } 2748 } 2749 2750 if (!match) { 2751 ret = EINVAL; 2752 goto label_return; 2753 } 2754 } 2755 2756 /* 2757 * Access via index narenas is deprecated, and scheduled for removal in 2758 * 6.0.0. 2759 */ 2760 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == 2761 ctl_arenas->narenas) { 2762 if (dss_prec != dss_prec_limit && 2763 extent_dss_prec_set(dss_prec)) { 2764 ret = EFAULT; 2765 goto label_return; 2766 } 2767 dss_prec_old = extent_dss_prec_get(); 2768 } else { 2769 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2770 if (arena == NULL || (dss_prec != dss_prec_limit && 2771 arena_dss_prec_set(arena, dss_prec))) { 2772 ret = EFAULT; 2773 goto label_return; 2774 } 2775 dss_prec_old = arena_dss_prec_get(arena); 2776 } 2777 2778 dss = dss_prec_names[dss_prec_old]; 2779 READ(dss, const char *); 2780 2781 ret = 0; 2782 label_return: 2783 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2784 return ret; 2785 } 2786 2787 static int 2788 arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2789 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2790 int ret; 2791 2792 unsigned arena_ind; 2793 MIB_UNSIGNED(arena_ind, 1); 2794 2795 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2796 if (arena == NULL) { 2797 ret = EFAULT; 2798 goto label_return; 2799 } 2800 2801 if (oldp != NULL && oldlenp != NULL) { 2802 size_t oldval = atomic_load_zu( 2803 &arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED); 2804 READ(oldval, size_t); 2805 } 2806 if (newp != NULL) { 2807 if (newlen != sizeof(size_t)) { 2808 ret = EINVAL; 2809 goto label_return; 2810 } 2811 atomic_store_zu(&arena->pa_shard.pac.oversize_threshold, 2812 *(size_t *)newp, ATOMIC_RELAXED); 2813 } 2814 ret = 0; 2815 label_return: 2816 return ret; 2817 } 2818 2819 static int 2820 arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2821 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2822 int ret; 2823 unsigned arena_ind; 2824 arena_t *arena; 2825 2826 MIB_UNSIGNED(arena_ind, 1); 2827 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2828 if (arena == NULL) { 2829 ret = EFAULT; 2830 goto label_return; 2831 } 2832 extent_state_t state = dirty ? extent_state_dirty : extent_state_muzzy; 2833 2834 if (oldp != NULL && oldlenp != NULL) { 2835 size_t oldval = arena_decay_ms_get(arena, state); 2836 READ(oldval, ssize_t); 2837 } 2838 if (newp != NULL) { 2839 if (newlen != sizeof(ssize_t)) { 2840 ret = EINVAL; 2841 goto label_return; 2842 } 2843 if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { 2844 /* 2845 * By default the huge arena purges eagerly. If it is 2846 * set to non-zero decay time afterwards, background 2847 * thread might be needed. 2848 */ 2849 if (background_thread_create(tsd, arena_ind)) { 2850 ret = EFAULT; 2851 goto label_return; 2852 } 2853 } 2854 2855 if (arena_decay_ms_set(tsd_tsdn(tsd), arena, state, 2856 *(ssize_t *)newp)) { 2857 ret = EFAULT; 2858 goto label_return; 2859 } 2860 } 2861 2862 ret = 0; 2863 label_return: 2864 return ret; 2865 } 2866 2867 static int 2868 arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2869 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2870 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2871 newlen, true); 2872 } 2873 2874 static int 2875 arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2876 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2877 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2878 newlen, false); 2879 } 2880 2881 static int 2882 arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2883 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2884 int ret; 2885 unsigned arena_ind; 2886 arena_t *arena; 2887 2888 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2889 MIB_UNSIGNED(arena_ind, 1); 2890 if (arena_ind < narenas_total_get()) { 2891 extent_hooks_t *old_extent_hooks; 2892 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2893 if (arena == NULL) { 2894 if (arena_ind >= narenas_auto) { 2895 ret = EFAULT; 2896 goto label_return; 2897 } 2898 old_extent_hooks = 2899 (extent_hooks_t *)__UNCONST(&ehooks_default_extent_hooks); 2900 READ(old_extent_hooks, extent_hooks_t *); 2901 if (newp != NULL) { 2902 /* Initialize a new arena as a side effect. */ 2903 extent_hooks_t *new_extent_hooks 2904 JEMALLOC_CC_SILENCE_INIT(NULL); 2905 WRITE(new_extent_hooks, extent_hooks_t *); 2906 arena_config_t config = arena_config_default; 2907 config.extent_hooks = new_extent_hooks; 2908 2909 arena = arena_init(tsd_tsdn(tsd), arena_ind, 2910 &config); 2911 if (arena == NULL) { 2912 ret = EFAULT; 2913 goto label_return; 2914 } 2915 } 2916 } else { 2917 if (newp != NULL) { 2918 extent_hooks_t *new_extent_hooks 2919 JEMALLOC_CC_SILENCE_INIT(NULL); 2920 WRITE(new_extent_hooks, extent_hooks_t *); 2921 old_extent_hooks = arena_set_extent_hooks(tsd, 2922 arena, new_extent_hooks); 2923 READ(old_extent_hooks, extent_hooks_t *); 2924 } else { 2925 old_extent_hooks = 2926 ehooks_get_extent_hooks_ptr( 2927 arena_get_ehooks(arena)); 2928 READ(old_extent_hooks, extent_hooks_t *); 2929 } 2930 } 2931 } else { 2932 ret = EFAULT; 2933 goto label_return; 2934 } 2935 ret = 0; 2936 label_return: 2937 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2938 return ret; 2939 } 2940 2941 static int 2942 arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, 2943 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2944 size_t newlen) { 2945 int ret; 2946 unsigned arena_ind; 2947 arena_t *arena; 2948 2949 if (!opt_retain) { 2950 /* Only relevant when retain is enabled. */ 2951 return ENOENT; 2952 } 2953 2954 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2955 MIB_UNSIGNED(arena_ind, 1); 2956 if (arena_ind < narenas_total_get() && (arena = 2957 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 2958 size_t old_limit, new_limit; 2959 if (newp != NULL) { 2960 WRITE(new_limit, size_t); 2961 } 2962 bool err = arena_retain_grow_limit_get_set(tsd, arena, 2963 &old_limit, newp != NULL ? &new_limit : NULL); 2964 if (!err) { 2965 READ(old_limit, size_t); 2966 ret = 0; 2967 } else { 2968 ret = EFAULT; 2969 } 2970 } else { 2971 ret = EFAULT; 2972 } 2973 label_return: 2974 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2975 return ret; 2976 } 2977 2978 static const ctl_named_node_t * 2979 arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2980 size_t i) { 2981 const ctl_named_node_t *ret; 2982 2983 malloc_mutex_lock(tsdn, &ctl_mtx); 2984 switch (i) { 2985 case MALLCTL_ARENAS_ALL: 2986 case MALLCTL_ARENAS_DESTROYED: 2987 break; 2988 default: 2989 if (i > ctl_arenas->narenas) { 2990 ret = NULL; 2991 goto label_return; 2992 } 2993 break; 2994 } 2995 2996 ret = super_arena_i_node; 2997 label_return: 2998 malloc_mutex_unlock(tsdn, &ctl_mtx); 2999 return ret; 3000 } 3001 3002 /******************************************************************************/ 3003 3004 static int 3005 arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3006 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3007 int ret; 3008 unsigned narenas; 3009 3010 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3011 READONLY(); 3012 narenas = ctl_arenas->narenas; 3013 READ(narenas, unsigned); 3014 3015 ret = 0; 3016 label_return: 3017 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3018 return ret; 3019 } 3020 3021 static int 3022 arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, 3023 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 3024 size_t newlen, bool dirty) { 3025 int ret; 3026 3027 if (oldp != NULL && oldlenp != NULL) { 3028 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() : 3029 arena_muzzy_decay_ms_default_get()); 3030 READ(oldval, ssize_t); 3031 } 3032 if (newp != NULL) { 3033 if (newlen != sizeof(ssize_t)) { 3034 ret = EINVAL; 3035 goto label_return; 3036 } 3037 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) 3038 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { 3039 ret = EFAULT; 3040 goto label_return; 3041 } 3042 } 3043 3044 ret = 0; 3045 label_return: 3046 return ret; 3047 } 3048 3049 static int 3050 arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3051 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3052 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 3053 newlen, true); 3054 } 3055 3056 static int 3057 arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3058 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3059 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 3060 newlen, false); 3061 } 3062 3063 CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) 3064 CTL_RO_NL_GEN(arenas_page, PAGE, size_t) 3065 CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) 3066 CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) 3067 CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) 3068 CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) 3069 CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) 3070 CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) 3071 CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) 3072 static const ctl_named_node_t * 3073 arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, 3074 size_t miblen, size_t i) { 3075 if (i > SC_NBINS) { 3076 return NULL; 3077 } 3078 return super_arenas_bin_i_node; 3079 } 3080 3081 CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) 3082 CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), 3083 size_t) 3084 static const ctl_named_node_t * 3085 arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, 3086 size_t miblen, size_t i) { 3087 if (i > SC_NSIZES - SC_NBINS) { 3088 return NULL; 3089 } 3090 return super_arenas_lextent_i_node; 3091 } 3092 3093 static int 3094 arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3095 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3096 int ret; 3097 unsigned arena_ind; 3098 3099 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3100 3101 VERIFY_READ(unsigned); 3102 arena_config_t config = arena_config_default; 3103 WRITE(config.extent_hooks, extent_hooks_t *); 3104 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) { 3105 ret = EAGAIN; 3106 goto label_return; 3107 } 3108 READ(arena_ind, unsigned); 3109 3110 ret = 0; 3111 label_return: 3112 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3113 return ret; 3114 } 3115 3116 static int 3117 experimental_arenas_create_ext_ctl(tsd_t *tsd, 3118 const size_t *mib, size_t miblen, 3119 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3120 int ret; 3121 unsigned arena_ind; 3122 3123 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3124 3125 arena_config_t config = arena_config_default; 3126 VERIFY_READ(unsigned); 3127 WRITE(config, arena_config_t); 3128 3129 if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) { 3130 ret = EAGAIN; 3131 goto label_return; 3132 } 3133 READ(arena_ind, unsigned); 3134 ret = 0; 3135 label_return: 3136 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3137 return ret; 3138 } 3139 3140 static int 3141 arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, 3142 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 3143 size_t newlen) { 3144 int ret; 3145 unsigned arena_ind; 3146 void *ptr; 3147 edata_t *edata; 3148 arena_t *arena; 3149 3150 ptr = NULL; 3151 ret = EINVAL; 3152 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3153 WRITE(ptr, void *); 3154 edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr); 3155 if (edata == NULL) { 3156 goto label_return; 3157 } 3158 3159 arena = arena_get_from_edata(edata); 3160 if (arena == NULL) { 3161 goto label_return; 3162 } 3163 3164 arena_ind = arena_ind_get(arena); 3165 READ(arena_ind, unsigned); 3166 3167 ret = 0; 3168 label_return: 3169 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3170 return ret; 3171 } 3172 3173 /******************************************************************************/ 3174 3175 static int 3176 prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, 3177 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 3178 size_t newlen) { 3179 int ret; 3180 bool oldval; 3181 3182 if (!config_prof) { 3183 return ENOENT; 3184 } 3185 3186 if (newp != NULL) { 3187 if (!opt_prof) { 3188 ret = ENOENT; 3189 goto label_return; 3190 } 3191 if (newlen != sizeof(bool)) { 3192 ret = EINVAL; 3193 goto label_return; 3194 } 3195 oldval = prof_thread_active_init_set(tsd_tsdn(tsd), 3196 *(bool *)newp); 3197 } else { 3198 oldval = opt_prof ? prof_thread_active_init_get(tsd_tsdn(tsd)) : 3199 false; 3200 } 3201 READ(oldval, bool); 3202 3203 ret = 0; 3204 label_return: 3205 return ret; 3206 } 3207 3208 static int 3209 prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3210 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3211 int ret; 3212 bool oldval; 3213 3214 if (!config_prof) { 3215 ret = ENOENT; 3216 goto label_return; 3217 } 3218 3219 if (newp != NULL) { 3220 if (newlen != sizeof(bool)) { 3221 ret = EINVAL; 3222 goto label_return; 3223 } 3224 bool val = *(bool *)newp; 3225 if (!opt_prof) { 3226 if (val) { 3227 ret = ENOENT; 3228 goto label_return; 3229 } else { 3230 /* No change needed (already off). */ 3231 oldval = false; 3232 } 3233 } else { 3234 oldval = prof_active_set(tsd_tsdn(tsd), val); 3235 } 3236 } else { 3237 oldval = opt_prof ? prof_active_get(tsd_tsdn(tsd)) : false; 3238 } 3239 READ(oldval, bool); 3240 3241 ret = 0; 3242 label_return: 3243 return ret; 3244 } 3245 3246 static int 3247 prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3248 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3249 int ret; 3250 const char *filename = NULL; 3251 3252 if (!config_prof || !opt_prof) { 3253 return ENOENT; 3254 } 3255 3256 WRITEONLY(); 3257 WRITE(filename, const char *); 3258 3259 if (prof_mdump(tsd, filename)) { 3260 ret = EFAULT; 3261 goto label_return; 3262 } 3263 3264 ret = 0; 3265 label_return: 3266 return ret; 3267 } 3268 3269 static int 3270 prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3271 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3272 int ret; 3273 bool oldval; 3274 3275 if (!config_prof) { 3276 return ENOENT; 3277 } 3278 3279 if (newp != NULL) { 3280 if (!opt_prof) { 3281 ret = ENOENT; 3282 goto label_return; 3283 } 3284 if (newlen != sizeof(bool)) { 3285 ret = EINVAL; 3286 goto label_return; 3287 } 3288 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp); 3289 } else { 3290 oldval = opt_prof ? prof_gdump_get(tsd_tsdn(tsd)) : false; 3291 } 3292 READ(oldval, bool); 3293 3294 ret = 0; 3295 label_return: 3296 return ret; 3297 } 3298 3299 static int 3300 prof_prefix_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3301 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3302 int ret; 3303 const char *prefix = NULL; 3304 3305 if (!config_prof || !opt_prof) { 3306 return ENOENT; 3307 } 3308 3309 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3310 WRITEONLY(); 3311 WRITE(prefix, const char *); 3312 3313 ret = prof_prefix_set(tsd_tsdn(tsd), prefix) ? EFAULT : 0; 3314 label_return: 3315 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3316 return ret; 3317 } 3318 3319 static int 3320 prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3321 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3322 int ret; 3323 size_t lg_sample = lg_prof_sample; 3324 3325 if (!config_prof || !opt_prof) { 3326 return ENOENT; 3327 } 3328 3329 WRITEONLY(); 3330 WRITE(lg_sample, size_t); 3331 if (lg_sample >= (sizeof(uint64_t) << 3)) { 3332 lg_sample = (sizeof(uint64_t) << 3) - 1; 3333 } 3334 3335 prof_reset(tsd, lg_sample); 3336 3337 ret = 0; 3338 label_return: 3339 return ret; 3340 } 3341 3342 CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) 3343 CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) 3344 3345 static int 3346 prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 3347 size_t *oldlenp, void *newp, size_t newlen) { 3348 int ret; 3349 3350 const char *filename = NULL; 3351 3352 if (!config_prof || !opt_prof) { 3353 return ENOENT; 3354 } 3355 3356 WRITEONLY(); 3357 WRITE(filename, const char *); 3358 3359 if (prof_log_start(tsd_tsdn(tsd), filename)) { 3360 ret = EFAULT; 3361 goto label_return; 3362 } 3363 3364 ret = 0; 3365 label_return: 3366 return ret; 3367 } 3368 3369 static int 3370 prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 3371 size_t *oldlenp, void *newp, size_t newlen) { 3372 if (!config_prof || !opt_prof) { 3373 return ENOENT; 3374 } 3375 3376 if (prof_log_stop(tsd_tsdn(tsd))) { 3377 return EFAULT; 3378 } 3379 3380 return 0; 3381 } 3382 3383 static int 3384 experimental_hooks_prof_backtrace_ctl(tsd_t *tsd, const size_t *mib, 3385 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3386 int ret; 3387 3388 if (oldp == NULL && newp == NULL) { 3389 ret = EINVAL; 3390 goto label_return; 3391 } 3392 if (oldp != NULL) { 3393 prof_backtrace_hook_t old_hook = 3394 prof_backtrace_hook_get(); 3395 READ(old_hook, prof_backtrace_hook_t); 3396 } 3397 if (newp != NULL) { 3398 if (!opt_prof) { 3399 ret = ENOENT; 3400 goto label_return; 3401 } 3402 prof_backtrace_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL); 3403 WRITE(new_hook, prof_backtrace_hook_t); 3404 if (new_hook == NULL) { 3405 ret = EINVAL; 3406 goto label_return; 3407 } 3408 prof_backtrace_hook_set(new_hook); 3409 } 3410 ret = 0; 3411 label_return: 3412 return ret; 3413 } 3414 3415 static int 3416 experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib, 3417 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3418 int ret; 3419 3420 if (oldp == NULL && newp == NULL) { 3421 ret = EINVAL; 3422 goto label_return; 3423 } 3424 if (oldp != NULL) { 3425 prof_dump_hook_t old_hook = 3426 prof_dump_hook_get(); 3427 READ(old_hook, prof_dump_hook_t); 3428 } 3429 if (newp != NULL) { 3430 if (!opt_prof) { 3431 ret = ENOENT; 3432 goto label_return; 3433 } 3434 prof_dump_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL); 3435 WRITE(new_hook, prof_dump_hook_t); 3436 prof_dump_hook_set(new_hook); 3437 } 3438 ret = 0; 3439 label_return: 3440 return ret; 3441 } 3442 3443 /* For integration test purpose only. No plan to move out of experimental. */ 3444 static int 3445 experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib, 3446 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3447 int ret; 3448 3449 WRITEONLY(); 3450 if (newp != NULL) { 3451 if (newlen != sizeof(safety_check_abort_hook_t)) { 3452 ret = EINVAL; 3453 goto label_return; 3454 } 3455 safety_check_abort_hook_t hook JEMALLOC_CC_SILENCE_INIT(NULL); 3456 WRITE(hook, safety_check_abort_hook_t); 3457 safety_check_set_abort(hook); 3458 } 3459 ret = 0; 3460 label_return: 3461 return ret; 3462 } 3463 3464 /******************************************************************************/ 3465 3466 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) 3467 CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) 3468 CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) 3469 CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t) 3470 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) 3471 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) 3472 CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) 3473 3474 CTL_RO_CGEN(config_stats, stats_background_thread_num_threads, 3475 ctl_stats->background_thread.num_threads, size_t) 3476 CTL_RO_CGEN(config_stats, stats_background_thread_num_runs, 3477 ctl_stats->background_thread.num_runs, uint64_t) 3478 CTL_RO_CGEN(config_stats, stats_background_thread_run_interval, 3479 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t) 3480 3481 CTL_RO_CGEN(config_stats, stats_zero_reallocs, 3482 atomic_load_zu(&zero_realloc_count, ATOMIC_RELAXED), size_t) 3483 3484 CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) 3485 CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, 3486 ssize_t) 3487 CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, 3488 ssize_t) 3489 CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) 3490 CTL_RO_GEN(stats_arenas_i_uptime, 3491 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t) 3492 CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) 3493 CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) 3494 CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t) 3495 CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, 3496 arenas_i(mib[2])->astats->astats.mapped, size_t) 3497 CTL_RO_CGEN(config_stats, stats_arenas_i_retained, 3498 arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.retained, size_t) 3499 CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, 3500 arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t) 3501 3502 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, 3503 locked_read_u64_unsynchronized( 3504 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge), 3505 uint64_t) 3506 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, 3507 locked_read_u64_unsynchronized( 3508 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise), 3509 uint64_t) 3510 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, 3511 locked_read_u64_unsynchronized( 3512 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged), 3513 uint64_t) 3514 3515 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, 3516 locked_read_u64_unsynchronized( 3517 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge), 3518 uint64_t) 3519 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, 3520 locked_read_u64_unsynchronized( 3521 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise), 3522 uint64_t) 3523 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, 3524 locked_read_u64_unsynchronized( 3525 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged), 3526 uint64_t) 3527 3528 CTL_RO_CGEN(config_stats, stats_arenas_i_base, 3529 arenas_i(mib[2])->astats->astats.base, 3530 size_t) 3531 CTL_RO_CGEN(config_stats, stats_arenas_i_internal, 3532 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), 3533 size_t) 3534 CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp, 3535 arenas_i(mib[2])->astats->astats.metadata_thp, size_t) 3536 CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, 3537 arenas_i(mib[2])->astats->astats.tcache_bytes, size_t) 3538 CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_stashed_bytes, 3539 arenas_i(mib[2])->astats->astats.tcache_stashed_bytes, size_t) 3540 CTL_RO_CGEN(config_stats, stats_arenas_i_resident, 3541 arenas_i(mib[2])->astats->astats.resident, 3542 size_t) 3543 CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm, 3544 atomic_load_zu( 3545 &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.abandoned_vm, 3546 ATOMIC_RELAXED), size_t) 3547 3548 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes, 3549 arenas_i(mib[2])->astats->secstats.bytes, size_t) 3550 3551 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, 3552 arenas_i(mib[2])->astats->allocated_small, size_t) 3553 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, 3554 arenas_i(mib[2])->astats->nmalloc_small, uint64_t) 3555 CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, 3556 arenas_i(mib[2])->astats->ndalloc_small, uint64_t) 3557 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, 3558 arenas_i(mib[2])->astats->nrequests_small, uint64_t) 3559 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills, 3560 arenas_i(mib[2])->astats->nfills_small, uint64_t) 3561 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes, 3562 arenas_i(mib[2])->astats->nflushes_small, uint64_t) 3563 CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, 3564 arenas_i(mib[2])->astats->astats.allocated_large, size_t) 3565 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, 3566 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t) 3567 CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, 3568 arenas_i(mib[2])->astats->astats.ndalloc_large, uint64_t) 3569 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, 3570 arenas_i(mib[2])->astats->astats.nrequests_large, uint64_t) 3571 /* 3572 * Note: "nmalloc_large" here instead of "nfills" in the read. This is 3573 * intentional (large has no batch fill). 3574 */ 3575 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills, 3576 arenas_i(mib[2])->astats->astats.nmalloc_large, uint64_t) 3577 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes, 3578 arenas_i(mib[2])->astats->astats.nflushes_large, uint64_t) 3579 3580 /* Lock profiling related APIs below. */ 3581 #define RO_MUTEX_CTL_GEN(n, l) \ 3582 CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ 3583 l.n_lock_ops, uint64_t) \ 3584 CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ 3585 l.n_wait_times, uint64_t) \ 3586 CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ 3587 l.n_spin_acquired, uint64_t) \ 3588 CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ 3589 l.n_owner_switches, uint64_t) \ 3590 CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ 3591 nstime_ns(&l.tot_wait_time), uint64_t) \ 3592 CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ 3593 nstime_ns(&l.max_wait_time), uint64_t) \ 3594 CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ 3595 l.max_n_thds, uint32_t) 3596 3597 /* Global mutexes. */ 3598 #define OP(mtx) \ 3599 RO_MUTEX_CTL_GEN(mutexes_##mtx, \ 3600 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) 3601 MUTEX_PROF_GLOBAL_MUTEXES 3602 #undef OP 3603 3604 /* Per arena mutexes */ 3605 #define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ 3606 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) 3607 MUTEX_PROF_ARENA_MUTEXES 3608 #undef OP 3609 3610 /* tcache bin mutex */ 3611 RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, 3612 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) 3613 #undef RO_MUTEX_CTL_GEN 3614 3615 /* Resets all mutex stats, including global, arena and bin mutexes. */ 3616 static int 3617 stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, 3618 size_t miblen, void *oldp, size_t *oldlenp, 3619 void *newp, size_t newlen) { 3620 if (!config_stats) { 3621 return ENOENT; 3622 } 3623 3624 tsdn_t *tsdn = tsd_tsdn(tsd); 3625 3626 #define MUTEX_PROF_RESET(mtx) \ 3627 malloc_mutex_lock(tsdn, &mtx); \ 3628 malloc_mutex_prof_data_reset(tsdn, &mtx); \ 3629 malloc_mutex_unlock(tsdn, &mtx); 3630 3631 /* Global mutexes: ctl and prof. */ 3632 MUTEX_PROF_RESET(ctl_mtx); 3633 if (have_background_thread) { 3634 MUTEX_PROF_RESET(background_thread_lock); 3635 } 3636 if (config_prof && opt_prof) { 3637 MUTEX_PROF_RESET(bt2gctx_mtx); 3638 MUTEX_PROF_RESET(tdatas_mtx); 3639 MUTEX_PROF_RESET(prof_dump_mtx); 3640 MUTEX_PROF_RESET(prof_recent_alloc_mtx); 3641 MUTEX_PROF_RESET(prof_recent_dump_mtx); 3642 MUTEX_PROF_RESET(prof_stats_mtx); 3643 } 3644 3645 /* Per arena mutexes. */ 3646 unsigned n = narenas_total_get(); 3647 3648 for (unsigned i = 0; i < n; i++) { 3649 arena_t *arena = arena_get(tsdn, i, false); 3650 if (!arena) { 3651 continue; 3652 } 3653 MUTEX_PROF_RESET(arena->large_mtx); 3654 MUTEX_PROF_RESET(arena->pa_shard.edata_cache.mtx); 3655 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_dirty.mtx); 3656 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_muzzy.mtx); 3657 MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_retained.mtx); 3658 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx); 3659 MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx); 3660 MUTEX_PROF_RESET(arena->tcache_ql_mtx); 3661 MUTEX_PROF_RESET(arena->base->mtx); 3662 3663 for (szind_t j = 0; j < SC_NBINS; j++) { 3664 for (unsigned k = 0; k < bin_infos[j].n_shards; k++) { 3665 bin_t *bin = arena_get_bin(arena, j, k); 3666 MUTEX_PROF_RESET(bin->lock); 3667 } 3668 } 3669 } 3670 #undef MUTEX_PROF_RESET 3671 return 0; 3672 } 3673 3674 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, 3675 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nmalloc, uint64_t) 3676 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, 3677 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.ndalloc, uint64_t) 3678 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, 3679 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nrequests, uint64_t) 3680 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, 3681 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curregs, size_t) 3682 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills, 3683 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nfills, uint64_t) 3684 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes, 3685 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nflushes, uint64_t) 3686 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, 3687 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nslabs, uint64_t) 3688 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, 3689 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.reslabs, uint64_t) 3690 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, 3691 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.curslabs, size_t) 3692 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, 3693 arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nonfull_slabs, size_t) 3694 3695 static const ctl_named_node_t * 3696 stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, 3697 size_t miblen, size_t j) { 3698 if (j > SC_NBINS) { 3699 return NULL; 3700 } 3701 return super_stats_arenas_i_bins_j_node; 3702 } 3703 3704 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, 3705 locked_read_u64_unsynchronized( 3706 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t) 3707 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, 3708 locked_read_u64_unsynchronized( 3709 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) 3710 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, 3711 locked_read_u64_unsynchronized( 3712 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) 3713 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, 3714 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) 3715 3716 static const ctl_named_node_t * 3717 stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, 3718 size_t miblen, size_t j) { 3719 if (j > SC_NSIZES - SC_NBINS) { 3720 return NULL; 3721 } 3722 return super_stats_arenas_i_lextents_j_node; 3723 } 3724 3725 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, 3726 arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t) 3727 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, 3728 arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t) 3729 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, 3730 arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t) 3731 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, 3732 arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t) 3733 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, 3734 arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t) 3735 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, 3736 arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t) 3737 3738 static const ctl_named_node_t * 3739 stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, 3740 size_t miblen, size_t j) { 3741 if (j >= SC_NPSIZES) { 3742 return NULL; 3743 } 3744 return super_stats_arenas_i_extents_j_node; 3745 } 3746 3747 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes, 3748 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes, uint64_t) 3749 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges, 3750 arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t) 3751 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies, 3752 arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t) 3753 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies, 3754 arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t) 3755 3756 /* Full, nonhuge */ 3757 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge, 3758 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs, 3759 size_t) 3760 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge, 3761 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t) 3762 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge, 3763 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty, size_t) 3764 3765 /* Full, huge */ 3766 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge, 3767 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs, 3768 size_t) 3769 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge, 3770 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t) 3771 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge, 3772 arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty, size_t) 3773 3774 /* Empty, nonhuge */ 3775 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge, 3776 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs, 3777 size_t) 3778 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge, 3779 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive, size_t) 3780 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge, 3781 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty, size_t) 3782 3783 /* Empty, huge */ 3784 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge, 3785 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs, 3786 size_t) 3787 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge, 3788 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive, size_t) 3789 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge, 3790 arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty, size_t) 3791 3792 /* Nonfull, nonhuge */ 3793 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge, 3794 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs, 3795 size_t) 3796 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge, 3797 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive, 3798 size_t) 3799 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge, 3800 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ndirty, 3801 size_t) 3802 3803 /* Nonfull, huge */ 3804 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, 3805 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs, 3806 size_t) 3807 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge, 3808 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive, 3809 size_t) 3810 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge, 3811 arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ndirty, 3812 size_t) 3813 3814 static const ctl_named_node_t * 3815 stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib, 3816 size_t miblen, size_t j) { 3817 if (j >= PSSET_NPSIZES) { 3818 return NULL; 3819 } 3820 return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node; 3821 } 3822 3823 static bool 3824 ctl_arenas_i_verify(size_t i) { 3825 size_t a = arenas_i2a_impl(i, true, true); 3826 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { 3827 return true; 3828 } 3829 3830 return false; 3831 } 3832 3833 static const ctl_named_node_t * 3834 stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, 3835 size_t miblen, size_t i) { 3836 const ctl_named_node_t *ret; 3837 3838 malloc_mutex_lock(tsdn, &ctl_mtx); 3839 if (ctl_arenas_i_verify(i)) { 3840 ret = NULL; 3841 goto label_return; 3842 } 3843 3844 ret = super_stats_arenas_i_node; 3845 label_return: 3846 malloc_mutex_unlock(tsdn, &ctl_mtx); 3847 return ret; 3848 } 3849 3850 static int 3851 experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3852 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3853 int ret; 3854 if (oldp == NULL || oldlenp == NULL|| newp == NULL) { 3855 ret = EINVAL; 3856 goto label_return; 3857 } 3858 /* 3859 * Note: this is a *private* struct. This is an experimental interface; 3860 * forcing the user to know the jemalloc internals well enough to 3861 * extract the ABI hopefully ensures nobody gets too comfortable with 3862 * this API, which can change at a moment's notice. 3863 */ 3864 hooks_t hooks; 3865 WRITE(hooks, hooks_t); 3866 void *handle = hook_install(tsd_tsdn(tsd), &hooks); 3867 if (handle == NULL) { 3868 ret = EAGAIN; 3869 goto label_return; 3870 } 3871 READ(handle, void *); 3872 3873 ret = 0; 3874 label_return: 3875 return ret; 3876 } 3877 3878 static int 3879 experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3880 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3881 int ret; 3882 WRITEONLY(); 3883 void *handle = NULL; 3884 WRITE(handle, void *); 3885 if (handle == NULL) { 3886 ret = EINVAL; 3887 goto label_return; 3888 } 3889 hook_remove(tsd_tsdn(tsd), handle); 3890 ret = 0; 3891 label_return: 3892 return ret; 3893 } 3894 3895 static int 3896 experimental_thread_activity_callback_ctl(tsd_t *tsd, const size_t *mib, 3897 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3898 int ret; 3899 3900 if (!config_stats) { 3901 return ENOENT; 3902 } 3903 3904 activity_callback_thunk_t t_old = tsd_activity_callback_thunk_get(tsd); 3905 READ(t_old, activity_callback_thunk_t); 3906 3907 if (newp != NULL) { 3908 /* 3909 * This initialization is unnecessary. If it's omitted, though, 3910 * clang gets confused and warns on the subsequent use of t_new. 3911 */ 3912 activity_callback_thunk_t t_new = {NULL, NULL}; 3913 WRITE(t_new, activity_callback_thunk_t); 3914 tsd_activity_callback_thunk_set(tsd, t_new); 3915 } 3916 ret = 0; 3917 label_return: 3918 return ret; 3919 } 3920 3921 /* 3922 * Output six memory utilization entries for an input pointer, the first one of 3923 * type (void *) and the remaining five of type size_t, describing the following 3924 * (in the same order): 3925 * 3926 * (a) memory address of the extent a potential reallocation would go into, 3927 * == the five fields below describe about the extent the pointer resides in == 3928 * (b) number of free regions in the extent, 3929 * (c) number of regions in the extent, 3930 * (d) size of the extent in terms of bytes, 3931 * (e) total number of free regions in the bin the extent belongs to, and 3932 * (f) total number of regions in the bin the extent belongs to. 3933 * 3934 * Note that "(e)" and "(f)" are only available when stats are enabled; 3935 * otherwise their values are undefined. 3936 * 3937 * This API is mainly intended for small class allocations, where extents are 3938 * used as slab. Note that if the bin the extent belongs to is completely 3939 * full, "(a)" will be NULL. 3940 * 3941 * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)" 3942 * will be zero (if stats are enabled; otherwise undefined). The other three 3943 * fields will be properly set though the values are trivial: "(b)" will be 0, 3944 * "(c)" will be 1, and "(d)" will be the usable size. 3945 * 3946 * The input pointer and size are respectively passed in by newp and newlen, 3947 * and the output fields and size are respectively oldp and *oldlenp. 3948 * 3949 * It can be beneficial to define the following macros to make it easier to 3950 * access the output: 3951 * 3952 * #define SLABCUR_READ(out) (*(void **)out) 3953 * #define COUNTS(out) ((size_t *)((void **)out + 1)) 3954 * #define NFREE_READ(out) COUNTS(out)[0] 3955 * #define NREGS_READ(out) COUNTS(out)[1] 3956 * #define SIZE_READ(out) COUNTS(out)[2] 3957 * #define BIN_NFREE_READ(out) COUNTS(out)[3] 3958 * #define BIN_NREGS_READ(out) COUNTS(out)[4] 3959 * 3960 * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test 3961 * test_query in test/unit/extent_util.c for an example. 3962 * 3963 * For a typical defragmentation workflow making use of this API for 3964 * understanding the fragmentation level, please refer to the comment for 3965 * experimental_utilization_batch_query_ctl. 3966 * 3967 * It's up to the application how to determine the significance of 3968 * fragmentation relying on the outputs returned. Possible choices are: 3969 * 3970 * (a) if extent utilization ratio is below certain threshold, 3971 * (b) if extent memory consumption is above certain threshold, 3972 * (c) if extent utilization ratio is significantly below bin utilization ratio, 3973 * (d) if input pointer deviates a lot from potential reallocation address, or 3974 * (e) some selection/combination of the above. 3975 * 3976 * The caller needs to make sure that the input/output arguments are valid, 3977 * in particular, that the size of the output is correct, i.e.: 3978 * 3979 * *oldlenp = sizeof(void *) + sizeof(size_t) * 5 3980 * 3981 * Otherwise, the function immediately returns EINVAL without touching anything. 3982 * 3983 * In the rare case where there's no associated extent found for the input 3984 * pointer, the function zeros out all output fields and return. Please refer 3985 * to the comment for experimental_utilization_batch_query_ctl to understand the 3986 * motivation from C++. 3987 */ 3988 static int 3989 experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, 3990 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3991 int ret; 3992 3993 assert(sizeof(inspect_extent_util_stats_verbose_t) 3994 == sizeof(void *) + sizeof(size_t) * 5); 3995 3996 if (oldp == NULL || oldlenp == NULL 3997 || *oldlenp != sizeof(inspect_extent_util_stats_verbose_t) 3998 || newp == NULL) { 3999 ret = EINVAL; 4000 goto label_return; 4001 } 4002 4003 void *ptr = NULL; 4004 WRITE(ptr, void *); 4005 inspect_extent_util_stats_verbose_t *util_stats 4006 = (inspect_extent_util_stats_verbose_t *)oldp; 4007 inspect_extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr, 4008 &util_stats->nfree, &util_stats->nregs, &util_stats->size, 4009 &util_stats->bin_nfree, &util_stats->bin_nregs, 4010 &util_stats->slabcur_addr); 4011 ret = 0; 4012 4013 label_return: 4014 return ret; 4015 } 4016 4017 /* 4018 * Given an input array of pointers, output three memory utilization entries of 4019 * type size_t for each input pointer about the extent it resides in: 4020 * 4021 * (a) number of free regions in the extent, 4022 * (b) number of regions in the extent, and 4023 * (c) size of the extent in terms of bytes. 4024 * 4025 * This API is mainly intended for small class allocations, where extents are 4026 * used as slab. In case of large class allocations, the outputs are trivial: 4027 * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size. 4028 * 4029 * Note that multiple input pointers may reside on a same extent so the output 4030 * fields may contain duplicates. 4031 * 4032 * The format of the input/output looks like: 4033 * 4034 * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions 4035 * | output[1]: 1st_extent_n_regions 4036 * | output[2]: 1st_extent_size 4037 * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions 4038 * | output[4]: 2nd_extent_n_regions 4039 * | output[5]: 2nd_extent_size 4040 * ... | ... 4041 * 4042 * The input array and size are respectively passed in by newp and newlen, and 4043 * the output array and size are respectively oldp and *oldlenp. 4044 * 4045 * It can be beneficial to define the following macros to make it easier to 4046 * access the output: 4047 * 4048 * #define NFREE_READ(out, i) out[(i) * 3] 4049 * #define NREGS_READ(out, i) out[(i) * 3 + 1] 4050 * #define SIZE_READ(out, i) out[(i) * 3 + 2] 4051 * 4052 * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit 4053 * test test_batch in test/unit/extent_util.c for a concrete example. 4054 * 4055 * A typical workflow would be composed of the following steps: 4056 * 4057 * (1) flush tcache: mallctl("thread.tcache.flush", ...) 4058 * (2) initialize input array of pointers to query fragmentation 4059 * (3) allocate output array to hold utilization statistics 4060 * (4) query utilization: mallctl("experimental.utilization.batch_query", ...) 4061 * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here 4062 * (6) disable tcache: mallctl("thread.tcache.enabled", ...) 4063 * (7) defragment allocations with significant fragmentation, e.g.: 4064 * for each allocation { 4065 * if it's fragmented { 4066 * malloc(...); 4067 * memcpy(...); 4068 * free(...); 4069 * } 4070 * } 4071 * (8) enable tcache: mallctl("thread.tcache.enabled", ...) 4072 * 4073 * The application can determine the significance of fragmentation themselves 4074 * relying on the statistics returned, both at the overall level i.e. step "(5)" 4075 * and at individual allocation level i.e. within step "(7)". Possible choices 4076 * are: 4077 * 4078 * (a) whether memory utilization ratio is below certain threshold, 4079 * (b) whether memory consumption is above certain threshold, or 4080 * (c) some combination of the two. 4081 * 4082 * The caller needs to make sure that the input/output arrays are valid and 4083 * their sizes are proper as well as matched, meaning: 4084 * 4085 * (a) newlen = n_pointers * sizeof(const void *) 4086 * (b) *oldlenp = n_pointers * sizeof(size_t) * 3 4087 * (c) n_pointers > 0 4088 * 4089 * Otherwise, the function immediately returns EINVAL without touching anything. 4090 * 4091 * In the rare case where there's no associated extent found for some pointers, 4092 * rather than immediately terminating the computation and raising an error, 4093 * the function simply zeros out the corresponding output fields and continues 4094 * the computation until all input pointers are handled. The motivations of 4095 * such a design are as follows: 4096 * 4097 * (a) The function always either processes nothing or processes everything, and 4098 * never leaves the output half touched and half untouched. 4099 * 4100 * (b) It facilitates usage needs especially common in C++. A vast variety of 4101 * C++ objects are instantiated with multiple dynamic memory allocations. For 4102 * example, std::string and std::vector typically use at least two allocations, 4103 * one for the metadata and one for the actual content. Other types may use 4104 * even more allocations. When inquiring about utilization statistics, the 4105 * caller often wants to examine into all such allocations, especially internal 4106 * one(s), rather than just the topmost one. The issue comes when some 4107 * implementations do certain optimizations to reduce/aggregate some internal 4108 * allocations, e.g. putting short strings directly into the metadata, and such 4109 * decisions are not known to the caller. Therefore, we permit pointers to 4110 * memory usages that may not be returned by previous malloc calls, and we 4111 * provide the caller a convenient way to identify such cases. 4112 */ 4113 static int 4114 experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib, 4115 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4116 int ret; 4117 4118 assert(sizeof(inspect_extent_util_stats_t) == sizeof(size_t) * 3); 4119 4120 const size_t len = newlen / sizeof(const void *); 4121 if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0 4122 || newlen != len * sizeof(const void *) 4123 || *oldlenp != len * sizeof(inspect_extent_util_stats_t)) { 4124 ret = EINVAL; 4125 goto label_return; 4126 } 4127 4128 void **ptrs = (void **)newp; 4129 inspect_extent_util_stats_t *util_stats = 4130 (inspect_extent_util_stats_t *)oldp; 4131 size_t i; 4132 for (i = 0; i < len; ++i) { 4133 inspect_extent_util_stats_get(tsd_tsdn(tsd), ptrs[i], 4134 &util_stats[i].nfree, &util_stats[i].nregs, 4135 &util_stats[i].size); 4136 } 4137 ret = 0; 4138 4139 label_return: 4140 return ret; 4141 } 4142 4143 static const ctl_named_node_t * 4144 experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib, 4145 size_t miblen, size_t i) { 4146 const ctl_named_node_t *ret; 4147 4148 malloc_mutex_lock(tsdn, &ctl_mtx); 4149 if (ctl_arenas_i_verify(i)) { 4150 ret = NULL; 4151 goto label_return; 4152 } 4153 ret = super_experimental_arenas_i_node; 4154 label_return: 4155 malloc_mutex_unlock(tsdn, &ctl_mtx); 4156 return ret; 4157 } 4158 4159 static int 4160 experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, 4161 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4162 if (!config_stats) { 4163 return ENOENT; 4164 } 4165 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) { 4166 return EINVAL; 4167 } 4168 4169 unsigned arena_ind; 4170 arena_t *arena; 4171 int ret; 4172 size_t *pactivep; 4173 4174 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 4175 READONLY(); 4176 MIB_UNSIGNED(arena_ind, 2); 4177 if (arena_ind < narenas_total_get() && (arena = 4178 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 4179 #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \ 4180 defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER) 4181 /* Expose the underlying counter for fast read. */ 4182 pactivep = (size_t *)&(arena->pa_shard.nactive.repr); 4183 READ(pactivep, size_t *); 4184 ret = 0; 4185 #else 4186 ret = EFAULT; 4187 #endif 4188 } else { 4189 ret = EFAULT; 4190 } 4191 label_return: 4192 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 4193 return ret; 4194 } 4195 4196 static int 4197 experimental_prof_recent_alloc_max_ctl(tsd_t *tsd, const size_t *mib, 4198 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4199 int ret; 4200 4201 if (!(config_prof && opt_prof)) { 4202 ret = ENOENT; 4203 goto label_return; 4204 } 4205 4206 ssize_t old_max; 4207 if (newp != NULL) { 4208 if (newlen != sizeof(ssize_t)) { 4209 ret = EINVAL; 4210 goto label_return; 4211 } 4212 ssize_t max = *(ssize_t *)newp; 4213 if (max < -1) { 4214 ret = EINVAL; 4215 goto label_return; 4216 } 4217 old_max = prof_recent_alloc_max_ctl_write(tsd, max); 4218 } else { 4219 old_max = prof_recent_alloc_max_ctl_read(); 4220 } 4221 READ(old_max, ssize_t); 4222 4223 ret = 0; 4224 4225 label_return: 4226 return ret; 4227 } 4228 4229 typedef struct write_cb_packet_s write_cb_packet_t; 4230 struct write_cb_packet_s { 4231 write_cb_t *write_cb; 4232 void *cbopaque; 4233 }; 4234 4235 static int 4236 experimental_prof_recent_alloc_dump_ctl(tsd_t *tsd, const size_t *mib, 4237 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4238 int ret; 4239 4240 if (!(config_prof && opt_prof)) { 4241 ret = ENOENT; 4242 goto label_return; 4243 } 4244 4245 assert(sizeof(write_cb_packet_t) == sizeof(void *) * 2); 4246 4247 WRITEONLY(); 4248 write_cb_packet_t write_cb_packet; 4249 ASSURED_WRITE(write_cb_packet, write_cb_packet_t); 4250 4251 prof_recent_alloc_dump(tsd, write_cb_packet.write_cb, 4252 write_cb_packet.cbopaque); 4253 4254 ret = 0; 4255 4256 label_return: 4257 return ret; 4258 } 4259 4260 typedef struct batch_alloc_packet_s batch_alloc_packet_t; 4261 struct batch_alloc_packet_s { 4262 void **ptrs; 4263 size_t num; 4264 size_t size; 4265 int flags; 4266 }; 4267 4268 static int 4269 experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib, 4270 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4271 int ret; 4272 4273 VERIFY_READ(size_t); 4274 4275 batch_alloc_packet_t batch_alloc_packet; 4276 ASSURED_WRITE(batch_alloc_packet, batch_alloc_packet_t); 4277 size_t filled = batch_alloc(batch_alloc_packet.ptrs, 4278 batch_alloc_packet.num, batch_alloc_packet.size, 4279 batch_alloc_packet.flags); 4280 READ(filled, size_t); 4281 4282 ret = 0; 4283 4284 label_return: 4285 return ret; 4286 } 4287 4288 static int 4289 prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 4290 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4291 int ret; 4292 unsigned binind; 4293 prof_stats_t stats; 4294 4295 if (!(config_prof && opt_prof && opt_prof_stats)) { 4296 ret = ENOENT; 4297 goto label_return; 4298 } 4299 4300 READONLY(); 4301 MIB_UNSIGNED(binind, 3); 4302 if (binind >= SC_NBINS) { 4303 ret = EINVAL; 4304 goto label_return; 4305 } 4306 prof_stats_get_live(tsd, (szind_t)binind, &stats); 4307 READ(stats, prof_stats_t); 4308 4309 ret = 0; 4310 label_return: 4311 return ret; 4312 } 4313 4314 static int 4315 prof_stats_bins_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 4316 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4317 int ret; 4318 unsigned binind; 4319 prof_stats_t stats; 4320 4321 if (!(config_prof && opt_prof && opt_prof_stats)) { 4322 ret = ENOENT; 4323 goto label_return; 4324 } 4325 4326 READONLY(); 4327 MIB_UNSIGNED(binind, 3); 4328 if (binind >= SC_NBINS) { 4329 ret = EINVAL; 4330 goto label_return; 4331 } 4332 prof_stats_get_accum(tsd, (szind_t)binind, &stats); 4333 READ(stats, prof_stats_t); 4334 4335 ret = 0; 4336 label_return: 4337 return ret; 4338 } 4339 4340 static const ctl_named_node_t * 4341 prof_stats_bins_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 4342 size_t i) { 4343 if (!(config_prof && opt_prof && opt_prof_stats)) { 4344 return NULL; 4345 } 4346 if (i >= SC_NBINS) { 4347 return NULL; 4348 } 4349 return super_prof_stats_bins_i_node; 4350 } 4351 4352 static int 4353 prof_stats_lextents_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 4354 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4355 int ret; 4356 unsigned lextent_ind; 4357 prof_stats_t stats; 4358 4359 if (!(config_prof && opt_prof && opt_prof_stats)) { 4360 ret = ENOENT; 4361 goto label_return; 4362 } 4363 4364 READONLY(); 4365 MIB_UNSIGNED(lextent_ind, 3); 4366 if (lextent_ind >= SC_NSIZES - SC_NBINS) { 4367 ret = EINVAL; 4368 goto label_return; 4369 } 4370 prof_stats_get_live(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats); 4371 READ(stats, prof_stats_t); 4372 4373 ret = 0; 4374 label_return: 4375 return ret; 4376 } 4377 4378 static int 4379 prof_stats_lextents_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 4380 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 4381 int ret; 4382 unsigned lextent_ind; 4383 prof_stats_t stats; 4384 4385 if (!(config_prof && opt_prof && opt_prof_stats)) { 4386 ret = ENOENT; 4387 goto label_return; 4388 } 4389 4390 READONLY(); 4391 MIB_UNSIGNED(lextent_ind, 3); 4392 if (lextent_ind >= SC_NSIZES - SC_NBINS) { 4393 ret = EINVAL; 4394 goto label_return; 4395 } 4396 prof_stats_get_accum(tsd, (szind_t)(lextent_ind + SC_NBINS), &stats); 4397 READ(stats, prof_stats_t); 4398 4399 ret = 0; 4400 label_return: 4401 return ret; 4402 } 4403 4404 static const ctl_named_node_t * 4405 prof_stats_lextents_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 4406 size_t i) { 4407 if (!(config_prof && opt_prof && opt_prof_stats)) { 4408 return NULL; 4409 } 4410 if (i >= SC_NSIZES - SC_NBINS) { 4411 return NULL; 4412 } 4413 return super_prof_stats_lextents_i_node; 4414 } 4415