1*8e33eff8Schristos #ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H 2*8e33eff8Schristos #define JEMALLOC_INTERNAL_PROF_INLINES_A_H 3*8e33eff8Schristos 4*8e33eff8Schristos #include "jemalloc/internal/mutex.h" 5*8e33eff8Schristos 6*8e33eff8Schristos static inline bool 7*8e33eff8Schristos prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { 8*8e33eff8Schristos cassert(config_prof); 9*8e33eff8Schristos 10*8e33eff8Schristos bool overflow; 11*8e33eff8Schristos uint64_t a0, a1; 12*8e33eff8Schristos 13*8e33eff8Schristos /* 14*8e33eff8Schristos * If the application allocates fast enough (and/or if idump is slow 15*8e33eff8Schristos * enough), extreme overflow here (a1 >= prof_interval * 2) can cause 16*8e33eff8Schristos * idump trigger coalescing. This is an intentional mechanism that 17*8e33eff8Schristos * avoids rate-limiting allocation. 18*8e33eff8Schristos */ 19*8e33eff8Schristos #ifdef JEMALLOC_ATOMIC_U64 20*8e33eff8Schristos a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 21*8e33eff8Schristos do { 22*8e33eff8Schristos a1 = a0 + accumbytes; 23*8e33eff8Schristos assert(a1 >= a0); 24*8e33eff8Schristos overflow = (a1 >= prof_interval); 25*8e33eff8Schristos if (overflow) { 26*8e33eff8Schristos a1 %= prof_interval; 27*8e33eff8Schristos } 28*8e33eff8Schristos } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 29*8e33eff8Schristos a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 30*8e33eff8Schristos #else 31*8e33eff8Schristos malloc_mutex_lock(tsdn, &prof_accum->mtx); 32*8e33eff8Schristos a0 = prof_accum->accumbytes; 33*8e33eff8Schristos a1 = a0 + accumbytes; 34*8e33eff8Schristos overflow = (a1 >= prof_interval); 35*8e33eff8Schristos if (overflow) { 36*8e33eff8Schristos a1 %= prof_interval; 37*8e33eff8Schristos } 38*8e33eff8Schristos prof_accum->accumbytes = a1; 39*8e33eff8Schristos malloc_mutex_unlock(tsdn, &prof_accum->mtx); 40*8e33eff8Schristos #endif 41*8e33eff8Schristos return overflow; 42*8e33eff8Schristos } 43*8e33eff8Schristos 44*8e33eff8Schristos static JEMALLOC_NORETURN inline void 45*8e33eff8Schristos prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { 46*8e33eff8Schristos cassert(config_prof); 47*8e33eff8Schristos 48*8e33eff8Schristos /* 49*8e33eff8Schristos * Cancel out as much of the excessive prof_accumbytes increase as 50*8e33eff8Schristos * possible without underflowing. Interval-triggered dumps occur 51*8e33eff8Schristos * slightly more often than intended as a result of incomplete 52*8e33eff8Schristos * canceling. 53*8e33eff8Schristos */ 54*8e33eff8Schristos uint64_t a0, a1; 55*8e33eff8Schristos #ifdef JEMALLOC_ATOMIC_U64 56*8e33eff8Schristos a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 57*8e33eff8Schristos do { 58*8e33eff8Schristos a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - 59*8e33eff8Schristos usize) : 0; 60*8e33eff8Schristos } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 61*8e33eff8Schristos a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 62*8e33eff8Schristos #else 63*8e33eff8Schristos malloc_mutex_lock(tsdn, &prof_accum->mtx); 64*8e33eff8Schristos a0 = prof_accum->accumbytes; 65*8e33eff8Schristos a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : 66*8e33eff8Schristos 0; 67*8e33eff8Schristos prof_accum->accumbytes = a1; 68*8e33eff8Schristos malloc_mutex_unlock(tsdn, &prof_accum->mtx); 69*8e33eff8Schristos #endif 70*8e33eff8Schristos } 71*8e33eff8Schristos 72*8e33eff8Schristos JEMALLOC_ALWAYS_INLINE bool 73*8e33eff8Schristos prof_active_get_unlocked(void) { 74*8e33eff8Schristos /* 75*8e33eff8Schristos * Even if opt_prof is true, sampling can be temporarily disabled by 76*8e33eff8Schristos * setting prof_active to false. No locking is used when reading 77*8e33eff8Schristos * prof_active in the fast path, so there are no guarantees regarding 78*8e33eff8Schristos * how long it will take for all threads to notice state changes. 79*8e33eff8Schristos */ 80*8e33eff8Schristos return prof_active; 81*8e33eff8Schristos } 82*8e33eff8Schristos 83*8e33eff8Schristos #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ 84