1*8e33eff8Schristos #define JEMALLOC_EXTENT_DSS_C_ 2*8e33eff8Schristos #include "jemalloc/internal/jemalloc_preamble.h" 3*8e33eff8Schristos #include "jemalloc/internal/jemalloc_internal_includes.h" 4*8e33eff8Schristos 5*8e33eff8Schristos #include "jemalloc/internal/assert.h" 6*8e33eff8Schristos #include "jemalloc/internal/extent_dss.h" 7*8e33eff8Schristos #include "jemalloc/internal/spin.h" 8*8e33eff8Schristos 9*8e33eff8Schristos /******************************************************************************/ 10*8e33eff8Schristos /* Data. */ 11*8e33eff8Schristos 12*8e33eff8Schristos const char *opt_dss = DSS_DEFAULT; 13*8e33eff8Schristos 14*8e33eff8Schristos const char *dss_prec_names[] = { 15*8e33eff8Schristos "disabled", 16*8e33eff8Schristos "primary", 17*8e33eff8Schristos "secondary", 18*8e33eff8Schristos "N/A" 19*8e33eff8Schristos }; 20*8e33eff8Schristos 21*8e33eff8Schristos /* 22*8e33eff8Schristos * Current dss precedence default, used when creating new arenas. NB: This is 23*8e33eff8Schristos * stored as unsigned rather than dss_prec_t because in principle there's no 24*8e33eff8Schristos * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use 25*8e33eff8Schristos * atomic operations to synchronize the setting. 26*8e33eff8Schristos */ 27*8e33eff8Schristos static atomic_u_t dss_prec_default = ATOMIC_INIT( 28*8e33eff8Schristos (unsigned)DSS_PREC_DEFAULT); 29*8e33eff8Schristos 30*8e33eff8Schristos /* Base address of the DSS. */ 31*8e33eff8Schristos static void *dss_base; 32*8e33eff8Schristos /* Atomic boolean indicating whether a thread is currently extending DSS. */ 33*8e33eff8Schristos static atomic_b_t dss_extending; 34*8e33eff8Schristos /* Atomic boolean indicating whether the DSS is exhausted. */ 35*8e33eff8Schristos static atomic_b_t dss_exhausted; 36*8e33eff8Schristos /* Atomic current upper limit on DSS addresses. */ 37*8e33eff8Schristos static atomic_p_t dss_max; 38*8e33eff8Schristos 39*8e33eff8Schristos /******************************************************************************/ 40*8e33eff8Schristos 41*8e33eff8Schristos static void * 42*8e33eff8Schristos extent_dss_sbrk(intptr_t increment) { 43*8e33eff8Schristos #ifdef JEMALLOC_DSS 44*8e33eff8Schristos return sbrk(increment); 45*8e33eff8Schristos #else 46*8e33eff8Schristos not_implemented(); 47*8e33eff8Schristos return NULL; 48*8e33eff8Schristos #endif 49*8e33eff8Schristos } 50*8e33eff8Schristos 51*8e33eff8Schristos dss_prec_t 52*8e33eff8Schristos extent_dss_prec_get(void) { 53*8e33eff8Schristos dss_prec_t ret; 54*8e33eff8Schristos 55*8e33eff8Schristos if (!have_dss) { 56*8e33eff8Schristos return dss_prec_disabled; 57*8e33eff8Schristos } 58*8e33eff8Schristos ret = (dss_prec_t)atomic_load_u(&dss_prec_default, ATOMIC_ACQUIRE); 59*8e33eff8Schristos return ret; 60*8e33eff8Schristos } 61*8e33eff8Schristos 62*8e33eff8Schristos bool 63*8e33eff8Schristos extent_dss_prec_set(dss_prec_t dss_prec) { 64*8e33eff8Schristos if (!have_dss) { 65*8e33eff8Schristos return (dss_prec != dss_prec_disabled); 66*8e33eff8Schristos } 67*8e33eff8Schristos atomic_store_u(&dss_prec_default, (unsigned)dss_prec, ATOMIC_RELEASE); 68*8e33eff8Schristos return false; 69*8e33eff8Schristos } 70*8e33eff8Schristos 71*8e33eff8Schristos static void 72*8e33eff8Schristos extent_dss_extending_start(void) { 73*8e33eff8Schristos spin_t spinner = SPIN_INITIALIZER; 74*8e33eff8Schristos while (true) { 75*8e33eff8Schristos bool expected = false; 76*8e33eff8Schristos if (atomic_compare_exchange_weak_b(&dss_extending, &expected, 77*8e33eff8Schristos true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) { 78*8e33eff8Schristos break; 79*8e33eff8Schristos } 80*8e33eff8Schristos spin_adaptive(&spinner); 81*8e33eff8Schristos } 82*8e33eff8Schristos } 83*8e33eff8Schristos 84*8e33eff8Schristos static void 85*8e33eff8Schristos extent_dss_extending_finish(void) { 86*8e33eff8Schristos assert(atomic_load_b(&dss_extending, ATOMIC_RELAXED)); 87*8e33eff8Schristos 88*8e33eff8Schristos atomic_store_b(&dss_extending, false, ATOMIC_RELEASE); 89*8e33eff8Schristos } 90*8e33eff8Schristos 91*8e33eff8Schristos static void * 92*8e33eff8Schristos extent_dss_max_update(void *new_addr) { 93*8e33eff8Schristos /* 94*8e33eff8Schristos * Get the current end of the DSS as max_cur and assure that dss_max is 95*8e33eff8Schristos * up to date. 96*8e33eff8Schristos */ 97*8e33eff8Schristos void *max_cur = extent_dss_sbrk(0); 98*8e33eff8Schristos if (max_cur == (void *)-1) { 99*8e33eff8Schristos return NULL; 100*8e33eff8Schristos } 101*8e33eff8Schristos atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASE); 102*8e33eff8Schristos /* Fixed new_addr can only be supported if it is at the edge of DSS. */ 103*8e33eff8Schristos if (new_addr != NULL && max_cur != new_addr) { 104*8e33eff8Schristos return NULL; 105*8e33eff8Schristos } 106*8e33eff8Schristos return max_cur; 107*8e33eff8Schristos } 108*8e33eff8Schristos 109*8e33eff8Schristos void * 110*8e33eff8Schristos extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, 111*8e33eff8Schristos size_t alignment, bool *zero, bool *commit) { 112*8e33eff8Schristos extent_t *gap; 113*8e33eff8Schristos 114*8e33eff8Schristos cassert(have_dss); 115*8e33eff8Schristos assert(size > 0); 116*8e33eff8Schristos assert(alignment > 0); 117*8e33eff8Schristos 118*8e33eff8Schristos /* 119*8e33eff8Schristos * sbrk() uses a signed increment argument, so take care not to 120*8e33eff8Schristos * interpret a large allocation request as a negative increment. 121*8e33eff8Schristos */ 122*8e33eff8Schristos if ((intptr_t)size < 0) { 123*8e33eff8Schristos return NULL; 124*8e33eff8Schristos } 125*8e33eff8Schristos 126*8e33eff8Schristos gap = extent_alloc(tsdn, arena); 127*8e33eff8Schristos if (gap == NULL) { 128*8e33eff8Schristos return NULL; 129*8e33eff8Schristos } 130*8e33eff8Schristos 131*8e33eff8Schristos extent_dss_extending_start(); 132*8e33eff8Schristos if (!atomic_load_b(&dss_exhausted, ATOMIC_ACQUIRE)) { 133*8e33eff8Schristos /* 134*8e33eff8Schristos * The loop is necessary to recover from races with other 135*8e33eff8Schristos * threads that are using the DSS for something other than 136*8e33eff8Schristos * malloc. 137*8e33eff8Schristos */ 138*8e33eff8Schristos while (true) { 139*8e33eff8Schristos void *max_cur = extent_dss_max_update(new_addr); 140*8e33eff8Schristos if (max_cur == NULL) { 141*8e33eff8Schristos goto label_oom; 142*8e33eff8Schristos } 143*8e33eff8Schristos 144*8e33eff8Schristos /* 145*8e33eff8Schristos * Compute how much page-aligned gap space (if any) is 146*8e33eff8Schristos * necessary to satisfy alignment. This space can be 147*8e33eff8Schristos * recycled for later use. 148*8e33eff8Schristos */ 149*8e33eff8Schristos void *gap_addr_page = (void *)(PAGE_CEILING( 150*8e33eff8Schristos (uintptr_t)max_cur)); 151*8e33eff8Schristos void *ret = (void *)ALIGNMENT_CEILING( 152*8e33eff8Schristos (uintptr_t)gap_addr_page, alignment); 153*8e33eff8Schristos size_t gap_size_page = (uintptr_t)ret - 154*8e33eff8Schristos (uintptr_t)gap_addr_page; 155*8e33eff8Schristos if (gap_size_page != 0) { 156*8e33eff8Schristos extent_init(gap, arena, gap_addr_page, 157*8e33eff8Schristos gap_size_page, false, NSIZES, 158*8e33eff8Schristos arena_extent_sn_next(arena), 159*8e33eff8Schristos extent_state_active, false, true, true); 160*8e33eff8Schristos } 161*8e33eff8Schristos /* 162*8e33eff8Schristos * Compute the address just past the end of the desired 163*8e33eff8Schristos * allocation space. 164*8e33eff8Schristos */ 165*8e33eff8Schristos void *dss_next = (void *)((uintptr_t)ret + size); 166*8e33eff8Schristos if ((uintptr_t)ret < (uintptr_t)max_cur || 167*8e33eff8Schristos (uintptr_t)dss_next < (uintptr_t)max_cur) { 168*8e33eff8Schristos goto label_oom; /* Wrap-around. */ 169*8e33eff8Schristos } 170*8e33eff8Schristos /* Compute the increment, including subpage bytes. */ 171*8e33eff8Schristos void *gap_addr_subpage = max_cur; 172*8e33eff8Schristos size_t gap_size_subpage = (uintptr_t)ret - 173*8e33eff8Schristos (uintptr_t)gap_addr_subpage; 174*8e33eff8Schristos intptr_t incr = gap_size_subpage + size; 175*8e33eff8Schristos 176*8e33eff8Schristos assert((uintptr_t)max_cur + incr == (uintptr_t)ret + 177*8e33eff8Schristos size); 178*8e33eff8Schristos 179*8e33eff8Schristos /* Try to allocate. */ 180*8e33eff8Schristos void *dss_prev = extent_dss_sbrk(incr); 181*8e33eff8Schristos if (dss_prev == max_cur) { 182*8e33eff8Schristos /* Success. */ 183*8e33eff8Schristos atomic_store_p(&dss_max, dss_next, 184*8e33eff8Schristos ATOMIC_RELEASE); 185*8e33eff8Schristos extent_dss_extending_finish(); 186*8e33eff8Schristos 187*8e33eff8Schristos if (gap_size_page != 0) { 188*8e33eff8Schristos extent_dalloc_gap(tsdn, arena, gap); 189*8e33eff8Schristos } else { 190*8e33eff8Schristos extent_dalloc(tsdn, arena, gap); 191*8e33eff8Schristos } 192*8e33eff8Schristos if (!*commit) { 193*8e33eff8Schristos *commit = pages_decommit(ret, size); 194*8e33eff8Schristos } 195*8e33eff8Schristos if (*zero && *commit) { 196*8e33eff8Schristos extent_hooks_t *extent_hooks = 197*8e33eff8Schristos EXTENT_HOOKS_INITIALIZER; 198*8e33eff8Schristos extent_t extent; 199*8e33eff8Schristos 200*8e33eff8Schristos extent_init(&extent, arena, ret, size, 201*8e33eff8Schristos size, false, NSIZES, 202*8e33eff8Schristos extent_state_active, false, true, 203*8e33eff8Schristos true); 204*8e33eff8Schristos if (extent_purge_forced_wrapper(tsdn, 205*8e33eff8Schristos arena, &extent_hooks, &extent, 0, 206*8e33eff8Schristos size)) { 207*8e33eff8Schristos memset(ret, 0, size); 208*8e33eff8Schristos } 209*8e33eff8Schristos } 210*8e33eff8Schristos return ret; 211*8e33eff8Schristos } 212*8e33eff8Schristos /* 213*8e33eff8Schristos * Failure, whether due to OOM or a race with a raw 214*8e33eff8Schristos * sbrk() call from outside the allocator. 215*8e33eff8Schristos */ 216*8e33eff8Schristos if (dss_prev == (void *)-1) { 217*8e33eff8Schristos /* OOM. */ 218*8e33eff8Schristos atomic_store_b(&dss_exhausted, true, 219*8e33eff8Schristos ATOMIC_RELEASE); 220*8e33eff8Schristos goto label_oom; 221*8e33eff8Schristos } 222*8e33eff8Schristos } 223*8e33eff8Schristos } 224*8e33eff8Schristos label_oom: 225*8e33eff8Schristos extent_dss_extending_finish(); 226*8e33eff8Schristos extent_dalloc(tsdn, arena, gap); 227*8e33eff8Schristos return NULL; 228*8e33eff8Schristos } 229*8e33eff8Schristos 230*8e33eff8Schristos static bool 231*8e33eff8Schristos extent_in_dss_helper(void *addr, void *max) { 232*8e33eff8Schristos return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr < 233*8e33eff8Schristos (uintptr_t)max); 234*8e33eff8Schristos } 235*8e33eff8Schristos 236*8e33eff8Schristos bool 237*8e33eff8Schristos extent_in_dss(void *addr) { 238*8e33eff8Schristos cassert(have_dss); 239*8e33eff8Schristos 240*8e33eff8Schristos return extent_in_dss_helper(addr, atomic_load_p(&dss_max, 241*8e33eff8Schristos ATOMIC_ACQUIRE)); 242*8e33eff8Schristos } 243*8e33eff8Schristos 244*8e33eff8Schristos bool 245*8e33eff8Schristos extent_dss_mergeable(void *addr_a, void *addr_b) { 246*8e33eff8Schristos void *max; 247*8e33eff8Schristos 248*8e33eff8Schristos cassert(have_dss); 249*8e33eff8Schristos 250*8e33eff8Schristos if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b < 251*8e33eff8Schristos (uintptr_t)dss_base) { 252*8e33eff8Schristos return true; 253*8e33eff8Schristos } 254*8e33eff8Schristos 255*8e33eff8Schristos max = atomic_load_p(&dss_max, ATOMIC_ACQUIRE); 256*8e33eff8Schristos return (extent_in_dss_helper(addr_a, max) == 257*8e33eff8Schristos extent_in_dss_helper(addr_b, max)); 258*8e33eff8Schristos } 259*8e33eff8Schristos 260*8e33eff8Schristos void 261*8e33eff8Schristos extent_dss_boot(void) { 262*8e33eff8Schristos cassert(have_dss); 263*8e33eff8Schristos 264*8e33eff8Schristos dss_base = extent_dss_sbrk(0); 265*8e33eff8Schristos atomic_store_b(&dss_extending, false, ATOMIC_RELAXED); 266*8e33eff8Schristos atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXED); 267*8e33eff8Schristos atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXED); 268*8e33eff8Schristos } 269*8e33eff8Schristos 270*8e33eff8Schristos /******************************************************************************/ 271