xref: /freebsd-src/contrib/jemalloc/include/jemalloc/internal/tsd_generic.h (revision c5ad81420c495d1d5de04209b0ec4fcb435c322c)

#ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H
#error This file should be included only once, by tsd.h.
#endif
#define JEMALLOC_INTERNAL_TSD_GENERIC_H

typedef struct tsd_init_block_s tsd_init_block_t;
struct tsd_init_block_s {
	ql_elm(tsd_init_block_t) link;
	pthread_t thread;
	void *data;
};

/* Defined in tsd.c, to allow the mutex headers to have tsd dependencies. */
typedef struct tsd_init_head_s tsd_init_head_t;

typedef struct {
	bool initialized;
	tsd_t val;
} tsd_wrapper_t;

void *tsd_init_check_recursion(tsd_init_head_t *head,
    tsd_init_block_t *block);
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);

extern pthread_key_t tsd_tsd;
extern tsd_init_head_t tsd_init_head;
extern tsd_wrapper_t tsd_boot_wrapper;
extern bool tsd_booted;

/* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE void
tsd_cleanup_wrapper(void *arg) {
	tsd_wrapper_t *wrapper = (tsd_wrapper_t *)arg;

	if (wrapper->initialized) {
		wrapper->initialized = false;
		tsd_cleanup(&wrapper->val);
		if (wrapper->initialized) {
			/* Trigger another cleanup round. */
			if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0)
			{
				malloc_write("<jemalloc>: Error setting TSD\n");
				if (opt_abort) {
					abort();
				}
			}
			return;
		}
	}
	malloc_tsd_dalloc(wrapper);
}

JEMALLOC_ALWAYS_INLINE void
tsd_wrapper_set(tsd_wrapper_t *wrapper) {
	if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) {
		malloc_write("<jemalloc>: Error setting TSD\n");
		abort();
	}
}

JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
tsd_wrapper_get(bool init) {
	tsd_wrapper_t *wrapper = (tsd_wrapper_t *)pthread_getspecific(tsd_tsd);

	if (init && unlikely(wrapper == NULL)) {
		tsd_init_block_t block;
		wrapper = (tsd_wrapper_t *)
		    tsd_init_check_recursion(&tsd_init_head, &block);
		if (wrapper) {
			return wrapper;
		}
		wrapper = (tsd_wrapper_t *)
		    malloc_tsd_malloc(sizeof(tsd_wrapper_t));
		block.data = (void *)wrapper;
		if (wrapper == NULL) {
			malloc_write("<jemalloc>: Error allocating TSD\n");
			abort();
		} else {
			wrapper->initialized = false;
      JEMALLOC_DIAGNOSTIC_PUSH
      JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
			tsd_t initializer = TSD_INITIALIZER;
      JEMALLOC_DIAGNOSTIC_POP
			wrapper->val = initializer;
		}
		tsd_wrapper_set(wrapper);
		tsd_init_finish(&tsd_init_head, &block);
	}
	return wrapper;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_boot0(void) {
	if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
		return true;
	}
	tsd_wrapper_set(&tsd_boot_wrapper);
	tsd_booted = true;
	return false;
}

JEMALLOC_ALWAYS_INLINE void
tsd_boot1(void) {
	tsd_wrapper_t *wrapper;
	wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t));
	if (wrapper == NULL) {
		malloc_write("<jemalloc>: Error allocating TSD\n");
		abort();
	}
	tsd_boot_wrapper.initialized = false;
	tsd_cleanup(&tsd_boot_wrapper.val);
	wrapper->initialized = false;
  JEMALLOC_DIAGNOSTIC_PUSH
  JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
	tsd_t initializer = TSD_INITIALIZER;
  JEMALLOC_DIAGNOSTIC_POP
	wrapper->val = initializer;
	tsd_wrapper_set(wrapper);
}

JEMALLOC_ALWAYS_INLINE bool
tsd_boot(void) {
	if (tsd_boot0()) {
		return true;
	}
	tsd_boot1();
	return false;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_booted_get(void) {
	return tsd_booted;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_get_allocates(void) {
	return true;
}

/* Get/set. */
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init) {
	tsd_wrapper_t *wrapper;

	assert(tsd_booted);
	wrapper = tsd_wrapper_get(init);
	if (tsd_get_allocates() && !init && wrapper == NULL) {
		return NULL;
	}
	return &wrapper->val;
}

JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t *val) {
	tsd_wrapper_t *wrapper;

	assert(tsd_booted);
	wrapper = tsd_wrapper_get(true);
	if (likely(&wrapper->val != val)) {
		wrapper->val = *(val);
	}
	wrapper->initialized = true;
}