Lines Matching +full:- +full:- +full:requested +full:- +full:by
2 * kmp_alloc.cpp -- private/shared dynamic memory allocation and management
5 //===----------------------------------------------------------------------===//
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11 //===----------------------------------------------------------------------===//
38 /* The three modes of operation are, fifo search, lifo search, and best-fit */
59 /* On IA-32 architecture with Linux* OS, malloc() does not
128 // Declare the interface, including the requested buffer size type, bufsize.
148 char b_pad[sizeof(bhead2_t) + (SizeQuant - (sizeof(bhead2_t) % SizeQuant))];
153 /* Header in directly allocated buffers (by acqfcn) */
186 -1: not all pool blocks are the same size
189 bfhead_t *last_pool; /* Last pool owned by this thread (delay deallocation) */
197 ~(((bufsize)(1) << (sizeof(bufsize) * CHAR_BIT - 1)) | (SizeQuant - 1)))
198 // Maximum for the requested size.
205 ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))
210 int lo = 0, hi = MAX_BGET_BINS - 1; in bget_get_bin()
214 while ((hi - lo) > 1) { in bget_get_bin()
217 hi = mid - 1; in bget_get_bin()
231 data = (thr_data_t *)((!th->th.th_local.bget_data) in set_thr_data()
233 : th->th.th_local.bget_data); in set_thr_data()
238 data->freelist[i].ql.flink = &data->freelist[i]; in set_thr_data()
239 data->freelist[i].ql.blink = &data->freelist[i]; in set_thr_data()
242 th->th.th_local.bget_data = data; in set_thr_data()
243 th->th.th_local.bget_list = 0; in set_thr_data()
246 __kmp_init_lock(&th->th.th_local.bget_lock); in set_thr_data()
248 __kmp_init_bootstrap_lock(&th->th.th_local.bget_lock); in set_thr_data()
256 data = (thr_data_t *)th->th.th_local.bget_data; in get_thr_data()
265 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
270 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
271 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list, in __kmp_bget_dequeue()
274 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
280 __kmp_acquire_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th)); in __kmp_bget_dequeue()
282 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_dequeue()
285 p = (void *)th->th.th_local.bget_list; in __kmp_bget_dequeue()
286 th->th.th_local.bget_list = 0; in __kmp_bget_dequeue()
289 __kmp_release_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th)); in __kmp_bget_dequeue()
291 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_dequeue()
298 bfhead_t *b = BFH(((char *)p) - sizeof(bhead_t)); in __kmp_bget_dequeue()
300 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0); in __kmp_bget_dequeue()
301 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) == in __kmp_bget_dequeue()
303 KMP_DEBUG_ASSERT(b->ql.blink == 0); in __kmp_bget_dequeue()
305 p = (void *)b->ql.flink; in __kmp_bget_dequeue()
312 /* Chain together the free buffers by using the thread owner field */
319 bfhead_t *b = BFH(((char *)buf) - sizeof(bhead_t)); in __kmp_bget_enqueue()
321 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0); in __kmp_bget_enqueue()
322 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) == in __kmp_bget_enqueue()
325 b->ql.blink = 0; in __kmp_bget_enqueue()
332 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list); in __kmp_bget_enqueue()
335 b->ql.flink = BFH(CCAST(void *, old_value)); in __kmp_bget_enqueue()
337 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list, in __kmp_bget_enqueue()
340 old_value = TCR_PTR(th->th.th_local.bget_list); in __kmp_bget_enqueue()
343 b->ql.flink = BFH(CCAST(void *, old_value)); in __kmp_bget_enqueue()
348 __kmp_acquire_lock(&th->th.th_local.bget_lock, rel_gtid); in __kmp_bget_enqueue()
350 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_enqueue()
353 b->ql.flink = BFH(th->th.th_local.bget_list); in __kmp_bget_enqueue()
354 th->th.th_local.bget_list = (void *)buf; in __kmp_bget_enqueue()
357 __kmp_release_lock(&th->th.th_local.bget_lock, rel_gtid); in __kmp_bget_enqueue()
359 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_enqueue()
369 KMP_DEBUG_ASSERT(b->bh.bb.bsize % SizeQuant == 0); in __kmp_bget_insert_into_freelist()
371 bin = bget_get_bin(b->bh.bb.bsize); in __kmp_bget_insert_into_freelist()
373 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.blink->ql.flink == in __kmp_bget_insert_into_freelist()
374 &thr->freelist[bin]); in __kmp_bget_insert_into_freelist()
375 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.flink->ql.blink == in __kmp_bget_insert_into_freelist()
376 &thr->freelist[bin]); in __kmp_bget_insert_into_freelist()
378 b->ql.flink = &thr->freelist[bin]; in __kmp_bget_insert_into_freelist()
379 b->ql.blink = thr->freelist[bin].ql.blink; in __kmp_bget_insert_into_freelist()
381 thr->freelist[bin].ql.blink = b; in __kmp_bget_insert_into_freelist()
382 b->ql.blink->ql.flink = b; in __kmp_bget_insert_into_freelist()
387 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b); in __kmp_bget_remove_from_freelist()
388 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b); in __kmp_bget_remove_from_freelist()
390 b->ql.blink->ql.flink = b->ql.flink; in __kmp_bget_remove_from_freelist()
391 b->ql.flink->ql.blink = b->ql.blink; in __kmp_bget_remove_from_freelist()
394 /* GET STATS -- check info on free list */
404 best = &thr->freelist[bin]; in bcheck()
405 b = best->ql.flink; in bcheck()
407 while (b != &thr->freelist[bin]) { in bcheck()
408 *total_free += (b->bh.bb.bsize - sizeof(bhead_t)); in bcheck()
409 if ((best == &thr->freelist[bin]) || (b->bh.bb.bsize < best->bh.bb.bsize)) in bcheck()
413 b = b->ql.flink; in bcheck()
416 if (*max_free < best->bh.bb.bsize) in bcheck()
417 *max_free = best->bh.bb.bsize; in bcheck()
421 *max_free -= sizeof(bhead_t); in bcheck()
424 /* BGET -- Allocate a buffer. */
445 size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1)); in bget()
452 use_blink = (thr->mode == bget_mode_lifo); in bget()
463 b = (use_blink ? thr->freelist[bin].ql.blink in bget()
464 : thr->freelist[bin].ql.flink); in bget()
466 if (thr->mode == bget_mode_best) { in bget()
467 best = &thr->freelist[bin]; in bget()
470 to hold the requested size buffer. */ in bget()
471 while (b != &thr->freelist[bin]) { in bget()
472 if (b->bh.bb.bsize >= (bufsize)size) { in bget()
473 if ((best == &thr->freelist[bin]) || in bget()
474 (b->bh.bb.bsize < best->bh.bb.bsize)) { in bget()
480 b = (use_blink ? b->ql.blink : b->ql.flink); in bget()
485 while (b != &thr->freelist[bin]) { in bget()
486 if ((bufsize)b->bh.bb.bsize >= (bufsize)size) { in bget()
495 if ((b->bh.bb.bsize - (bufsize)size) > in bget()
499 ba = BH(((char *)b) + (b->bh.bb.bsize - (bufsize)size)); in bget()
502 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize); in bget()
505 b->bh.bb.bsize -= (bufsize)size; in bget()
508 ba->bb.prevfree = b->bh.bb.bsize; in bget()
511 ba->bb.bsize = -size; in bget()
513 /* Mark this buffer as owned by this thread. */ in bget()
514 TCW_PTR(ba->bb.bthr, in bget()
516 /* Mark buffer after this one not preceded by free block. */ in bget()
517 bn->bb.prevfree = 0; in bget()
523 thr->totalloc += (size_t)size; in bget()
524 thr->numget++; /* Increment number of bget() calls */ in bget()
532 ba = BH(((char *)b) + b->bh.bb.bsize); in bget()
534 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize); in bget()
541 thr->totalloc += (size_t)b->bh.bb.bsize; in bget()
542 thr->numget++; /* Increment number of bget() calls */ in bget()
545 b->bh.bb.bsize = -(b->bh.bb.bsize); in bget()
547 /* Mark this buffer as owned by this thread. */ in bget()
548 TCW_PTR(ba->bb.bthr, th); // not an allocated address (do not mark) in bget()
551 ba->bb.prevfree = 0; in bget()
554 buf = (void *)&(b->ql); in bget()
561 b = (use_blink ? b->ql.blink : b->ql.flink); in bget()
566 notify it of the size requested. If it returns TRUE, try the allocation in bget()
569 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) { in bget()
574 /* No buffer available with requested size free. */ in bget()
576 /* Don't give up yet -- look in the reserve supply. */ in bget()
577 if (thr->acqfcn != 0) { in bget()
578 if (size > (bufsize)(thr->exp_incr - sizeof(bhead_t))) { in bget()
580 Try to satisfy it by a direct buffer acquisition. */ in bget()
583 size += sizeof(bdhead_t) - sizeof(bhead_t); in bget()
588 bdh = BDH((*thr->acqfcn)((bufsize)size)); in bget()
591 // Mark the buffer special by setting size field of its header to zero. in bget()
592 bdh->bh.bb.bsize = 0; in bget()
594 /* Mark this buffer as owned by this thread. */ in bget()
595 TCW_PTR(bdh->bh.bb.bthr, th); // don't mark buffer as allocated, in bget()
597 bdh->bh.bb.prevfree = 0; in bget()
598 bdh->tsize = size; in bget()
600 thr->totalloc += (size_t)size; in bget()
601 thr->numget++; /* Increment number of bget() calls */ in bget()
602 thr->numdget++; /* Direct bget() call count */ in bget()
614 KE_TRACE(10, ("%%%%%% MALLOCB( %d )\n", (int)thr->exp_incr)); in bget()
617 newpool = (*thr->acqfcn)((bufsize)thr->exp_incr); in bget()
620 bpool(th, newpool, thr->exp_incr); in bget()
633 /* BGETZ -- Allocate a buffer and clear its contents to zero. We clear
635 region requested by the caller. */
644 b = BH(buf - sizeof(bhead_t)); in bgetz()
645 rsize = -(b->bb.bsize); in bgetz()
649 bd = BDH(buf - sizeof(bdhead_t)); in bgetz()
650 rsize = bd->tsize - (bufsize)sizeof(bdhead_t); in bgetz()
652 rsize -= sizeof(bhead_t); in bgetz()
662 /* BGETR -- Reallocate a buffer. This is a minimal implementation,
679 b = BH(((char *)buf) - sizeof(bhead_t)); in bgetr()
680 osize = -b->bb.bsize; in bgetr()
685 bd = BDH(((char *)buf) - sizeof(bdhead_t)); in bgetr()
686 osize = bd->tsize - (bufsize)sizeof(bdhead_t); in bgetr()
688 osize -= sizeof(bhead_t); in bgetr()
700 /* BREL -- Release a buffer. */
709 b = BFH(((char *)buf) - sizeof(bhead_t)); in brel()
711 if (b->bh.bb.bsize == 0) { /* Directly-acquired buffer? */ in brel()
714 bdh = BDH(((char *)buf) - sizeof(bdhead_t)); in brel()
715 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0); in brel()
717 thr->totalloc -= (size_t)bdh->tsize; in brel()
718 thr->numdrel++; /* Number of direct releases */ in brel()
719 thr->numrel++; /* Increment number of brel() calls */ in brel()
722 (void)memset((char *)buf, 0x55, (size_t)(bdh->tsize - sizeof(bdhead_t))); in brel()
727 KMP_DEBUG_ASSERT(thr->relfcn != 0); in brel()
728 (*thr->relfcn)((void *)bdh); /* Release it directly. */ in brel()
732 bth = (kmp_info_t *)((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & in brel()
735 /* Add this buffer to be released by the owning thread later */ in brel()
746 if (b->bh.bb.bsize >= 0) { in brel()
749 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0); in brel()
753 KMP_DEBUG_ASSERT(BH((char *)b - b->bh.bb.bsize)->bb.prevfree == 0); in brel()
756 thr->numrel++; /* Increment number of brel() calls */ in brel()
757 thr->totalloc += (size_t)b->bh.bb.bsize; in brel()
762 if (b->bh.bb.prevfree != 0) { in brel()
763 /* The previous buffer is free. Consolidate this buffer with it by adding in brel()
767 bufsize size = b->bh.bb.bsize; in brel()
770 KMP_DEBUG_ASSERT(BH((char *)b - b->bh.bb.prevfree)->bb.bsize == in brel()
771 b->bh.bb.prevfree); in brel()
772 b = BFH(((char *)b) - b->bh.bb.prevfree); in brel()
773 b->bh.bb.bsize -= size; in brel()
781 b->bh.bb.bsize = -b->bh.bb.bsize; in brel()
787 /* Now we look at the next buffer in memory, located by advancing from in brel()
788 the start of this buffer by its size, to see if that buffer is in brel()
791 bn = BFH(((char *)b) + b->bh.bb.bsize); in brel()
792 if (bn->bh.bb.bsize > 0) { in brel()
796 KMP_DEBUG_ASSERT(BH((char *)bn + bn->bh.bb.bsize)->bb.prevfree == in brel()
797 bn->bh.bb.bsize); in brel()
801 b->bh.bb.bsize += bn->bh.bb.bsize; in brel()
814 bn = BFH(((char *)b) + b->bh.bb.bsize); in brel()
818 (size_t)(b->bh.bb.bsize - sizeof(bfhead_t))); in brel()
820 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0); in brel()
825 bn->bh.bb.prevfree = b->bh.bb.bsize; in brel()
827 /* If a block-release function is defined, and this free buffer in brel()
831 if (thr->relfcn != 0 && in brel()
832 b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t))) { in brel()
834 if (thr->numpblk != in brel()
838 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0); in brel()
839 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.bsize == ESent); in brel()
840 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.prevfree == in brel()
841 b->bh.bb.bsize); in brel()
848 (*thr->relfcn)(b); in brel()
850 thr->numprel++; /* Nr of expansion block releases */ in brel()
851 thr->numpblk--; /* Total number of blocks */ in brel()
852 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel); in brel()
855 if (thr->last_pool == b) in brel()
856 thr->last_pool = 0; in brel()
858 thr->last_pool = b; in brel()
864 /* BECTL -- Establish automatic pool expansion control */
870 thr->compfcn = compact; in bectl()
871 thr->acqfcn = acquire; in bectl()
872 thr->relfcn = release; in bectl()
873 thr->exp_incr = pool_incr; in bectl()
876 /* BPOOL -- Add a region of memory to the buffer pool. */
886 len &= ~((bufsize)(SizeQuant - 1)); in bpool()
888 if (thr->pool_len == 0) { in bpool()
889 thr->pool_len = len; in bpool()
890 } else if (len != thr->pool_len) { in bpool()
891 thr->pool_len = -1; in bpool()
894 thr->numpget++; /* Number of block acquisitions */ in bpool()
895 thr->numpblk++; /* Number of blocks total */ in bpool()
896 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel); in bpool()
899 /* Since the block is initially occupied by a single free buffer, in bpool()
902 KMP_DEBUG_ASSERT(len - sizeof(bhead_t) <= -((bufsize)ESent + 1)); in bpool()
907 b->bh.bb.prevfree = 0; in bpool()
914 routines (this specific value is not counted on by the actual in bpool()
916 len -= sizeof(bhead_t); in bpool()
917 b->bh.bb.bsize = (bufsize)len; in bpool()
919 TCW_PTR(b->bh.bb.bthr, in bpool()
928 (size_t)(len - sizeof(bfhead_t))); in bpool()
931 bn->bb.prevfree = (bufsize)len; in bpool()
933 KMP_DEBUG_ASSERT((~0) == -1 && (bn != 0)); in bpool()
935 bn->bb.bsize = ESent; in bpool()
938 /* BFREED -- Dump the free lists for this thread. */
950 gtid, (kmp_uint64)thr->totalloc, (kmp_int64)thr->numget, in bfreed()
951 (kmp_int64)thr->numrel, (kmp_int64)thr->numpblk, in bfreed()
952 (kmp_int64)thr->numpget, (kmp_int64)thr->numprel, in bfreed()
953 (kmp_int64)thr->numdget, (kmp_int64)thr->numdrel); in bfreed()
959 for (b = thr->freelist[bin].ql.flink; b != &thr->freelist[bin]; in bfreed()
960 b = b->ql.flink) { in bfreed()
961 bufsize bs = b->bh.bb.bsize; in bfreed()
963 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b); in bfreed()
964 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b); in bfreed()
977 (memcmp(lerr, lerr + 1, (size_t)(bs - (sizeof(bfhead_t) + 1))) != in bfreed()
1008 thr = (thr_data_t *)th->th.th_local.bget_data; in __kmp_finalize_bget()
1010 b = thr->last_pool; in __kmp_finalize_bget()
1012 /* If a block-release function is defined, and this free buffer constitutes in __kmp_finalize_bget()
1017 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 && in __kmp_finalize_bget()
1018 b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t))) { in __kmp_finalize_bget()
1019 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0); in __kmp_finalize_bget()
1020 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.bsize == ESent); in __kmp_finalize_bget()
1021 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.prevfree == in __kmp_finalize_bget()
1022 b->bh.bb.bsize); in __kmp_finalize_bget()
1029 (*thr->relfcn)(b); in __kmp_finalize_bget()
1030 thr->numprel++; /* Nr of expansion block releases */ in __kmp_finalize_bget()
1031 thr->numpblk--; /* Total number of blocks */ in __kmp_finalize_bget()
1032 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel); in __kmp_finalize_bget()
1037 if (th->th.th_local.bget_data != NULL) { in __kmp_finalize_bget()
1038 __kmp_free(th->th.th_local.bget_data); in __kmp_finalize_bget()
1039 th->th.th_local.bget_data = NULL; in __kmp_finalize_bget()
1053 return p->exp_incr; in kmpc_get_poolsize()
1062 p->mode = (bget_mode_t)mode; in kmpc_set_poolmode()
1071 return p->mode; in kmpc_get_poolmode()
1107 #define IS_POWER_OF_TWO(n) (((n) & ((n)-1)) == 0)
1123 ~(alignment - 1)); in kmpc_aligned_malloc()
1124 *((void **)ptr - 1) = ptr_allocated; in kmpc_aligned_malloc()
1154 // The thread must be registered by the call to kmpc_malloc() or in kmpc_realloc()
1158 KMP_ASSERT(*((void **)ptr - 1)); in kmpc_realloc()
1159 brel(__kmp_get_thread(), *((void **)ptr - 1)); in kmpc_realloc()
1161 result = bgetr(__kmp_entry_thread(), *((void **)ptr - 1), in kmpc_realloc()
1171 // NOTE: the library must have already been initialized by a previous allocate
1180 KMP_ASSERT(*((void **)ptr - 1)); in kmpc_free()
1181 brel(th, *((void **)ptr - 1)); in kmpc_free()
1187 KE_TRACE(30, ("-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n", th, in ___kmp_thread_malloc()
1190 KE_TRACE(30, ("<- __kmp_thread_malloc() returns %p\n", ptr)); in ___kmp_thread_malloc()
1197 KE_TRACE(30, ("-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n", th, in ___kmp_thread_calloc()
1200 KE_TRACE(30, ("<- __kmp_thread_calloc() returns %p\n", ptr)); in ___kmp_thread_calloc()
1206 KE_TRACE(30, ("-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n", th, in ___kmp_thread_realloc()
1209 KE_TRACE(30, ("<- __kmp_thread_realloc() returns %p\n", ptr)); in ___kmp_thread_realloc()
1214 KE_TRACE(30, ("-> __kmp_thread_free( %p, %p ) called from %s:%d\n", th, in ___kmp_thread_free()
1220 KE_TRACE(30, ("<- __kmp_thread_free()\n")); in ___kmp_thread_free()
1273 // as of 2018-07-31 memkind does not support Windows*, exclude it for now in __kmp_init_memkind()
1389 al->memspace = ms; // not used currently in __kmpc_init_allocator()
1396 al->pinned = true; in __kmpc_init_allocator()
1399 __kmp_type_convert(traits[i].value, &(al->alignment)); in __kmpc_init_allocator()
1400 KMP_ASSERT(IS_POWER_OF_TWO(al->alignment)); in __kmpc_init_allocator()
1403 al->pool_size = traits[i].value; in __kmpc_init_allocator()
1406 al->fb = (omp_alloctrait_value_t)traits[i].value; in __kmpc_init_allocator()
1408 al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb || in __kmpc_init_allocator()
1409 al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb); in __kmpc_init_allocator()
1412 al->fb_data = RCAST(kmp_allocator_t *, traits[i].value); in __kmpc_init_allocator()
1415 al->memkind = RCAST(void **, traits[i].value); in __kmpc_init_allocator()
1421 if (al->fb == 0) { in __kmpc_init_allocator()
1423 al->fb = omp_atv_default_mem_fb; in __kmpc_init_allocator()
1424 al->fb_data = (kmp_allocator_t *)omp_default_mem_alloc; in __kmpc_init_allocator()
1425 } else if (al->fb == omp_atv_allocator_fb) { in __kmpc_init_allocator()
1426 KMP_ASSERT(al->fb_data != NULL); in __kmpc_init_allocator()
1427 } else if (al->fb == omp_atv_default_mem_fb) { in __kmpc_init_allocator()
1428 al->fb_data = (kmp_allocator_t *)omp_default_mem_alloc; in __kmpc_init_allocator()
1433 if (al->memkind == (void *)omp_atv_interleaved && mk_hbw_interleave) { in __kmpc_init_allocator()
1434 al->memkind = mk_hbw_interleave; in __kmpc_init_allocator()
1441 al->memkind = mk_hbw_preferred; in __kmpc_init_allocator()
1443 // HBW is requested but not available --> return NULL allocator in __kmpc_init_allocator()
1450 al->memkind = mk_dax_kmem_all; in __kmpc_init_allocator()
1453 al->memkind = mk_dax_kmem; in __kmpc_init_allocator()
1459 if (al->memkind == (void *)omp_atv_interleaved && mk_interleave) { in __kmpc_init_allocator()
1460 al->memkind = mk_interleave; in __kmpc_init_allocator()
1462 al->memkind = mk_default; in __kmpc_init_allocator()
1486 __kmp_threads[gtid]->th.th_def_allocator = allocator; in __kmpc_set_default_allocator()
1490 return __kmp_threads[gtid]->th.th_def_allocator; in __kmpc_get_default_allocator()
1494 void *ptr_alloc; // Pointer returned by allocator
1496 size_t size_orig; // Original size requested
1500 static int alignment = sizeof(void *); // align to pointer size by default
1554 allocator = __kmp_threads[gtid]->th.th_def_allocator; in __kmp_alloc()
1556 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device; in __kmp_alloc()
1562 kmp_uintptr_t addr; // address returned by allocator in __kmp_alloc()
1566 if (allocator > kmp_max_mem_alloc && al->alignment > align) in __kmp_alloc()
1567 align = al->alignment; // alignment required by allocator trait in __kmp_alloc()
1574 is_pinned = al->pinned; in __kmp_alloc()
1584 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device; in __kmp_alloc()
1597 if (allocator >= kmp_max_mem_alloc && KMP_IS_TARGET_MEM_SPACE(al->memspace)) { in __kmp_alloc()
1600 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device; in __kmp_alloc()
1601 if (al->memspace == llvm_omp_target_host_mem_space) in __kmp_alloc()
1603 else if (al->memspace == llvm_omp_target_shared_mem_space) in __kmp_alloc()
1605 else // al->memspace == llvm_omp_target_device_mem_space in __kmp_alloc()
1615 // pre-defined allocator in __kmp_alloc()
1623 } else if (al->pool_size > 0) { in __kmp_alloc()
1624 // custom allocator with pool size requested in __kmp_alloc()
1626 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, desc.size_a); in __kmp_alloc()
1627 if (used + desc.size_a > al->pool_size) { in __kmp_alloc()
1629 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a); in __kmp_alloc()
1630 if (al->fb == omp_atv_default_mem_fb) { in __kmp_alloc()
1633 } else if (al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1634 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1635 } else if (al->fb == omp_atv_allocator_fb) { in __kmp_alloc()
1636 KMP_ASSERT(al != al->fb_data); in __kmp_alloc()
1637 al = al->fb_data; in __kmp_alloc()
1645 ptr = kmp_mk_alloc(*al->memkind, desc.size_a); in __kmp_alloc()
1647 if (al->fb == omp_atv_default_mem_fb) { in __kmp_alloc()
1650 } else if (al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1651 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1652 } else if (al->fb == omp_atv_allocator_fb) { in __kmp_alloc()
1653 KMP_ASSERT(al != al->fb_data); in __kmp_alloc()
1654 al = al->fb_data; in __kmp_alloc()
1663 // custom allocator, pool size not requested in __kmp_alloc()
1664 ptr = kmp_mk_alloc(*al->memkind, desc.size_a); in __kmp_alloc()
1666 if (al->fb == omp_atv_default_mem_fb) { in __kmp_alloc()
1669 } else if (al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1670 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1671 } else if (al->fb == omp_atv_allocator_fb) { in __kmp_alloc()
1672 KMP_ASSERT(al != al->fb_data); in __kmp_alloc()
1673 al = al->fb_data; in __kmp_alloc()
1682 // pre-defined allocator in __kmp_alloc()
1697 } else { // default allocator requested in __kmp_alloc()
1704 } else if (al->pool_size > 0) { in __kmp_alloc()
1705 // custom allocator with pool size requested in __kmp_alloc()
1707 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, desc.size_a); in __kmp_alloc()
1708 if (used + desc.size_a > al->pool_size) { in __kmp_alloc()
1710 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a); in __kmp_alloc()
1711 if (al->fb == omp_atv_default_mem_fb) { in __kmp_alloc()
1714 } else if (al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1715 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1716 } else if (al->fb == omp_atv_allocator_fb) { in __kmp_alloc()
1717 KMP_ASSERT(al != al->fb_data); in __kmp_alloc()
1718 al = al->fb_data; in __kmp_alloc()
1727 if (ptr == NULL && al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1728 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1732 // custom allocator, pool size not requested in __kmp_alloc()
1734 if (ptr == NULL && al->fb == omp_atv_abort_fb) { in __kmp_alloc()
1735 KMP_ASSERT(0); // abort fallback requested in __kmp_alloc()
1746 addr_align = (addr + sz_desc + align - 1) & ~(align - 1); in __kmp_alloc()
1747 addr_descr = addr_align - sz_desc; in __kmp_alloc()
1765 allocator = __kmp_threads[gtid]->th.th_def_allocator; in __kmp_calloc()
1772 if ((SIZE_MAX - sizeof(kmp_mem_desc_t)) / size < nmemb) { in __kmp_calloc()
1773 if (al->fb == omp_atv_abort_fb) { in __kmp_calloc()
1807 addr_descr = addr_align - sizeof(kmp_mem_desc_t); in __kmp_realloc()
1836 KMP_IS_TARGET_MEM_SPACE(al->memspace)))) { in ___kmpc_free()
1838 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device; in ___kmpc_free()
1850 addr_descr = addr_align - sizeof(kmp_mem_desc_t); in ___kmpc_free()
1855 KMP_DEBUG_ASSERT(desc.allocator == al || desc.allocator == al->fb_data); in ___kmpc_free()
1861 if (allocator > kmp_max_mem_alloc && kmp_target_unlock_mem && al->pinned) { in ___kmpc_free()
1863 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device; in ___kmpc_free()
1869 // pre-defined allocator in ___kmpc_free()
1878 if (al->pool_size > 0) { // custom allocator with pool size requested in ___kmpc_free()
1880 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a); in ___kmpc_free()
1884 kmp_mk_free(*al->memkind, desc.ptr_alloc); in ___kmpc_free()
1887 if (oal > kmp_max_mem_alloc && al->pool_size > 0) { in ___kmpc_free()
1889 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a); in ___kmpc_free()
1902 void *ptr_allocated; // Pointer returned by malloc(), subject for free().
1904 void *ptr_aligned; // Pointer to aligned memory, to be used by client code.
1909 /* Allocate memory on requested boundary, fill allocated memory with 0x00.
1911 error. Must use __kmp_free when freeing memory allocated by this routine! */
1914 /* __kmp_allocate() allocates (by call to malloc()) bigger memory block than in ___kmp_allocate_align()
1915 requested to return properly aligned pointer. Original pointer returned in ___kmp_allocate_align()
1916 by malloc() and size of allocated block is saved in descriptor just in ___kmp_allocate_align()
1917 before the aligned pointer. This information used by __kmp_free() -- it in ___kmp_allocate_align()
1920 +---------+------------+-----------------------------------+---------+ in ___kmp_allocate_align()
1922 +---------+------------+-----------------------------------+---------+ in ___kmp_allocate_align()
1925 | +- Aligned pointer returned to caller in ___kmp_allocate_align()
1926 +- Pointer returned by malloc() in ___kmp_allocate_align()
1931 kmp_uintptr_t addr_allocated; // Address returned by malloc(). in ___kmp_allocate_align()
1935 KE_TRACE(25, ("-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n", in ___kmp_allocate_align()
1959 (addr_allocated + sizeof(kmp_mem_descr_t) + alignment) & ~(alignment - 1); in ___kmp_allocate_align()
1960 addr_descr = addr_aligned - sizeof(kmp_mem_descr_t); in ___kmp_allocate_align()
1980 // Fill the aligned memory block (which is intended for using by caller) with in ___kmp_allocate_align()
1989 KE_TRACE(25, ("<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned)); in ___kmp_allocate_align()
1996 error. Must use __kmp_free when freeing memory allocated by this routine! */
1999 KE_TRACE(25, ("-> __kmp_allocate( %d ) called from %s:%d\n", in ___kmp_allocate()
2002 KE_TRACE(25, ("<- __kmp_allocate() returns %p\n", ptr)); in ___kmp_allocate()
2009 error. Must use __kmp_free when freeing memory allocated by this routine! */
2014 KE_TRACE(25, ("-> __kmp_page_allocate( %d ) called from %s:%d\n", in ___kmp_page_allocate()
2017 KE_TRACE(25, ("<- __kmp_page_allocate( %d ) returns %p\n", (int)size, ptr)); in ___kmp_page_allocate()
2021 /* Free memory allocated by __kmp_allocate() and __kmp_page_allocate().
2026 kmp_uintptr_t addr_allocated; // Address returned by malloc(). in ___kmp_free()
2027 kmp_uintptr_t addr_aligned; // Aligned address passed by caller. in ___kmp_free()
2030 ("-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM)); in ___kmp_free()
2033 descr = *(kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t)); in ___kmp_free()
2062 KE_TRACE(25, ("<- __kmp_free() returns\n")); in ___kmp_free()
2066 // Allocate fast memory by first scanning the thread's free lists
2084 KE_TRACE(25, ("-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n", in ___kmp_fast_allocate()
2087 num_lines = (size + DCACHE_LINE - 1) / DCACHE_LINE; in ___kmp_fast_allocate()
2088 idx = num_lines - 1; in ___kmp_fast_allocate()
2106 ptr = this_thr->th.th_free_lists[index].th_free_list_self; in ___kmp_fast_allocate()
2108 // pop the head of no-sync free list in ___kmp_fast_allocate()
2109 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr); in ___kmp_fast_allocate()
2110 KMP_DEBUG_ASSERT(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - in ___kmp_fast_allocate()
2112 ->ptr_aligned); in ___kmp_fast_allocate()
2115 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_allocate()
2117 // no-sync free list is empty, use sync free list (filled in by other in ___kmp_fast_allocate()
2121 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, nullptr)) { in ___kmp_fast_allocate()
2123 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_allocate()
2125 // push the rest of chain into no-sync free list (can be NULL if there was in ___kmp_fast_allocate()
2127 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr); in ___kmp_fast_allocate()
2128 KMP_DEBUG_ASSERT(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - in ___kmp_fast_allocate()
2130 ->ptr_aligned); in ___kmp_fast_allocate()
2147 ~(DCACHE_LINE - 1)); in ___kmp_fast_allocate()
2148 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t)); in ___kmp_fast_allocate()
2150 descr->ptr_allocated = alloc_ptr; // remember allocated pointer in ___kmp_fast_allocate()
2152 descr->ptr_aligned = (void *)this_thr; // remember allocating thread in ___kmp_fast_allocate()
2155 descr->size_aligned = size; in ___kmp_fast_allocate()
2158 KE_TRACE(25, ("<- __kmp_fast_allocate( T#%d ) returns %p\n", in ___kmp_fast_allocate()
2172 KE_TRACE(25, ("-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n", in ___kmp_fast_free()
2176 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t)); in ___kmp_fast_free()
2179 (int)descr->size_aligned)); in ___kmp_fast_free()
2181 size = descr->size_aligned; // 2, 4, 16, 64, 65, 66, ... cache lines in ___kmp_fast_free()
2197 alloc_thr = (kmp_info_t *)descr->ptr_aligned; // get thread owning the block in ___kmp_fast_free()
2199 // push block to self no-sync free list, linking previous head (LIFO) in ___kmp_fast_free()
2200 *((void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self; in ___kmp_fast_free()
2201 this_thr->th.th_free_lists[index].th_free_list_self = ptr; in ___kmp_fast_free()
2203 void *head = this_thr->th.th_free_lists[index].th_free_list_other; in ___kmp_fast_free()
2206 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
2208 descr->size_allocated = (size_t)1; // head of the list keeps its length in ___kmp_fast_free()
2212 (kmp_mem_descr_t *)((char *)head - sizeof(kmp_mem_descr_t)); in ___kmp_fast_free()
2214 kmp_info_t *q_th = (kmp_info_t *)(dsc->ptr_aligned); in ___kmp_fast_free()
2216 dsc->size_allocated + 1; // new size in case we add current task in ___kmp_fast_free()
2220 descr->size_allocated = q_sz; in ___kmp_fast_free()
2221 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
2231 // queue size should decrease by 1 each step through the list in ___kmp_fast_free()
2232 ((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) in ___kmp_fast_free()
2233 ->size_allocated + in ___kmp_fast_free()
2235 ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) in ___kmp_fast_free()
2236 ->size_allocated); in ___kmp_fast_free()
2242 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_free()
2248 &q_th->th.th_free_lists[index].th_free_list_sync, old_ptr, head)) { in ___kmp_fast_free()
2250 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_free()
2254 // start new list of not-selt tasks in ___kmp_fast_free()
2255 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
2257 descr->size_allocated = (size_t)1; // head of queue keeps its length in ___kmp_fast_free()
2267 brel(this_thr, descr->ptr_allocated); in ___kmp_fast_free()
2270 KE_TRACE(25, ("<- __kmp_fast_free() returns\n")); in ___kmp_fast_free()
2279 memset(this_thr->th.th_free_lists, 0, NUM_LISTS * sizeof(kmp_free_list_t)); in __kmp_initialize_fast_memory()
2297 bfhead_t *b = thr->freelist[bin].ql.flink; in __kmp_free_fast_memory()
2298 while (b != &thr->freelist[bin]) { in __kmp_free_fast_memory()
2299 if ((kmp_uintptr_t)b->bh.bb.bthr & 1) { // the buffer is allocated address in __kmp_free_fast_memory()
2304 b = b->ql.flink; // get next buffer in __kmp_free_fast_memory()
2311 (*thr->relfcn)(lst); in __kmp_free_fast_memory()
2314 thr->numprel++; /* Nr of expansion block releases */ in __kmp_free_fast_memory()
2315 thr->numpblk--; /* Total number of blocks */ in __kmp_free_fast_memory()