Lines Matching full:region
33 // PrimaryEnableRandomOffset is set, each Region actually starts at a random
41 // The 1st Region (for size class 0) holds the TransferBatches. This is a
57 "Group size shouldn't be greater than the region size");
138 RegionInfo *Region = getRegionInfo(I);
140 initRegion(Region, I, RegionMemMap, Config::getEnableRandomOffset());
145 // The binding should be done after region shuffling so that it won't bind
146 // the FLLock from the wrong region.
158 RegionInfo *Region = getRegionInfo(I);
160 ScopedLock ML(Region->MMLock);
161 MemMapT MemMap = Region->MemMapInfo.MemMap;
165 *Region = {};
177 RegionInfo *Region = getRegionInfo(I);
178 ScopedLock ML(Region->MMLock);
179 ScopedLock FL(Region->FLLock);
182 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) {
189 DCHECK_EQ(TotalBlocks, Region->MemMapInfo.AllocatedUser / BlockSize);
190 DCHECK_EQ(Region->FreeListInfo.PushedBlocks,
191 Region->FreeListInfo.PoppedBlocks);
194 RegionInfo *Region = getRegionInfo(SizeClassMap::BatchClassId);
195 ScopedLock ML(Region->MMLock);
196 ScopedLock FL(Region->FLLock);
199 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) {
210 Region->MemMapInfo.AllocatedUser / BlockSize);
211 DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
212 Region->FreeListInfo.PushedBlocks);
214 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
221 RegionInfo *Region = getRegionInfo(ClassId);
225 ScopedLock L(Region->FLLock);
226 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
234 PopCount = popBlocksWithCV(C, ClassId, Region, ToArray, MaxBlockCount,
238 // When two threads compete for `Region->MMLock`, we only want one of
241 ScopedLock ML(Region->MMLock);
243 ScopedLock FL(Region->FLLock);
244 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
249 const bool RegionIsExhausted = Region->Exhausted;
251 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
254 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
277 RegionInfo *Region = getRegionInfo(ClassId);
279 ScopedLock L(Region->FLLock);
280 pushBatchClassBlocks(Region, Array, Size);
282 Region->FLLockCV.notifyAll(Region->FLLock);
307 ScopedLock L(Region->FLLock);
308 pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup);
310 Region->FLLockCV.notifyAll(Region->FLLock);
341 RegionInfo *Region = getRegionInfo(I);
343 // SizeClassAllocator64. We may consider locking each region on demand
345 Region->FLLock.assertHeld();
346 Region->MMLock.assertHeld();
348 const uptr From = Region->RegionBeg;
349 const uptr To = From + Region->MemMapInfo.AllocatedUser;
356 // TODO(kostyak): get the RSS per region.
361 RegionInfo *Region = getRegionInfo(I);
363 ScopedLock L(Region->MMLock);
364 TotalMapped += Region->MemMapInfo.MappedUser;
367 ScopedLock L(Region->FLLock);
368 PoppedBlocks += Region->FreeListInfo.PoppedBlocks;
369 PushedBlocks += Region->FreeListInfo.PushedBlocks;
379 RegionInfo *Region = getRegionInfo(I);
380 ScopedLock L1(Region->MMLock);
381 ScopedLock L2(Region->FLLock);
382 getStats(Str, I, Region);
392 RegionInfo *Region = getRegionInfo(I);
393 ScopedLock L(Region->MMLock);
394 getRegionFragmentationInfo(Region, I, Str);
411 RegionInfo *Region = getRegionInfo(ClassId);
415 if (Region->MMLock.tryLock()) {
416 uptr BytesReleased = releaseToOSMaybe(Region, ClassId, ReleaseType);
417 Region->MMLock.unlock();
428 RegionInfo *Region = getRegionInfo(I);
429 ScopedLock L(Region->MMLock);
430 TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType);
569 RegionInfo *Region = getRegionInfo(ClassId);
570 Region->MMLock.assertHeld();
573 !Region->MemMapInfo.MemMap.isAllocated()) {
576 return Region->MemMapInfo.MemMap.getBase();
606 ALWAYS_INLINE void initRegion(RegionInfo *Region, uptr ClassId,
608 REQUIRES(Region->MMLock) {
609 DCHECK(!Region->MemMapInfo.MemMap.isAllocated());
614 Region->MemMapInfo.MemMap = MemMap;
616 Region->RegionBeg = MemMap.getBase();
618 Region->RegionBeg +=
619 (getRandomModN(&Region->RandState, 16) + 1) * PageSize;
625 Region->TryReleaseThreshold = PageSize * SmallerBlockReleasePageDelta;
627 Region->TryReleaseThreshold = PageSize;
630 void pushBatchClassBlocks(RegionInfo *Region, CompactPtrT *Array, u32 Size)
631 REQUIRES(Region->FLLock) {
632 DCHECK_EQ(Region, getRegionInfo(SizeClassMap::BatchClassId));
669 Region->FreeListInfo.PushedBlocks += Size;
670 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
688 Region->FreeListInfo.BlockList.push_front(BG);
757 void pushBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
759 REQUIRES(Region->FLLock) {
805 Region->FreeListInfo.PushedBlocks += Size;
806 BatchGroupT *Cur = Region->FreeListInfo.BlockList.front();
822 Region->FreeListInfo.BlockList.push_front(Cur);
824 Region->FreeListInfo.BlockList.insert(Prev, Cur);
855 Region->FreeListInfo.BlockList.insert(Prev, Cur);
867 u16 popBlocksWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
879 ScopedLock FL(Region->FLLock);
880 if (!Region->isPopulatingFreeList) {
881 Region->isPopulatingFreeList = true;
887 ScopedLock ML(Region->MMLock);
889 const bool RegionIsExhausted = Region->Exhausted;
891 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
894 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
900 // `Region->isPopulatingFreeList` to false so the threads about to
902 ScopedLock FL(Region->FLLock);
903 Region->isPopulatingFreeList = false;
904 Region->FLLockCV.notifyAll(Region->FLLock);
912 // 2. Region->isPopulatingFreeList == true, i.e, someone is still doing
916 // Region->isPopulatingFreeList == false because all the new populated
919 ScopedLock FL(Region->FLLock);
920 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
924 if (!Region->isPopulatingFreeList)
932 Region->FLLockCV.wait(Region->FLLock);
934 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
942 u16 popBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
944 REQUIRES(Region->FLLock) {
945 if (Region->FreeListInfo.BlockList.empty())
949 Region->FreeListInfo.BlockList.front()->Batches;
953 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
954 Region->FreeListInfo.BlockList.pop_front();
961 Region->FreeListInfo.PoppedBlocks += 1;
995 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
996 Region->FreeListInfo.BlockList.pop_front();
1008 Region->FreeListInfo.PoppedBlocks += PopCount;
1014 RegionInfo *Region,
1017 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1019 !Region->MemMapInfo.MemMap.isAllocated()) {
1028 initRegion(Region, ClassId,
1034 DCHECK(Region->MemMapInfo.MemMap.isAllocated());
1037 const uptr RegionBeg = Region->RegionBeg;
1038 const uptr MappedUser = Region->MemMapInfo.MappedUser;
1040 Region->MemMapInfo.AllocatedUser + MaxCount * Size;
1048 Region->Exhausted = true;
1052 if (UNLIKELY(!Region->MemMapInfo.MemMap.remap(
1059 Region->MemMapInfo.MappedUser += MapSize;
1065 static_cast<u32>((Region->MemMapInfo.MappedUser -
1066 Region->MemMapInfo.AllocatedUser) /
1076 uptr P = RegionBeg + Region->MemMapInfo.AllocatedUser;
1080 ScopedLock L(Region->FLLock);
1087 shuffle(ShuffleArray + I - N, N, &Region->RandState);
1088 pushBlocksImpl(C, ClassId, Region, ShuffleArray + I - N, N,
1097 shuffle(ShuffleArray + NumberOfBlocks - N, N, &Region->RandState);
1098 pushBlocksImpl(C, ClassId, Region, &ShuffleArray[NumberOfBlocks - N], N,
1101 pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks);
1105 popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
1112 Region->FreeListInfo.PushedBlocks -= NumberOfBlocks;
1116 Region->MemMapInfo.AllocatedUser += AllocatedUser;
1121 void getStats(ScopedString *Str, uptr ClassId, RegionInfo *Region)
1122 REQUIRES(Region->MMLock, Region->FLLock) {
1123 if (Region->MemMapInfo.MappedUser == 0)
1127 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
1129 Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize;
1132 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) {
1134 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
1136 const uptr TotalChunks = Region->MemMapInfo.AllocatedUser / BlockSize;
1140 "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n",
1141 Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId),
1142 Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks,
1143 Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks,
1144 Region->ReleaseInfo.RangesReleased,
1145 Region->ReleaseInfo.LastReleasedBytes >> 10,
1146 RegionPushedBytesDelta >> 10, Region->RegionBeg,
1150 void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId,
1151 ScopedString *Str) REQUIRES(Region->MMLock) {
1154 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1158 ScopedLock L(Region->FLLock);
1159 GroupsToRelease = Region->FreeListInfo.BlockList;
1160 Region->FreeListInfo.BlockList.clear();
1166 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1171 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1174 ScopedLock L(Region->FLLock);
1176 const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize;
1178 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
1180 roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize;
1196 NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
1198 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1202 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1206 ScopedLock L(Region->FLLock);
1208 BytesInFreeList = Region->MemMapInfo.AllocatedUser -
1209 (Region->FreeListInfo.PoppedBlocks -
1210 Region->FreeListInfo.PushedBlocks) *
1220 !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList,
1230 GroupsToRelease = Region->FreeListInfo.BlockList;
1231 Region->FreeListInfo.BlockList.clear();
1234 collectGroupsToRelease(Region, BlockSize, AllocatedUserEnd,
1241 // Note that we have extracted the `GroupsToRelease` from region freelist.
1242 // It's safe to let pushBlocks()/popBlocks() access the remaining region
1252 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1255 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1262 RegionReleaseRecorder<MemMapT> Recorder(&Region->MemMapInfo.MemMap,
1263 Region->RegionBeg,
1268 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
1269 Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
1270 Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
1272 Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast();
1277 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1282 bool hasChanceToReleasePages(RegionInfo *Region, uptr BlockSize,
1284 REQUIRES(Region->MMLock, Region->FLLock) {
1285 DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
1286 Region->FreeListInfo.PushedBlocks);
1291 // following is the region usage,
1306 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) {
1307 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
1311 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
1319 if (RegionPushedBytesDelta < Region->TryReleaseThreshold)
1334 if (Region->ReleaseInfo.LastReleaseAtNs +
1347 collectGroupsToRelease(RegionInfo *Region, const uptr BlockSize,
1349 REQUIRES(Region->MMLock, Region->FLLock) {
1355 // release threshold as the next Region::TryReleaseThreshold(). Note that if
1361 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(),
1379 DCHECK_LE(Region->RegionBeg, BatchGroupBase);
1381 DCHECK_EQ((Region->RegionBeg - BatchGroupBase) % GroupSize, 0U);
1493 Region->FreeListInfo.BlockList.extract(Prev, Cur);
1495 Region->FreeListInfo.BlockList.pop_front();
1506 Region->TryReleaseThreshold = MinDistToThreshold;
1513 markFreeBlocks(RegionInfo *Region, const uptr BlockSize,
1516 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1532 roundUpSlow(LastGroupEnd - Region->RegionBeg, BlockSize) +
1533 Region->RegionBeg;
1535 const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg;
1555 (BatchGroupUsedEnd - Region->RegionBeg) % BlockSize == 0;
1573 Region->RegionBeg, /*RegionIndex=*/0,
1574 Region->MemMapInfo.AllocatedUser);
1581 BG.Batches, DecompactPtr, Region->RegionBeg, /*RegionIndex=*/0,
1582 Region->MemMapInfo.AllocatedUser, MayContainLastBlockInRegion);
1591 void mergeGroupsToReleaseBack(RegionInfo *Region,
1593 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1594 ScopedLock L(Region->FLLock);
1608 DCHECK_NE(BatchClassRegion, Region);
1610 // Merge GroupsToRelease back to the Region::FreeListInfo.BlockList. Note
1611 // that both `Region->FreeListInfo.BlockList` and `GroupsToRelease` are
1613 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(),
1618 Region->FreeListInfo.BlockList.append_back(&GroupsToRelease);
1702 Region->FreeListInfo.BlockList.push_front(Cur);
1704 Region->FreeListInfo.BlockList.insert(Prev, Cur);
1717 BatchGroupT *Prev = Region->FreeListInfo.BlockList.front();
1725 Region->FLLockCV.notifyAll(Region->FLLock);