Lines Matching full:region
33 // PrimaryEnableRandomOffset is set, each Region actually starts at a random
41 // The 1st Region (for size class 0) holds the TransferBatches. This is a
57 "Group size shouldn't be greater than the region size");
138 RegionInfo *Region = getRegionInfo(I);
140 initRegion(Region, I, RegionMemMap, Config::getEnableRandomOffset());
145 // The binding should be done after region shuffling so that it won't bind
146 // the FLLock from the wrong region.
158 RegionInfo *Region = getRegionInfo(I);
160 ScopedLock ML(Region->MMLock);
161 MemMapT MemMap = Region->MemMapInfo.MemMap;
165 *Region = {};
177 RegionInfo *Region = getRegionInfo(I);
178 ScopedLock ML(Region->MMLock);
179 ScopedLock FL(Region->FLLock);
182 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) {
189 DCHECK_EQ(TotalBlocks, Region->MemMapInfo.AllocatedUser / BlockSize);
190 DCHECK_EQ(Region->FreeListInfo.PushedBlocks,
191 Region->FreeListInfo.PoppedBlocks);
194 RegionInfo *Region = getRegionInfo(SizeClassMap::BatchClassId);
195 ScopedLock ML(Region->MMLock);
196 ScopedLock FL(Region->FLLock);
199 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) {
210 Region->MemMapInfo.AllocatedUser / BlockSize);
211 DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
212 Region->FreeListInfo.PushedBlocks);
214 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
221 RegionInfo *Region = getRegionInfo(ClassId);
225 ScopedLock L(Region->FLLock);
226 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
234 PopCount = popBlocksWithCV(C, ClassId, Region, ToArray, MaxBlockCount,
238 // When two threads compete for `Region->MMLock`, we only want one of
241 ScopedLock ML(Region->MMLock);
243 ScopedLock FL(Region->FLLock);
244 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
249 const bool RegionIsExhausted = Region->Exhausted;
251 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
254 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
277 RegionInfo *Region = getRegionInfo(ClassId);
279 ScopedLock L(Region->FLLock);
280 pushBatchClassBlocks(Region, Array, Size);
282 Region->FLLockCV.notifyAll(Region->FLLock);
307 ScopedLock L(Region->FLLock);
308 pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup);
310 Region->FLLockCV.notifyAll(Region->FLLock);
341 RegionInfo *Region = getRegionInfo(I);
343 // SizeClassAllocator64. We may consider locking each region on demand
345 Region->FLLock.assertHeld();
346 Region->MMLock.assertHeld();
348 const uptr From = Region->RegionBeg;
349 const uptr To = From + Region->MemMapInfo.AllocatedUser;
356 // TODO(kostyak): get the RSS per region.
361 RegionInfo *Region = getRegionInfo(I);
363 ScopedLock L(Region->MMLock);
364 TotalMapped += Region->MemMapInfo.MappedUser;
367 ScopedLock L(Region->FLLock);
368 PoppedBlocks += Region->FreeListInfo.PoppedBlocks;
369 PushedBlocks += Region->FreeListInfo.PushedBlocks;
379 RegionInfo *Region = getRegionInfo(I);
380 ScopedLock L1(Region->MMLock);
381 ScopedLock L2(Region->FLLock);
382 getStats(Str, I, Region);
392 RegionInfo *Region = getRegionInfo(I);
393 ScopedLock L(Region->MMLock);
394 getRegionFragmentationInfo(Region, I, Str);
404 RegionInfo *Region = getRegionInfo(I);
405 ScopedLock L(Region->MMLock);
406 getMemoryGroupFragmentationInfoInRegion(Region, I, Str);
423 RegionInfo *Region = getRegionInfo(ClassId);
427 if (Region->MMLock.tryLock()) {
428 uptr BytesReleased = releaseToOSMaybe(Region, ClassId, ReleaseType);
429 Region->MMLock.unlock();
440 RegionInfo *Region = getRegionInfo(I);
441 ScopedLock L(Region->MMLock);
442 TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType);
584 RegionInfo *Region = getRegionInfo(ClassId);
585 Region->MMLock.assertHeld();
588 !Region->MemMapInfo.MemMap.isAllocated()) {
591 return Region->MemMapInfo.MemMap.getBase();
620 ALWAYS_INLINE void initRegion(RegionInfo *Region, uptr ClassId,
622 REQUIRES(Region->MMLock) {
623 DCHECK(!Region->MemMapInfo.MemMap.isAllocated());
628 Region->MemMapInfo.MemMap = MemMap;
630 Region->RegionBeg = MemMap.getBase();
632 Region->RegionBeg +=
633 (getRandomModN(&Region->RandState, 16) + 1) * PageSize;
640 Region->ReleaseInfo.TryReleaseThreshold =
643 Region->ReleaseInfo.TryReleaseThreshold =
648 void pushBatchClassBlocks(RegionInfo *Region, CompactPtrT *Array, u32 Size)
649 REQUIRES(Region->FLLock) {
650 DCHECK_EQ(Region, getRegionInfo(SizeClassMap::BatchClassId));
687 Region->FreeListInfo.PushedBlocks += Size;
688 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
703 Region->FreeListInfo.BlockList.push_front(BG);
767 void pushBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
769 REQUIRES(Region->FLLock) {
812 Region->FreeListInfo.PushedBlocks += Size;
813 BatchGroupT *Cur = Region->FreeListInfo.BlockList.front();
829 Region->FreeListInfo.BlockList.push_front(Cur);
831 Region->FreeListInfo.BlockList.insert(Prev, Cur);
862 Region->FreeListInfo.BlockList.insert(Prev, Cur);
874 u16 popBlocksWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
886 ScopedLock FL(Region->FLLock);
887 if (!Region->isPopulatingFreeList) {
888 Region->isPopulatingFreeList = true;
894 ScopedLock ML(Region->MMLock);
896 const bool RegionIsExhausted = Region->Exhausted;
898 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
901 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
907 // `Region->isPopulatingFreeList` to false so the threads about to
909 ScopedLock FL(Region->FLLock);
910 Region->isPopulatingFreeList = false;
911 Region->FLLockCV.notifyAll(Region->FLLock);
919 // 2. Region->isPopulatingFreeList == true, i.e, someone is still doing
923 // Region->isPopulatingFreeList == false because all the new populated
926 ScopedLock FL(Region->FLLock);
927 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
931 if (!Region->isPopulatingFreeList)
939 Region->FLLockCV.wait(Region->FLLock);
941 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
949 u16 popBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
951 REQUIRES(Region->FLLock) {
952 if (Region->FreeListInfo.BlockList.empty())
956 Region->FreeListInfo.BlockList.front()->Batches;
960 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
961 Region->FreeListInfo.BlockList.pop_front();
968 Region->FreeListInfo.PoppedBlocks += 1;
1002 BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
1003 Region->FreeListInfo.BlockList.pop_front();
1015 Region->FreeListInfo.PoppedBlocks += PopCount;
1021 RegionInfo *Region,
1024 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1026 !Region->MemMapInfo.MemMap.isAllocated()) {
1035 initRegion(Region, ClassId,
1041 DCHECK(Region->MemMapInfo.MemMap.isAllocated());
1044 const uptr RegionBeg = Region->RegionBeg;
1045 const uptr MappedUser = Region->MemMapInfo.MappedUser;
1047 Region->MemMapInfo.AllocatedUser + MaxCount * Size;
1055 Region->Exhausted = true;
1059 if (UNLIKELY(!Region->MemMapInfo.MemMap.remap(
1066 Region->MemMapInfo.MappedUser += MapSize;
1072 static_cast<u32>((Region->MemMapInfo.MappedUser -
1073 Region->MemMapInfo.AllocatedUser) /
1083 uptr P = RegionBeg + Region->MemMapInfo.AllocatedUser;
1087 ScopedLock L(Region->FLLock);
1094 shuffle(ShuffleArray + I - N, N, &Region->RandState);
1095 pushBlocksImpl(C, ClassId, Region, ShuffleArray + I - N, N,
1104 shuffle(ShuffleArray + NumberOfBlocks - N, N, &Region->RandState);
1105 pushBlocksImpl(C, ClassId, Region, &ShuffleArray[NumberOfBlocks - N], N,
1108 pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks);
1112 popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
1119 Region->FreeListInfo.PushedBlocks -= NumberOfBlocks;
1123 Region->MemMapInfo.AllocatedUser += AllocatedUser;
1128 void getStats(ScopedString *Str, uptr ClassId, RegionInfo *Region)
1129 REQUIRES(Region->MMLock, Region->FLLock) {
1130 if (Region->MemMapInfo.MappedUser == 0)
1134 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
1136 Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize;
1139 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) {
1141 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
1143 const uptr TotalChunks = Region->MemMapInfo.AllocatedUser / BlockSize;
1146 "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx "
1148 Region->Exhausted ? "E" : " ", ClassId,
1149 getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10,
1150 Region->FreeListInfo.PoppedBlocks,
1151 Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks,
1152 Region->ReleaseInfo.NumReleasesAttempted,
1153 Region->ReleaseInfo.LastReleasedBytes >> 10,
1154 RegionPushedBytesDelta >> 10, Region->RegionBeg,
1158 void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId,
1159 ScopedString *Str) REQUIRES(Region->MMLock) {
1162 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1166 ScopedLock L(Region->FLLock);
1167 GroupsToRelease = Region->FreeListInfo.BlockList;
1168 Region->FreeListInfo.BlockList.clear();
1174 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1179 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1182 ScopedLock L(Region->FLLock);
1184 const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize;
1186 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
1188 roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize;
1204 void getMemoryGroupFragmentationInfoInRegion(RegionInfo *Region, uptr ClassId,
1206 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1209 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1213 ScopedLock L(Region->FLLock);
1214 GroupsToRelease = Region->FreeListInfo.BlockList;
1215 Region->FreeListInfo.BlockList.clear();
1224 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1229 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1232 Str->append("MemoryGroupFragmentationInfo in Region %zu (%zu)\n", ClassId,
1236 roundUp(Region->MemMapInfo.AllocatedUser, GroupSize) / GroupSize;
1244 Region->RegionBeg + I * GroupSize, Integral, Fractional);
1248 NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
1250 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1254 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1259 ScopedLock L(Region->FLLock);
1261 BytesInFreeList = Region->MemMapInfo.AllocatedUser -
1262 (Region->FreeListInfo.PoppedBlocks -
1263 Region->FreeListInfo.PushedBlocks) *
1273 !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList,
1282 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
1289 GroupsToRelease = Region->FreeListInfo.BlockList;
1290 Region->FreeListInfo.BlockList.clear();
1293 collectGroupsToRelease(Region, BlockSize, AllocatedUserEnd,
1302 ++Region->ReleaseInfo.NumReleasesAttempted;
1304 // Note that we have extracted the `GroupsToRelease` from region freelist.
1305 // It's safe to let pushBlocks()/popBlocks() access the remaining region
1315 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1318 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1325 RegionReleaseRecorder<MemMapT> Recorder(&Region->MemMapInfo.MemMap,
1326 Region->RegionBeg,
1339 if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold &&
1341 Region->ReleaseInfo.LastReleasedBytes +
1343 Region->ReleaseInfo.TryReleaseThreshold =
1344 Max(Region->ReleaseInfo.TryReleaseThreshold / 2,
1349 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
1350 Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
1352 Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast();
1354 if (Region->ReleaseInfo.PendingPushedBytesDelta > 0) {
1359 Region->ReleaseInfo.TryReleaseThreshold +=
1360 Region->ReleaseInfo.PendingPushedBytesDelta / 2;
1364 Region->ReleaseInfo.TryReleaseThreshold = Min<uptr>(
1365 Region->ReleaseInfo.TryReleaseThreshold, (1UL << GroupSizeLog) / 2);
1366 Region->ReleaseInfo.PendingPushedBytesDelta = 0;
1372 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1377 bool hasChanceToReleasePages(RegionInfo *Region, uptr BlockSize,
1379 REQUIRES(Region->MMLock, Region->FLLock) {
1380 DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
1381 Region->FreeListInfo.PushedBlocks);
1384 // following is the region usage,
1399 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) {
1400 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
1404 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
1407 if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold / 2)
1417 CurTimeNs - Region->ReleaseInfo.LastReleaseAtNs;
1423 if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold) {
1439 Region->ReleaseInfo.PendingPushedBytesDelta = RegionPushedBytesDelta;
1450 collectGroupsToRelease(RegionInfo *Region, const uptr BlockSize,
1452 REQUIRES(Region->MMLock, Region->FLLock) {
1464 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(),
1482 DCHECK_LE(Region->RegionBeg, BatchGroupBase);
1484 DCHECK_EQ((Region->RegionBeg - BatchGroupBase) % GroupSize, 0U);
1601 Region->FreeListInfo.BlockList.extract(Prev, Cur);
1603 Region->FreeListInfo.BlockList.pop_front();
1614 Region->ReleaseInfo.TryReleaseThreshold = MinDistToThreshold;
1621 markFreeBlocks(RegionInfo *Region, const uptr BlockSize,
1624 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1640 roundUpSlow(LastGroupEnd - Region->RegionBeg, BlockSize) +
1641 Region->RegionBeg;
1643 const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg;
1663 (BatchGroupUsedEnd - Region->RegionBeg) % BlockSize == 0;
1681 Region->RegionBeg, /*RegionIndex=*/0,
1682 Region->MemMapInfo.AllocatedUser);
1689 BG.Batches, DecompactPtr, Region->RegionBeg, /*RegionIndex=*/0,
1690 Region->MemMapInfo.AllocatedUser, MayContainLastBlockInRegion);
1699 void mergeGroupsToReleaseBack(RegionInfo *Region,
1701 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1702 ScopedLock L(Region->FLLock);
1716 DCHECK_NE(BatchClassRegion, Region);
1718 // Merge GroupsToRelease back to the Region::FreeListInfo.BlockList. Note
1719 // that both `Region->FreeListInfo.BlockList` and `GroupsToRelease` are
1721 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(),
1726 Region->FreeListInfo.BlockList.append_back(&GroupsToRelease);
1809 Region->FreeListInfo.BlockList.push_front(Cur);
1811 Region->FreeListInfo.BlockList.insert(Prev, Cur);
1824 BatchGroupT *Prev = Region->FreeListInfo.BlockList.front();
1832 Region->FLLockCV.notifyAll(Region->FLLock);