Tracks the amount of committed vs uncommitted memory in tcmalloc's page heap's freelists

third_party/tcmalloc/page_heap.cc

 // Copyright (c) 2008, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 33 matching lines @@
               "to return memory slower.  Reasonable rates are in the "
               "range [0,10]");
 
 namespace tcmalloc {
 
 PageHeap::PageHeap()
     : pagemap_(MetaDataAlloc),
       pagemap_cache_(0),
       free_pages_(0),
       system_bytes_(0),
+      committed_bytes_(0),
       scavenge_counter_(0),
       // Start scavenging at kMaxPages list
       scavenge_index_(kMaxPages-1) {
   COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits);
   DLL_Init(&large_.normal);
   DLL_Init(&large_.returned);
   for (int i = 0; i < kMaxPages; i++) {
     DLL_Init(&free_[i].normal);
     DLL_Init(&free_[i].returned);
   }
@@ skipping 82 matching lines @@
   RecordSpan(leftover);
   pagemap_.set(span->start + n - 1, span); // Update map from pageid to span
   span->length = n;
 
   return leftover;
 }
 
 void PageHeap::CommitSpan(Span* span) {
   TCMalloc_SystemCommit(reinterpret_cast<void*>(span->start << kPageShift),
                         static_cast<size_t>(span->length << kPageShift));
+  committed_bytes_ += span->length << kPageShift;
 }
 
 void PageHeap::DecommitSpan(Span* span) {
   TCMalloc_SystemRelease(reinterpret_cast<void*>(span->start << kPageShift),
                          static_cast<size_t>(span->length << kPageShift));
+  committed_bytes_ -= span->length << kPageShift;
 }
 
 Span* PageHeap::Carve(Span* span, Length n) {
   ASSERT(n > 0);
   ASSERT(span->location != Span::IN_USE);
   const int old_location = span->location;
   DLL_Remove(span);
   span->location = Span::IN_USE;
   Event(span, 'A', n);
 
@@ skipping 55 matching lines @@
   // TODO(jar): We need a better strategy for deciding to commit, or decommit,
   // based on memory usage and free heap sizes.
 
   const PageID p = span->start;
   const Length n = span->length;
   Span* prev = GetDescriptor(p-1);
   if (prev != NULL && prev->location != Span::IN_USE) {
     // Merge preceding span into this span
     ASSERT(prev->start + prev->length == p);
     const Length len = prev->length;
+    if (prev->location == Span::ON_RETURNED_FREELIST) {
+      // We're about to put the merge span into the returned freelist and call
+      // DecommitSpan() on it, which will mark the entire span including this
+      // one as released and decrease committed_bytes_ by the size of the
+      // merged span.  To make the math work out we temporarily increase the
+      // committed_bytes_ amount.
+      committed_bytes_ += prev->length << kPageShift;
+    }
     DLL_Remove(prev);
     DeleteSpan(prev);
     span->start -= len;
     span->length += len;
     pagemap_.set(span->start, span);
     Event(span, 'L', len);
   }
   Span* next = GetDescriptor(p+n);
   if (next != NULL && next->location != Span::IN_USE) {
     // Merge next span into this span
     ASSERT(next->start == p+n);
     const Length len = next->length;
+    if (next->location == Span::ON_RETURNED_FREELIST) {
+      // See the comment below 'if (prev->location ...' for explanation.
+      committed_bytes_ += next->length << kPageShift;
+    }
     DLL_Remove(next);
     DeleteSpan(next);
     span->length += len;
     pagemap_.set(span->start + span->length - 1, span);
     Event(span, 'R', len);
   }
 
   Event(span, 'D', span->length);
   span->location = Span::ON_RETURNED_FREELIST;
   DecommitSpan(span);
@@ skipping 34 matching lines @@
   // Find index of free list to scavenge
   int index = scavenge_index_ + 1;
   for (int i = 0; i < kMaxPages+1; i++) {
     if (index > kMaxPages) index = 0;
     SpanList* slist = (index == kMaxPages) ? &large_ : &free_[index];
     if (!DLL_IsEmpty(&slist->normal)) {
       // Release the last span on the normal portion of this list
       Span* s = slist->normal.prev;
       ASSERT(s->location == Span::ON_NORMAL_FREELIST);
       DLL_Remove(s);
-      TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
-                             static_cast<size_t>(s->length << kPageShift));
+      DecommitSpan(s);
       s->location = Span::ON_RETURNED_FREELIST;
       DLL_Prepend(&slist->returned, s);
 
       // Compute how long to wait until we return memory.
       // FLAGS_tcmalloc_release_rate==1 means wait for 1000 pages
       // after releasing one page.
       const double mult = 1000.0 / rate;
       double wait = mult * static_cast<double>(s->length);
       if (wait > kMaxReleaseDelay) {
         // Avoid overflow and bound to reasonable range
@@ skipping 109 matching lines @@
       ask = n;
       ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
     }
     if (ptr == NULL) return false;
   }
   ask = actual_size >> kPageShift;
   RecordGrowth(ask << kPageShift);
 
   uint64_t old_system_bytes = system_bytes_;
   system_bytes_ += (ask << kPageShift);
+  committed_bytes_ += (ask << kPageShift);
   const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
   ASSERT(p > 0);
 
   // If we have already a lot of pages allocated, just pre allocate a bunch of
   // memory for the page map. This prevents fragmentation by pagemap metadata
   // when a program keeps allocating and freeing large blocks.
 
   if (old_system_bytes < kPageMapBigAllocationThreshold
       && system_bytes_ >= kPageMapBigAllocationThreshold) {
     pagemap_.PreallocateMoreMemory();
@@ skipping 41 matching lines @@
   for (Span* s = list->next; s != list; s = s->next) {
     CHECK_CONDITION(s->location == freelist);  // NORMAL or RETURNED
     CHECK_CONDITION(s->length >= min_pages);
     CHECK_CONDITION(s->length <= max_pages);
     CHECK_CONDITION(GetDescriptor(s->start) == s);
     CHECK_CONDITION(GetDescriptor(s->start+s->length-1) == s);
   }
   return true;
 }
 
-static void ReleaseFreeList(Span* list, Span* returned) {
+void PageHeap::ReleaseFreeList(Span* list, Span* returned) {
   // Walk backwards through list so that when we push these
   // spans on the "returned" list, we preserve the order.
   while (!DLL_IsEmpty(list)) {
     Span* s = list->prev;
     DLL_Remove(s);
     DLL_Prepend(returned, s);
     ASSERT(s->location == Span::ON_NORMAL_FREELIST);
     s->location = Span::ON_RETURNED_FREELIST;
-    TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
-                           static_cast<size_t>(s->length << kPageShift));
+    DecommitSpan(s);
   }
 }
 
 void PageHeap::ReleaseFreePages() {
   for (Length s = 0; s < kMaxPages; s++) {
     ReleaseFreeList(&free_[s].normal, &free_[s].returned);
   }
   ReleaseFreeList(&large_.normal, &large_.returned);
   ASSERT(Check());
 }
 
 }  // namespace tcmalloc