Avoid races in capacity accounting.

runtime/vm/pages.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/pages.h"
 
 #include "platform/assert.h"
 #include "vm/compiler_stats.h"
 #include "vm/gc_marker.h"
 #include "vm/gc_sweeper.h"
@@ skipping 182 matching lines @@
       if (FLAG_write_protect_code) {
         exec_pages_tail_->WriteProtect(false);
       }
       exec_pages_tail_->set_next(page);
       if (FLAG_write_protect_code) {
         exec_pages_tail_->WriteProtect(true);
       }
     }
     exec_pages_tail_ = page;
   }
-  usage_.capacity_in_words += kPageSizeInWords;
+  IncreaseCapacityInWordsLocked(kPageSizeInWords);
   page->set_object_end(page->memory_->end());
   return page;
 }
 
 
 HeapPage* PageSpace::AllocateLargePage(intptr_t size, HeapPage::PageType type) {
   intptr_t page_size_in_words = LargePageSizeInWordsFor(size);
   HeapPage* page = HeapPage::Allocate(page_size_in_words, type);
   page->set_next(large_pages_);
   large_pages_ = page;
-  usage_.capacity_in_words += page_size_in_words;
+  IncreaseCapacityInWords(page_size_in_words);
   // Only one object in this page (at least until String::MakeExternal or
   // Array::MakeArray is called).
   page->set_object_end(page->object_start() + size);
   return page;
 }
 
 
 void PageSpace::TruncateLargePage(HeapPage* page,
                                   intptr_t new_object_size_in_bytes) {
   const intptr_t old_object_size_in_bytes =
       page->object_end() - page->object_start();
   ASSERT(new_object_size_in_bytes <= old_object_size_in_bytes);
   const intptr_t new_page_size_in_words =
       LargePageSizeInWordsFor(new_object_size_in_bytes);
   VirtualMemory* memory = page->memory_;
   const intptr_t old_page_size_in_words = (memory->size() >> kWordSizeLog2);
   if (new_page_size_in_words < old_page_size_in_words) {
     memory->Truncate(new_page_size_in_words << kWordSizeLog2);
-    usage_.capacity_in_words -= old_page_size_in_words;
-    usage_.capacity_in_words += new_page_size_in_words;
+    IncreaseCapacityInWords(new_page_size_in_words - old_page_size_in_words);
     page->set_object_end(page->object_start() + new_object_size_in_bytes);
   }
 }
 
 
 void PageSpace::FreePage(HeapPage* page, HeapPage* previous_page) {
   bool is_exec = (page->type() == HeapPage::kExecutable);
   {
     MutexLocker ml(pages_lock_);
-    usage_.capacity_in_words -= (page->memory_->size() >> kWordSizeLog2);
+    IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2));
     if (!is_exec) {
       // Remove the page from the list of data pages.
       if (previous_page != NULL) {
         previous_page->set_next(page->next());
       } else {
         pages_ = page->next();
       }
       if (page == pages_tail_) {
         pages_tail_ = previous_page;
       }
     } else {
       // Remove the page from the list of executable pages.
       if (previous_page != NULL) {
         previous_page->set_next(page->next());
       } else {
         exec_pages_ = page->next();
       }
       if (page == exec_pages_tail_) {
         exec_pages_tail_ = previous_page;
       }
     }
   }
   // TODO(iposva): Consider adding to a pool of empty pages.
   page->Deallocate();
 }
 
 
 void PageSpace::FreeLargePage(HeapPage* page, HeapPage* previous_page) {
-  usage_.capacity_in_words -= (page->memory_->size() >> kWordSizeLog2);
+  IncreaseCapacityInWords(-(page->memory_->size() >> kWordSizeLog2));
   // Remove the page from the list.
   if (previous_page != NULL) {
     previous_page->set_next(page->next());
   } else {
     large_pages_ = page->next();
   }
   page->Deallocate();
 }
 
 
 void PageSpace::FreePages(HeapPage* pages) {
   HeapPage* page = pages;
   while (page != NULL) {
     HeapPage* next = page->next();
     page->Deallocate();
     page = next;
   }
 }
 
 
 uword PageSpace::TryAllocateInFreshPage(intptr_t size,
                                         HeapPage::PageType type,
                                         GrowthPolicy growth_policy,
                                         bool is_locked) {
   ASSERT(size < kAllocatablePageSize);
   uword result = 0;
-  SpaceUsage after_allocation = usage_;
+  SpaceUsage after_allocation = GetCurrentUsage();
   after_allocation.used_in_words += size >> kWordSizeLog2;
   // Can we grow by one page?
   after_allocation.capacity_in_words += kPageSizeInWords;
   if ((growth_policy == kForceGrowth ||
        !page_space_controller_.NeedsGarbageCollection(after_allocation)) &&
       CanIncreaseCapacityInWords(kPageSizeInWords)) {
     HeapPage* page = AllocatePage(type);
     ASSERT(page != NULL);
     // Start of the newly allocated page is the allocated object.
     result = page->object_start();
-    usage_ = after_allocation;
+    // Note: usage_.capacity_in_words is increased by AllocatePage.
+    usage_.used_in_words += size >> kWordSizeLog2;
     // Enqueue the remainder in the free list.
     uword free_start = result + size;
     intptr_t free_size = page->object_end() - free_start;
     if (free_size > 0) {
       if (is_locked) {
         freelist_[type].FreeLocked(free_start, free_size);
       } else {
         freelist_[type].Free(free_start, free_size);
       }
     }
   }
   return result;
 }
 
 
 uword PageSpace::TryAllocateInternal(intptr_t size,
                             HeapPage::PageType type,
                             GrowthPolicy growth_policy,
                             bool is_protected,
                             bool is_locked) {
   ASSERT(size >= kObjectAlignment);
   ASSERT(Utils::IsAligned(size, kObjectAlignment));
 #ifdef DEBUG
-  SpaceUsage usage_before = usage_;
+  SpaceUsage usage_before = GetCurrentUsage();
 #endif
   uword result = 0;
   if (size < kAllocatablePageSize) {
     if (is_locked) {
       result = freelist_[type].TryAllocateLocked(size, is_protected);
     } else {
       result = freelist_[type].TryAllocate(size, is_protected);
     }
     if (result == 0) {
       result = TryAllocateInFreshPage(size, type, growth_policy, is_locked);
       // usage_ is updated by the call above.
     } else {
       usage_.used_in_words += size >> kWordSizeLog2;
     }
   } else {
     // Large page allocation.
     intptr_t page_size_in_words = LargePageSizeInWordsFor(size);
     if ((page_size_in_words << kWordSizeLog2) < size) {
       // On overflow we fail to allocate.
       return 0;
     }
-    SpaceUsage after_allocation = usage_;
+    SpaceUsage after_allocation = GetCurrentUsage();
     after_allocation.used_in_words += size >> kWordSizeLog2;
     after_allocation.capacity_in_words += page_size_in_words;
     if ((growth_policy == kForceGrowth ||
          !page_space_controller_.NeedsGarbageCollection(after_allocation)) &&
         CanIncreaseCapacityInWords(page_size_in_words)) {
       HeapPage* page = AllocateLargePage(size, type);
       if (page != NULL) {
         result = page->object_start();
-        usage_ = after_allocation;
+        // Note: usage_.capacity_in_words is increased by AllocateLargePage.
+        usage_.used_in_words += size >> kWordSizeLog2;
       }
     }
   }
   if (result != 0) {
 #ifdef DEBUG
     // A successful allocation should increase usage_.
     ASSERT(usage_before.used_in_words < usage_.used_in_words);
 #endif
     if (FLAG_compiler_stats && (type == HeapPage::kExecutable)) {
       CompilerStats::code_allocated += size;
     }
   } else {
 #ifdef DEBUG
-    // A failed allocation should not change usage_.
+    // A failed allocation should not change used_in_words.
     ASSERT(usage_before.used_in_words == usage_.used_in_words);
-    ASSERT(usage_before.capacity_in_words == usage_.capacity_in_words);
 #endif
   }
   ASSERT((result & kObjectAlignmentMask) == kOldObjectAlignmentOffset);
   return result;
 }
 
 
   void PageSpace::AcquireDataLock() {
     freelist_[HeapPage::kData].mutex()->Lock();
   }
@@ skipping 269 matching lines @@
     heap_->Verify();
     OS::PrintErr(" done.\n");
   }
 
   const int64_t start = OS::GetCurrentTimeMicros();
 
   // Make code pages writable.
   WriteProtectCode(false);
 
   // Save old value before GCMarker visits the weak persistent handles.
-  SpaceUsage usage_before = usage_;
+  SpaceUsage usage_before = GetCurrentUsage();
 
   // Mark all reachable old-gen objects.
   bool collect_code = FLAG_collect_code && ShouldCollectCode();
   GCMarker marker(heap_);
   marker.MarkObjects(isolate, this, invoke_api_callbacks, collect_code);
   usage_.used_in_words = marker.marked_words();
 
   int64_t mid1 = OS::GetCurrentTimeMicros();
 
   // Abandon the remainder of the bump allocation block.
@@ skipping 80 matching lines @@
           isolate, pages_, pages_tail_, &freelist_[HeapPage::kData]);
     }
   }
 
   // Make code pages read-only.
   WriteProtectCode(true);
 
   int64_t end = OS::GetCurrentTimeMicros();
 
   // Record signals for growth control. Include size of external allocations.
-  page_space_controller_.EvaluateGarbageCollection(usage_before, usage_,
+  page_space_controller_.EvaluateGarbageCollection(usage_before,
+                                                   GetCurrentUsage(),
                                                    start, end);
 
   heap_->RecordTime(kMarkObjects, mid1 - start);
   heap_->RecordTime(kResetFreeLists, mid2 - mid1);
   heap_->RecordTime(kSweepPages, mid3 - mid2);
   heap_->RecordTime(kSweepLargePages, end - mid3);
 
   if (FLAG_print_free_list_after_gc) {
     OS::Print("Data Freelist (after GC):\n");
     freelist_[HeapPage::kData].Print();
@@ skipping 215 matching lines @@
     return 0;
   } else {
     ASSERT(total_time >= gc_time);
     int result= static_cast<int>((static_cast<double>(gc_time) /
                              static_cast<double>(total_time)) * 100);
     return result;
   }
 }
 
 }  // namespace dart