[heap] Fix computation of flags offset in memorychunk

src/heap/mark-compact.cc

Index: src/heap/mark-compact.cc
diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc
index e537689c4a5ca4ba35530fc8d81a55f3bfd37596..a123cad65ce0b305f48f9e4bf820ae6250744ca0 100644
--- a/src/heap/mark-compact.cc
+++ b/src/heap/mark-compact.cc
@@ -61,9 +61,8 @@ MarkCompactCollector::MarkCompactCollector(Heap* heap)
       code_flusher_(nullptr),
       have_code_to_deoptimize_(false),
       compacting_(false),
-      sweeping_in_progress_(false),
-      pending_sweeper_tasks_semaphore_(0),
-      pending_compaction_tasks_semaphore_(0) {
+      pending_compaction_tasks_semaphore_(0),
+      sweeper_(heap) {
 }
 
 #ifdef VERIFY_HEAP
@@ -459,11 +458,10 @@ void MarkCompactCollector::ClearMarkbits() {
   }
 }
 
-
-class MarkCompactCollector::SweeperTask : public v8::Task {
+class MarkCompactCollector::Sweeper::SweeperTask : public v8::Task {
  public:
-  SweeperTask(Heap* heap, AllocationSpace space_to_start)
-      : heap_(heap), space_to_start_(space_to_start) {}
+  SweeperTask(Sweeper* sweeper, Heap* heap, AllocationSpace space_to_start)
+      : sweeper_(sweeper), heap_(heap), space_to_start_(space_to_start) {}
 
   virtual ~SweeperTask() {}
 
@@ -478,33 +476,59 @@ class MarkCompactCollector::SweeperTask : public v8::Task {
       const int space_id = FIRST_PAGED_SPACE + ((i + offset) % num_spaces);
       DCHECK_GE(space_id, FIRST_PAGED_SPACE);
       DCHECK_LE(space_id, LAST_PAGED_SPACE);
-      heap_->mark_compact_collector()->SweepInParallel(
-          heap_->paged_space(space_id), 0);
+      sweeper_->HelpSweepInParallel(heap_->paged_space(space_id), 0);
+    }
+    for (int i = 0; i < Sweeper::kNumberOfSweepingSpaces; i++) {
+      SweepingList* late_list = nullptr;
+      SweepingSpace space = static_cast<SweepingSpace>(i);
+      {
+        base::LockGuard<base::Mutex> guard(&(sweeper_->mutex_));
+        late_list = sweeper_->late_sweeping_list_[space];
+      }
+      if (late_list != nullptr) {
+        for (Page* p : *late_list) {
+          sweeper_->HelpSweepInParallel(
+              p, heap_->paged_space(sweeper_->allocation_space(space)));
+        }
+      }
     }
-    heap_->mark_compact_collector()->pending_sweeper_tasks_semaphore_.Signal();
+    sweeper_->pending_sweeper_tasks_semaphore_.Signal();
   }
 
+  Sweeper* sweeper_;
   Heap* heap_;
   AllocationSpace space_to_start_;
 
   DISALLOW_COPY_AND_ASSIGN(SweeperTask);
 };
 
-
-void MarkCompactCollector::StartSweeperThreads() {
-  V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new SweeperTask(heap(), OLD_SPACE), v8::Platform::kShortRunningTask);
-  V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new SweeperTask(heap(), CODE_SPACE), v8::Platform::kShortRunningTask);
-  V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new SweeperTask(heap(), MAP_SPACE), v8::Platform::kShortRunningTask);
+void MarkCompactCollector::Sweeper::StartSweeping() {
+  sweeping_in_progress_ = true;
+  std::sort(sweeping_list_[kOldSpace].begin(), sweeping_list_[kOldSpace].end(),
+            [](Page* a, Page* b) { return a->LiveBytes() < b->LiveBytes(); });
+  std::sort(sweeping_list_[kCodeSpace].begin(),
+            sweeping_list_[kCodeSpace].end(),
+            [](Page* a, Page* b) { return a->LiveBytes() < b->LiveBytes(); });
+  std::sort(sweeping_list_[kMapSpace].begin(), sweeping_list_[kMapSpace].end(),
+            [](Page* a, Page* b) { return a->LiveBytes() < b->LiveBytes(); });
+  if (FLAG_concurrent_sweeping) {
+    V8::GetCurrentPlatform()->CallOnBackgroundThread(
+        new SweeperTask(this, heap_, OLD_SPACE),
+        v8::Platform::kShortRunningTask);
+    V8::GetCurrentPlatform()->CallOnBackgroundThread(
+        new SweeperTask(this, heap_, CODE_SPACE),
+        v8::Platform::kShortRunningTask);
+    V8::GetCurrentPlatform()->CallOnBackgroundThread(
+        new SweeperTask(this, heap_, MAP_SPACE),
+        v8::Platform::kShortRunningTask);
+  }
 }
 
-
-void MarkCompactCollector::SweepOrWaitUntilSweepingCompleted(Page* page) {
+void MarkCompactCollector::Sweeper::SweepOrWaitUntilSweepingCompleted(
+    Page* page) {
   PagedSpace* owner = reinterpret_cast<PagedSpace*>(page->owner());
   if (!page->SweepingDone()) {
-    SweepInParallel(page, owner);
+    HelpSweepInParallel(page, owner);
     if (!page->SweepingDone()) {
       // We were not able to sweep that page, i.e., a concurrent
       // sweeper thread currently owns this page. Wait for the sweeper
@@ -514,24 +538,31 @@ void MarkCompactCollector::SweepOrWaitUntilSweepingCompleted(Page* page) {
   }
 }
 
-
 void MarkCompactCollector::SweepAndRefill(CompactionSpace* space) {
-  if (FLAG_concurrent_sweeping && !IsSweepingCompleted()) {
-    SweepInParallel(heap()->paged_space(space->identity()), 0);
+  if (FLAG_concurrent_sweeping && !sweeper().IsSweepingCompleted()) {
+    sweeper().HelpSweepInParallel(heap()->paged_space(space->identity()), 0);
     space->RefillFreeList();
   }
 }
 
+Page* MarkCompactCollector::Sweeper::GetSweptPage(PagedSpace* space) {
+  base::LockGuard<base::Mutex> guard(&mutex_);
+  SweptList& list = swept_list_[sweeping_space(space->identity())];
+  if (list.length() > 0) {
+    return list.RemoveLast();
+  }
+  return nullptr;
+}
 
-void MarkCompactCollector::EnsureSweepingCompleted() {
+void MarkCompactCollector::Sweeper::EnsureCompleted() {
   DCHECK(sweeping_in_progress_ == true);
 
   // If sweeping is not completed or not running at all, we try to complete it
   // here.
   if (!FLAG_concurrent_sweeping || !IsSweepingCompleted()) {
-    SweepInParallel(heap()->paged_space(OLD_SPACE), 0);
-    SweepInParallel(heap()->paged_space(CODE_SPACE), 0);
-    SweepInParallel(heap()->paged_space(MAP_SPACE), 0);
+    HelpSweepInParallel(heap_->paged_space(OLD_SPACE), 0);
+    HelpSweepInParallel(heap_->paged_space(CODE_SPACE), 0);
+    HelpSweepInParallel(heap_->paged_space(MAP_SPACE), 0);
   }
 
   if (FLAG_concurrent_sweeping) {
@@ -542,6 +573,12 @@ void MarkCompactCollector::EnsureSweepingCompleted() {
 
   ParallelSweepSpacesComplete();
   sweeping_in_progress_ = false;
+}
+
+void MarkCompactCollector::EnsureSweepingCompleted() {
+  DCHECK(sweeper().sweeping_in_progress() == true);
+
+  sweeper().EnsureCompleted();
   heap()->old_space()->RefillFreeList();
   heap()->code_space()->RefillFreeList();
   heap()->map_space()->RefillFreeList();
@@ -553,8 +590,7 @@ void MarkCompactCollector::EnsureSweepingCompleted() {
 #endif
 }
 
-
-bool MarkCompactCollector::IsSweepingCompleted() {
+bool MarkCompactCollector::Sweeper::IsSweepingCompleted() {
   if (!pending_sweeper_tasks_semaphore_.WaitFor(
           base::TimeDelta::FromSeconds(0))) {
     return false;
@@ -563,7 +599,6 @@ bool MarkCompactCollector::IsSweepingCompleted() {
   return true;
 }
 
-
 void Marking::TransferMark(Heap* heap, Address old_start, Address new_start) {
   // This is only used when resizing an object.
   DCHECK(MemoryChunk::FromAddress(old_start) ==
@@ -3169,21 +3204,12 @@ class EvacuationWeakObjectRetainer : public WeakObjectRetainer {
   }
 };
 
-enum SweepingMode { SWEEP_ONLY, SWEEP_AND_VISIT_LIVE_OBJECTS };
-
-enum SkipListRebuildingMode { REBUILD_SKIP_LIST, IGNORE_SKIP_LIST };
-
-enum FreeSpaceTreatmentMode { IGNORE_FREE_SPACE, ZAP_FREE_SPACE };
-
-// Sweeps a page. After sweeping the page can be iterated.
-// Slots in live objects pointing into evacuation candidates are updated
-// if requested.
-// Returns the size of the biggest continuous freed memory chunk in bytes.
-template <SweepingMode sweeping_mode,
-          MarkCompactCollector::SweepingParallelism parallelism,
-          SkipListRebuildingMode skip_list_mode,
-          FreeSpaceTreatmentMode free_space_mode>
-static int Sweep(PagedSpace* space, Page* p, ObjectVisitor* v) {
+template <MarkCompactCollector::Sweeper::SweepingMode sweeping_mode,
+          MarkCompactCollector::Sweeper::SweepingParallelism parallelism,
+          MarkCompactCollector::Sweeper::SkipListRebuildingMode skip_list_mode,
+          MarkCompactCollector::Sweeper::FreeSpaceTreatmentMode free_space_mode>
+int MarkCompactCollector::Sweeper::UnmanagedSweep(PagedSpace* space, Page* p,
+                                                  ObjectVisitor* v) {
   DCHECK(!p->IsEvacuationCandidate() && !p->SweepingDone());
   DCHECK(!p->IsFlagSet(Page::BLACK_PAGE));
   DCHECK_EQ(skip_list_mode == REBUILD_SKIP_LIST,
@@ -3252,7 +3278,6 @@ static int Sweep(PagedSpace* space, Page* p, ObjectVisitor* v) {
   return FreeList::GuaranteedAllocatable(static_cast<int>(max_freed_bytes));
 }
 
-
 void MarkCompactCollector::InvalidateCode(Code* code) {
   if (heap_->incremental_marking()->IsCompacting() &&
       !ShouldSkipEvacuationSlotRecording(code)) {
@@ -3349,6 +3374,11 @@ void MarkCompactCollector::VisitLiveObjectsBody(Page* page,
   }
 }
 
+void MarkCompactCollector::Sweeper::AddSweptPageSafe(PagedSpace* space,
+                                                     Page* page) {
+  base::LockGuard<base::Mutex> guard(&mutex_);
+  swept_list_[sweeping_space(space->identity())].Add(page);
+}
 
 void MarkCompactCollector::SweepAbortedPages() {
   // Second pass on aborted pages.
@@ -3359,26 +3389,29 @@ void MarkCompactCollector::SweepAbortedPages() {
       PagedSpace* space = static_cast<PagedSpace*>(p->owner());
       switch (space->identity()) {
         case OLD_SPACE:
-          Sweep<SWEEP_ONLY, SWEEP_ON_MAIN_THREAD, IGNORE_SKIP_LIST,
-                IGNORE_FREE_SPACE>(space, p, nullptr);
+          Sweeper::UnmanagedSweep<
+              Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
+              Sweeper::IGNORE_SKIP_LIST, Sweeper::IGNORE_FREE_SPACE>(space, p,
+                                                                     nullptr);
           break;
         case CODE_SPACE:
           if (FLAG_zap_code_space) {
-            Sweep<SWEEP_ONLY, SWEEP_ON_MAIN_THREAD, REBUILD_SKIP_LIST,
-                  ZAP_FREE_SPACE>(space, p, nullptr);
+            Sweeper::UnmanagedSweep<
+                Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
+                Sweeper::REBUILD_SKIP_LIST, Sweeper::ZAP_FREE_SPACE>(space, p,
+                                                                     nullptr);
           } else {
-            Sweep<SWEEP_ONLY, SWEEP_ON_MAIN_THREAD, REBUILD_SKIP_LIST,
-                  IGNORE_FREE_SPACE>(space, p, nullptr);
+            Sweeper::UnmanagedSweep<
+                Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
+                Sweeper::REBUILD_SKIP_LIST, Sweeper::IGNORE_FREE_SPACE>(
+                space, p, nullptr);
           }
           break;
         default:
           UNREACHABLE();
           break;
       }
-      {
-        base::LockGuard<base::Mutex> guard(&swept_pages_mutex_);
-        swept_pages(space->identity())->Add(p);
-      }
+      sweeper().AddSweptPageSafe(space, p);
     }
   }
 }
@@ -3422,7 +3455,7 @@ void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
   }
 
 #ifdef VERIFY_HEAP
-  if (FLAG_verify_heap && !sweeping_in_progress_) {
+  if (FLAG_verify_heap && !sweeper().sweeping_in_progress()) {
     VerifyEvacuation(heap());
   }
 #endif
@@ -3609,15 +3642,14 @@ void MarkCompactCollector::ReleaseEvacuationCandidates() {
   heap()->FreeQueuedChunks();
 }
 
-
-int MarkCompactCollector::SweepInParallel(PagedSpace* space,
-                                          int required_freed_bytes,
-                                          int max_pages) {
+int MarkCompactCollector::Sweeper::HelpSweepInParallel(PagedSpace* space,
+                                                       int required_freed_bytes,
+                                                       int max_pages) {
   int max_freed = 0;
   int max_freed_overall = 0;
   int page_count = 0;
-  for (Page* p : sweeping_list(space)) {
-    max_freed = SweepInParallel(p, space);
+  for (Page* p : sweeping_list_[sweeping_space(space->identity())]) {
+    max_freed = HelpSweepInParallel(p, space);
     DCHECK(max_freed >= 0);
     if (required_freed_bytes > 0 && max_freed >= required_freed_bytes) {
       return max_freed;
@@ -3631,8 +3663,8 @@ int MarkCompactCollector::SweepInParallel(PagedSpace* space,
   return max_freed_overall;
 }
 
-
-int MarkCompactCollector::SweepInParallel(Page* page, PagedSpace* space) {
+int MarkCompactCollector::Sweeper::HelpSweepInParallel(Page* page,
+                                                       PagedSpace* space) {
   int max_freed = 0;
   if (page->mutex()->TryLock()) {
     // If this page was already swept in the meantime, we can return here.
@@ -3642,18 +3674,21 @@ int MarkCompactCollector::SweepInParallel(Page* page, PagedSpace* space) {
     }
     page->concurrent_sweeping_state().SetValue(Page::kSweepingInProgress);
     if (space->identity() == OLD_SPACE) {
-      max_freed = Sweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, IGNORE_SKIP_LIST,
-                        IGNORE_FREE_SPACE>(space, page, NULL);
+      max_freed =
+          UnmanagedSweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, IGNORE_SKIP_LIST,
+                         IGNORE_FREE_SPACE>(space, page, NULL);
     } else if (space->identity() == CODE_SPACE) {
-      max_freed = Sweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, REBUILD_SKIP_LIST,
-                        IGNORE_FREE_SPACE>(space, page, NULL);
+      max_freed =
+          UnmanagedSweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, REBUILD_SKIP_LIST,
+                         IGNORE_FREE_SPACE>(space, page, NULL);
     } else {
-      max_freed = Sweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, IGNORE_SKIP_LIST,
-                        IGNORE_FREE_SPACE>(space, page, NULL);
+      max_freed =
+          UnmanagedSweep<SWEEP_ONLY, SWEEP_IN_PARALLEL, IGNORE_SKIP_LIST,
+                         IGNORE_FREE_SPACE>(space, page, NULL);
     }
     {
-      base::LockGuard<base::Mutex> guard(&swept_pages_mutex_);
-      swept_pages(space->identity())->Add(page);
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      swept_list_[sweeping_space(space->identity())].Add(page);
     }
     page->concurrent_sweeping_state().SetValue(Page::kSweepingDone);
     page->mutex()->Unlock();
@@ -3661,6 +3696,36 @@ int MarkCompactCollector::SweepInParallel(Page* page, PagedSpace* space) {
   return max_freed;
 }
 
+void MarkCompactCollector::Sweeper::AddPage(PagedSpace* space, Page* page) {
+  DCHECK(!sweeping_in_progress_);
+  PrepareAddPage(sweeping_space(space->identity()), page);
+  sweeping_list_[sweeping_space(space->identity())].push_back(page);
+}
+
+void MarkCompactCollector::Sweeper::AddLatePage(SweepingSpace space,
+                                                Page* page) {
+  DCHECK(!sweeping_in_progress_);
+  PrepareAddPage(space, page);
+  if (tmp_late_sweeping_list_[space] == nullptr) {
+    tmp_late_sweeping_list_[space] = new SweepingList();
+  }
+  tmp_late_sweeping_list_[space]->push_back(page);
+}
+
+void MarkCompactCollector::Sweeper::PrepareAddPage(SweepingSpace space,
+                                                   Page* page) {
+  page->concurrent_sweeping_state().SetValue(Page::kSweepingPending);
+  int to_sweep = page->area_size() - page->LiveBytes();
+  heap_->paged_space(allocation_space(space))
+      ->accounting_stats_.ShrinkSpace(to_sweep);
+}
+
+void MarkCompactCollector::Sweeper::CommitLateList(SweepingSpace space) {
+  DCHECK(!sweeping_in_progress_);
+  base::LockGuard<base::Mutex> guard(&mutex_);
+  DCHECK_NULL(late_sweeping_list_[space]);
+  late_sweeping_list_[space] = tmp_late_sweeping_list_[space];
+}
 
 void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
   space->ClearStats();
@@ -3696,8 +3761,10 @@ void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
       // (in the free list) dropped again. Since we only use the flag for
       // testing this is fine.
       p->concurrent_sweeping_state().SetValue(Page::kSweepingInProgress);
-      Sweep<SWEEP_ONLY, SWEEP_ON_MAIN_THREAD, IGNORE_SKIP_LIST,
-            IGNORE_FREE_SPACE>(space, p, nullptr);
+      Sweeper::UnmanagedSweep<
+          Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
+          Sweeper::IGNORE_SKIP_LIST, Sweeper::IGNORE_FREE_SPACE>(space, p,
+                                                                 nullptr);
       continue;
     }
 
@@ -3713,10 +3780,7 @@ void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
       unused_page_present = true;
     }
 
-    p->concurrent_sweeping_state().SetValue(Page::kSweepingPending);
-    sweeping_list(space).push_back(p);
-    int to_sweep = p->area_size() - p->LiveBytes();
-    space->accounting_stats_.ShrinkSpace(to_sweep);
+    sweeper().AddPage(space, p);
     will_be_swept++;
   }
 
@@ -3724,8 +3788,6 @@ void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
     PrintIsolate(isolate(), "sweeping: space=%s initialized_for_sweeping=%d",
                  AllocationSpaceName(space->identity()), will_be_swept);
   }
-  std::sort(sweeping_list(space).begin(), sweeping_list(space).end(),
-            [](Page* a, Page* b) { return a->LiveBytes() < b->LiveBytes(); });
 }
 
 
@@ -3741,7 +3803,6 @@ void MarkCompactCollector::SweepSpaces() {
 #endif
 
   {
-    sweeping_in_progress_ = true;
     {
       GCTracer::Scope sweep_scope(heap()->tracer(),
                                   GCTracer::Scope::MC_SWEEP_OLD);
@@ -3757,9 +3818,7 @@ void MarkCompactCollector::SweepSpaces() {
                                   GCTracer::Scope::MC_SWEEP_MAP);
       StartSweepSpace(heap()->map_space());
     }
-    if (FLAG_concurrent_sweeping) {
-      StartSweeperThreads();
-    }
+    sweeper().StartSweeping();
   }
 
   // Deallocate unmarked large objects.
@@ -3771,11 +3830,15 @@ void MarkCompactCollector::SweepSpaces() {
   }
 }
 
-
-void MarkCompactCollector::ParallelSweepSpacesComplete() {
-  sweeping_list(heap()->old_space()).clear();
-  sweeping_list(heap()->code_space()).clear();
-  sweeping_list(heap()->map_space()).clear();
+void MarkCompactCollector::Sweeper::ParallelSweepSpacesComplete() {
+  for (int i = 0; i < kNumberOfSweepingSpaces; i++) {
+    SweepingSpace space = static_cast<SweepingSpace>(i);
+    DCHECK_NULL(tmp_late_sweeping_list_[space]);
+    sweeping_list_[space].clear();
+    if (late_sweeping_list_[space] != nullptr) {
+      delete late_sweeping_list_[space];
+    }
+  }
 }
 
 Isolate* MarkCompactCollector::isolate() const { return heap_->isolate(); }