Track thread activities in order to diagnose hangs.

base/debug/activity_tracker.cc

Index: base/debug/activity_tracker.cc
diff --git a/base/debug/activity_tracker.cc b/base/debug/activity_tracker.cc
new file mode 100644
index 0000000000000000000000000000000000000000..5fc8e02888ed6367ed596aa977dd2a1f434b489e
--- /dev/null
+++ b/base/debug/activity_tracker.cc
@@ -0,0 +1,535 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/debug/activity_tracker.h"
+
+#include <atomic>
+
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/files/memory_mapped_file.h"
+#include "base/logging.h"
+#include "base/memory/ptr_util.h"
+#include "base/metrics/field_trial.h"
+#include "base/pending_task.h"
+#include "base/stl_util.h"
+#include "base/strings/string_util.h"
+
+namespace base {
+namespace debug {
+
+namespace {
+
+// A number that identifies the memory as having been initialized. It's
+// arbitrary but happens to be the first 8 bytes of SHA1(ThreadActivityTracker).
+// A version number is added on so that major structure changes won't try to
+// read an older version (since the cookie won't match).
+const uint64_t kHeaderCookie = 0xC0029B240D4A3092ULL + 1;  // v1
+
+// The minimum depth a stack should support.
+const int kMinStackDepth = 2;
+
+}  // namespace
+
+
+#if !defined(OS_NACL)  // NACL doesn't support any kind of file access in build.
+void SetupGlobalActivityTrackerFieldTrial(const FilePath& file) {
+  const Feature kActivityTrackerFeature{
+      "ActivityTracking", FEATURE_DISABLED_BY_DEFAULT
+  };
+
+  if (!base::FeatureList::IsEnabled(kActivityTrackerFeature))
+    return;
+
+  // TODO(bcwhite): Adjust these numbers once there is real data to show
+  // just how much of an arena is necessary.
+  const size_t kMemorySize = 1 << 20;  // 1 MiB
+  const int kStackDepth = 3;
+  const uint64_t kAllocatorId = 0;
+  const char kAllocatorName[] = "ActivityTracker";
+
+  GlobalActivityTracker::CreateWithFile(
+      file.AddExtension(PersistentMemoryAllocator::kFileExtension),
+      kMemorySize, kAllocatorId, kAllocatorName, kStackDepth);
+}
+#endif  // !defined(OS_NACL)
+
+// This information is kept for every thread that is tracked. It is filled
+// the very first time the thread is seen. All fields must be of exact sizes
+// so there is no issue moving between 32 and 64-bit builds.
+struct ThreadActivityTracker::Header {
+  // This unique number indicates a valid initialization of the memory.
+  uint64_t cookie;
+
+  // The thread-id to which this data belongs. This identifier is not
+  // guaranteed to mean anything, just to be unique among all active
+  // trackers.
+  uint64_t thread_id;
+
+  // The start-time and start-ticks when the data was created. Each activity
+  // record has a |time_ticks| value that can be converted to a "wall time"
+  // with these two values.
+  int64_t start_time;
+  int64_t start_ticks;
+
+  // The number of Activity slots in the data.
+  uint32_t slots;
+
+  // The current depth of the stack. This may be greater than the number of
+  // slots. If the depth exceeds the number of slots, the newest entries
+  // won't be recorded.
+  std::atomic<uint32_t> depth;
+
+  // A memory location used to indicate if changes have been made to the stack
+  // that would invalidate an in-progress read of its contents. The active
+  // tracker will zero the value whenever something gets popped from the
+  // stack. A monitoring tracker can write a non-zero value here, copy the
+  // stack contents, and read the value to know, if it is still non-zero, that
+  // the contents didn't change while being copied.
+  std::atomic<uint32_t> unchanged;
+
+  // The name of the thread (up to a maximum length). Dynamic-length names
+  // are not practical since the memory has to come from the same persistent
+  // allocator that holds this structure and to which this object has no
+  // reference.
+  char name[32];
+};
+
+ThreadActivityTracker::ThreadActivityTracker(void* base, size_t size)
+    : header_(static_cast<Header*>(base)),
+      stack_(reinterpret_cast<StackEntry*>(reinterpret_cast<char*>(base) +
+                                           sizeof(Header))),
+      stack_slots_((size - sizeof(Header)) / sizeof(StackEntry)) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(base);
+
+  // Ensure there is enough space for the header and at least a few records.
+  DCHECK_LE(sizeof(Header) + kMinStackDepth * sizeof(StackEntry), size);
+
+  // Ensure that the |stack_slots_| calculation didn't overflow.
+  DCHECK_GE(std::numeric_limits<uint32_t>::max(),
+            (size - sizeof(Header)) / sizeof(StackEntry));
+
+  // Provided memory should either be completely initialized or all zeros.
+  if (header_->cookie == 0) {
+    // This is a new file. Double-check other fields and then initialize.
+    DCHECK_EQ(0U, header_->thread_id);
+    DCHECK_EQ(0, header_->start_time);
+    DCHECK_EQ(0, header_->start_ticks);
+    DCHECK_EQ(0U, header_->slots);
+    DCHECK_EQ(0U, header_->depth.load(std::memory_order_relaxed));
+    DCHECK_EQ(0U, header_->unchanged.load(std::memory_order_relaxed));
+    DCHECK_EQ(0, stack_[0].time_ticks);
+    DCHECK_EQ(0U, stack_[0].source_address);
+    DCHECK_EQ(0U, stack_[0].data.task.sequence_id);
+
+    header_->cookie = kHeaderCookie;
+    header_->thread_id = static_cast<uint64_t>(PlatformThread::CurrentId());
+    header_->start_time = base::Time::Now().ToInternalValue();
+    header_->start_ticks = base::TimeTicks::Now().ToInternalValue();
+    header_->slots = stack_slots_;
+    strlcpy(header_->name, PlatformThread::GetName(), sizeof(header_->name));
+    valid_ = true;
+  } else {
+    // This is a file with existing data. Perform basic consistency checks.
+    if (header_->cookie != kHeaderCookie ||
+        header_->slots != stack_slots_ ||
+        header_->start_time > base::Time::Now().ToInternalValue())
+      return;
+    valid_ = true;
+  }
+}
+
+ThreadActivityTracker::~ThreadActivityTracker() {}
+
+void ThreadActivityTracker::PushActivity(const void* source,
+                                         ActivityType activity,
+                                         const StackEntryData& data) {
+  // A thread-checker creates a lock to check the thread-id which means
+  // re-entry into this code when locks are being tracked.
+  DCHECK(activity == ACT_LOCK || thread_checker_.CalledOnValidThread());
+
+  // Get the current depth of the stack. No access to other memory guarded
+  // by this variable is done here so a "relaxed" load is acceptable.
+  uint32_t depth = header_->depth.load(std::memory_order_relaxed);
+
+  // Handle the case where the stack depth has exceeded the storage capacity.
+  // Extra entries will be lost leaving only the base of the stack.
+  if (depth >= stack_slots_) {
+    // Since no other memory is being modified, a "relaxed" store is acceptable.
+    header_->depth.store(depth + 1, std::memory_order_relaxed);
+    return;
+  }
+
+  // Get a pointer to the next entry and load it. No atomicity is required
+  // here because the memory is known only to this thread. It will be made
+  // known to other threads once the depth is incremented.
+  StackEntry* entry = &stack_[depth];
+  entry->time_ticks = base::TimeTicks::Now().ToInternalValue();
+  entry->source_address = reinterpret_cast<uintptr_t>(source);
+  entry->activity_type = activity;
+  entry->data = data;
+
+  // Save the incremented depth. Because this guards |entry| memory filled
+  // above that may be read by another thread once the recorded depth changes,
+  // a "release" store is required.
+  header_->depth.store(depth + 1, std::memory_order_release);
+}
+
+void ThreadActivityTracker::PopActivity(const void* source) {
+  // Do an atomic decrement of the depth. No changes to stack entries guarded
+  // by this variable is done here so a "relaxed" operation is acceptable.
+  // |depth| will receive the value before it was modified.
+  uint32_t depth = header_->depth.fetch_sub(1, std::memory_order_relaxed);
+
+  // Validate that everything is running correctly.
+  DCHECK_LT(0U, depth);
+  if (depth <= stack_slots_) {
+    DCHECK_EQ(reinterpret_cast<uintptr_t>(source),
+              stack_[depth - 1].source_address);
+    DCHECK(stack_[depth - 1].activity_type == ACT_LOCK ||
+           thread_checker_.CalledOnValidThread());
+  }
+
+  // The stack has shrunk meaning that some other thread trying to copy the
+  // contents for reporting purposes could get bad data. That thread would
+  // have written a non-zero value into |unchanged|; clearing it here will
+  // let that thread detect that something did change. It doesn't matter
+  // when this is done relative to the atomic |depth| operation above so a
+  // "relaxed" access is acceptable.
+  header_->unchanged.store(0, std::memory_order_relaxed);
+}
+
+// static
+size_t ThreadActivityTracker::SizeForStackDepth(int stack_depth) {
+  return static_cast<size_t>(stack_depth) * sizeof(StackEntry) + sizeof(Header);
+}
+
+ThreadActivityAnalyzer::ThreadActivityAnalyzer(ThreadActivityTracker* tracker)
+    : ThreadActivityAnalyzer(
+          tracker->header_,
+          ThreadActivityTracker::SizeForStackDepth(tracker->stack_slots_)) {}
+
+ThreadActivityAnalyzer::ThreadActivityAnalyzer(
+    PersistentMemoryAllocator* allocator,
+    PersistentMemoryAllocator::Reference reference)
+    : ThreadActivityAnalyzer(allocator->GetAsObject<char>(
+                                 reference,
+                                 GlobalActivityTracker::kTypeIdActivityTracker),
+                             allocator->GetAllocSize(reference)) {}
+
+ThreadActivityAnalyzer::ThreadActivityAnalyzer(void* base, size_t size)
+    : tracker_(base, size) {}
+
+ThreadActivityAnalyzer::~ThreadActivityAnalyzer() {}
+
+uint32_t ThreadActivityAnalyzer::SnapshotStack(
+    std::vector<StackEntry>* snapshot) {
+  // It's possible for the data to change while reading it in such a way that it
+  // invalidates the read. Make several attempts but don't try forever.
+  const int kMaxAttempts = 10;
+  uint32_t depth;
+
+  // Start with an empty return stack.
+  snapshot->clear();
+
+  // Stop here if the data isn't valid.
+  if (!tracker_.is_valid())
+    return 0;
+
+  ThreadActivityTracker::Header* header = tracker_.header_;
+  for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
+    // Write a non-zero value to |unchanged| so it's possible to detect at
+    // the end that nothing has changed since copying the data began.
+    header->unchanged.store(1, std::memory_order_relaxed);
+
+    // Fetching the current depth also "acquires" the contents of the stack.
+    depth = header->depth.load(std::memory_order_acquire);
+    if (depth == 0)
+      return 0;
+
+    // Copy the existing contents. Memcpy is used for speed.
+    uint32_t count = std::min(depth, tracker_.stack_slots_);
+    snapshot->resize(count);
+    memcpy(&(*snapshot)[0], tracker_.stack_, count * sizeof(StackEntry));
+
+    // Check to make sure everything was unchanged during the copy.
+    if (header->unchanged.load(std::memory_order_relaxed))
+      return depth;
+  }
+
+  // If all attempts failed, just return the depth with no content.
+  snapshot->clear();
+  return depth;
+}
+
+
+GlobalActivityTracker* GlobalActivityTracker::g_tracker_ = nullptr;
+
+GlobalActivityTracker::ManagedActivityTracker::ManagedActivityTracker(
+    PersistentMemoryAllocator::Reference mem_reference,
+    void* base,
+    size_t size)
+    : ThreadActivityTracker(base, size),
+      mem_reference_(mem_reference),
+      mem_base_(base) {}
+
+GlobalActivityTracker::ManagedActivityTracker::~ManagedActivityTracker() {
+  // The global |g_tracker_| must point to the owner of this class since all
+  // objects of this type must be destructed before |g_tracker_| can be changed
+  // (something that only occurs in tests).
+  DCHECK(g_tracker_);
+  g_tracker_->ReturnTrackerMemory(this, mem_reference_, mem_base_);
+}
+
+void GlobalActivityTracker::CreateWithAllocator(
+    std::unique_ptr<PersistentMemoryAllocator> allocator,
+    int stack_depth) {
+  // There's no need to do anything with the result. It is self-managing.
+  new GlobalActivityTracker(std::move(allocator), stack_depth);
+}
+
+// static
+void GlobalActivityTracker::CreateWithLocalMemory(size_t size,
+                                                  uint64_t id,
+                                                  StringPiece name,
+                                                  int stack_depth) {
+  CreateWithAllocator(
+      WrapUnique(new LocalPersistentMemoryAllocator(size, id, name)),
+      stack_depth);
+}
+
+#if !defined(OS_NACL)
+// static
+void GlobalActivityTracker::CreateWithFile(const FilePath& file_path,
+                                           size_t size,
+                                           uint64_t id,
+                                           StringPiece name,
+                                           int stack_depth) {
+  DCHECK(!file_path.empty());
+
+  // Create the file, overwriting anything that was there previously, and set
+  // the length. This will create a space that is zero-filled, a requirement
+  // for operation.
+  File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_READ |
+                       File::FLAG_WRITE | File::FLAG_SHARE_DELETE);
+  DCHECK(file.IsValid());
+  file.SetLength(size);
+
+  // Map the file into memory and make it globally available.
+  std::unique_ptr<MemoryMappedFile> mapped_file(new MemoryMappedFile());
+  bool success =
+      mapped_file->Initialize(std::move(file), MemoryMappedFile::READ_WRITE);
+  DCHECK(success);
+  CreateWithAllocator(WrapUnique(new FilePersistentMemoryAllocator(
+                          std::move(mapped_file), size, id, name, false)),
+                      stack_depth);
+}
+#endif  // !defined(OS_NACL)
+
+ThreadActivityTracker* GlobalActivityTracker::CreateTrackerForCurrentThread() {
+  DCHECK(!this_thread_tracker_.Get());
+
+  PersistentMemoryAllocator::Reference mem_reference = 0;
+  void* mem_base = nullptr;
+
+  // Get the current count of available memories, acquiring the array values.
+  int count = available_memories_count_.load(std::memory_order_acquire);
+  while (count > 0) {
+    // There is a memory block that was previously released (and zero'd) so
+    // just re-use that rather than allocating a new one.
+    mem_reference =
+        available_memories_[count - 1].load(std::memory_order_relaxed);
+    DCHECK(mem_reference);
+
+    // Decrement the count indicating that the value has been taken. If this
+    // fails then something else, another thread doing push or pop, has changed
+    // the stack; retry if so. |count| will receive the existing value.
+    if (!available_memories_count_.compare_exchange_weak(
+            count, count - 1,
+            std::memory_order_acquire, std::memory_order_acquire)) {
+      continue;
+    }
+
+    // Clear the value just read from the array so that the "push" operation
+    // knows there is no value there and will work correctly.
+    available_memories_[count - 1].store(0, std::memory_order_relaxed);
+
+    // Turn the reference back into one of the activity-tracker type.
+    mem_base = allocator_->GetAsObject<char>(mem_reference,
+                                             kTypeIdActivityTrackerFree);
+    DCHECK(mem_base);
+    DCHECK_LE(stack_memory_size_, allocator_->GetAllocSize(mem_reference));
+    allocator_->SetType(mem_reference, kTypeIdActivityTracker);
+
+    // Success.
+    break;
+  }
+
+  if (count == 0) {
+    // Allocate a block of memory from the persistent segment.
+    mem_reference =
+        allocator_->Allocate(stack_memory_size_, kTypeIdActivityTracker);
+    if (mem_reference) {
+      // Success. Convert the reference to an actual memory address.
+      mem_base =
+          allocator_->GetAsObject<char>(mem_reference, kTypeIdActivityTracker);
+      // Make the allocation iterable so it can be found by other processes.
+      allocator_->MakeIterable(mem_reference);
+    } else {
+      // Failure. This should never happen.

[manzagop (departed), 2016/06/01 21:59:40]

It can if we didn't plan for enough memory, right?

[bcwhite, 2016/06/02 16:18:16]

Right.  I'll add to the comment.

+      NOTREACHED();
+      // But if it does, handle it gracefully by allocating the required
+      // memory from the heap.
+      mem_base = new char[stack_memory_size_];
+      memset(mem_base, 0, stack_memory_size_);
+    }
+  }
+
+  // Create a tracker with the acquired memory and set it as the tracker
+  // for this particular thread in thread-local-storage.
+  DCHECK(mem_base);
+  ManagedActivityTracker* tracker =
+      new ManagedActivityTracker(mem_reference, mem_base, stack_memory_size_);
+  DCHECK(tracker->is_valid());
+  this_thread_tracker_.Set(tracker);
+  thread_tracker_count_.fetch_add(1, std::memory_order_relaxed);
+
+  return tracker;
+}
+
+void GlobalActivityTracker::ReleaseTrackerForCurrentThreadForTesting() {
+  ThreadActivityTracker* tracker =
+      reinterpret_cast<ThreadActivityTracker*>(this_thread_tracker_.Get());
+  if (tracker) {
+    this_thread_tracker_.Free();
+    delete tracker;
+  }
+}
+
+GlobalActivityTracker::GlobalActivityTracker(
+    std::unique_ptr<PersistentMemoryAllocator> allocator,
+    int stack_depth)
+    : allocator_(std::move(allocator)),
+      stack_memory_size_(ThreadActivityTracker::SizeForStackDepth(stack_depth)),
+      this_thread_tracker_(&OnTLSDestroy),
+      thread_tracker_count_(0),
+      available_memories_count_(0) {
+  // Clear the available-memories array.
+  memset(available_memories_, 0, sizeof(available_memories_));
+
+  // Ensure the passed memory is valid and empty (iterator finds nothing).
+  uint32_t type;
+  DCHECK(!PersistentMemoryAllocator::Iterator(allocator_.get()).GetNext(&type));
+
+  // Ensure that there is no other global object and then make this one such.
+  DCHECK(!g_tracker_);
+  g_tracker_ = this;
+
+  // Create a tracker for this thread since it is known.
+  CreateTrackerForCurrentThread();
+}
+
+GlobalActivityTracker::~GlobalActivityTracker() {
+  DCHECK_EQ(g_tracker_, this);
+  DCHECK_EQ(0, thread_tracker_count_.load(std::memory_order_relaxed));
+  g_tracker_ = nullptr;
+}
+
+void GlobalActivityTracker::ReturnTrackerMemory(
+    ManagedActivityTracker* tracker,
+    PersistentMemoryAllocator::Reference mem_reference,
+    void* mem_base) {
+  // Zero the memory so that it is ready for use if needed again later. It's
+  // better to clear the memory now, when a thread is exiting, than to do it
+  // when it is first needed by a thread doing actual work.
+  memset(mem_base, 0, stack_memory_size_);
+
+  // Remove the destructed tracker from the set of known ones.
+  DCHECK_LE(1, thread_tracker_count_.load(std::memory_order_relaxed));
+  thread_tracker_count_.fetch_sub(1, std::memory_order_relaxed);
+
+  // Deal with the memory that was used by the tracker.
+  if (mem_reference) {
+    // The memory was within the persistent memory allocator. Change its type
+    // so that iteration won't find it.
+    allocator_->SetType(mem_reference, kTypeIdActivityTrackerFree);
+    // There is no way to free memory from a persistent allocator so instead
+    // push it on the internal list of available memory blocks.
+    while (true) {
+      // Get the existing count of available memories and ensure we won't
+      // burst the array. Acquire the values in the array.
+      int count = available_memories_count_.load(std::memory_order_acquire);
+      if (count >= kMaxThreadCount) {
+        NOTREACHED();
+        // Storage is full. Just forget about this memory. It won't be re-used
+        // but there's no real loss.
+        break;
+      }
+
+      // Write the reference of the memory being returned to this slot in the
+      // array. Empty slots have a value of zero so do an atomic compare-and-
+      // exchange to ensure that a race condition doesn't exist with another
+      // thread doing the same.
+      PersistentMemoryAllocator::Reference mem_expected = 0;
+      if (!available_memories_[count].compare_exchange_weak(
+              mem_expected, mem_reference,
+              std::memory_order_release, std::memory_order_relaxed)) {
+        continue;  // Try again.
+      }
+
+      // Increment the count, releasing the value written to the array. This
+      // could fail if a simultaneous "pop" operation decremented the counter.
+      // If that happens, clear the array slot and start over. Do a "strong"
+      // exchange to avoid spurious retries that can occur with a "weak" one.
+      int expected = count;  // Updated by compare/exchange.
+      if (!available_memories_count_.compare_exchange_strong(
+              expected, count + 1,
+              std::memory_order_release, std::memory_order_relaxed)) {
+        available_memories_[count].store(0, std::memory_order_relaxed);
+        continue;
+      }
+
+      // Count was successfully incremented to refrect the new value added.

[manzagop (departed), 2016/06/01 21:59:41]

typo: reflect

[bcwhite, 2016/06/02 16:18:16]

Done.

+      break;
+    }
+  } else {
+    // The memory was allocated from the process heap. This shouldn't happen
+    // because the persistent memory segment should be big enough for all
+    // thread stacks but it's better to support falling back to allocation
+    // from the heap rather than crash. Everything will work as normal but
+    // the data won't be persisted.
+    delete[] reinterpret_cast<char*>(mem_base);
+  }
+}
+
+// static
+void GlobalActivityTracker::OnTLSDestroy(void* value) {
+  delete reinterpret_cast<ManagedActivityTracker*>(value);
+}
+
+
+ScopedTaskActivity::ScopedTaskActivity(const base::PendingTask& task)
+    : GlobalActivityTracker::ScopedThreadActivity(
+          task.posted_from.program_counter(),
+          ThreadActivityTracker::ACT_TASK,
+          ThreadActivityTracker::StackEntryData::ForTask(task.sequence_num),
+          /*lock_allowed=*/true) {}
+
+ScopedLockActivity::ScopedLockActivity(const base::internal::LockImpl* lock)
+    : GlobalActivityTracker::ScopedThreadActivity(
+          nullptr,
+          ThreadActivityTracker::ACT_LOCK,
+          ThreadActivityTracker::StackEntryData::ForLock(lock),
+          /*lock_allowed=*/false) {}
+
+ScopedEventActivity::ScopedEventActivity(const base::WaitableEvent* event)
+    : GlobalActivityTracker::ScopedThreadActivity(
+          nullptr,
+          ThreadActivityTracker::ACT_EVENT,
+          ThreadActivityTracker::StackEntryData::ForEvent(event),
+          /*lock_allowed=*/true) {}
+
+}  // namespace debug
+}  // namespace base