Added support for storing arbitrary user data.

base/debug/activity_tracker.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/debug/activity_tracker.h"
 
+#include <algorithm>
+
 #include "base/debug/stack_trace.h"
 #include "base/files/file.h"
 #include "base/files/file_path.h"
 #include "base/files/memory_mapped_file.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/pending_task.h"
 #include "base/process/process.h"
 #include "base/process/process_handle.h"
 #include "base/stl_util.h"
 #include "base/strings/string_util.h"
 #include "base/threading/platform_thread.h"
 
 namespace base {
 namespace debug {
 
 namespace {
 
 // A number that identifies the memory as having been initialized. It's
 // arbitrary but happens to be the first 4 bytes of SHA1(ThreadActivityTracker).
 // A version number is added on so that major structure changes won't try to
 // read an older version (since the cookie won't match).
 const uint32_t kHeaderCookie = 0xC0029B24UL + 2;  // v2
 
 // The minimum depth a stack should support.
 const int kMinStackDepth = 2;
 
+// The amount of memory set aside for holding arbitrary user data (key/value
+// pairs) globally or associated with ActivityData entries.
+const size_t kUserDataSize = 1024;    // bytes
+const size_t kGlobalDataSize = 1024;  // bytes
+
 union ThreadRef {
   int64_t as_id;
 #if defined(OS_WIN)
   // On Windows, the handle itself is often a pseudo-handle with a common
   // value meaning "this thread" and so the thread-id is used. The former
   // can be converted to a thread-id with a system call.
   PlatformThreadId as_tid;
 #elif defined(OS_POSIX)
   // On Posix, the handle is always a unique identifier so no conversion
   // needs to be done. However, it's value is officially opaque so there
@@ skipping 18 matching lines @@
   thread_ref.as_handle = handle.platform_handle();
 #endif
   return ForThread(thread_ref.as_id);
 }
 
 ActivityTrackerMemoryAllocator::ActivityTrackerMemoryAllocator(
     PersistentMemoryAllocator* allocator,
     uint32_t object_type,
     uint32_t object_free_type,
     size_t object_size,
-    size_t cache_size)
+    size_t cache_size,
+    bool make_iterable)
     : allocator_(allocator),
       object_type_(object_type),
       object_free_type_(object_free_type),
       object_size_(object_size),
       cache_size_(cache_size),
+      make_iterable_(make_iterable),
       iterator_(allocator),
       cache_values_(new Reference[cache_size]),
       cache_used_(0) {
   DCHECK(allocator);
 }
 
 ActivityTrackerMemoryAllocator::~ActivityTrackerMemoryAllocator() {}
 
 ActivityTrackerMemoryAllocator::Reference
 ActivityTrackerMemoryAllocator::GetObjectReference() {
@@ skipping 30 matching lines @@
       break;
     }
     if (!found) {
       // Reached end; start over at the beginning.
       iterator_.Reset();
     }
   }
 
   // No free block was found so instead allocate a new one.
   Reference allocated = allocator_->Allocate(object_size_, object_type_);
-  if (allocated)
+  if (allocated && make_iterable_)
     allocator_->MakeIterable(allocated);
   return allocated;
 }
 
 void ActivityTrackerMemoryAllocator::ReleaseObjectReference(Reference ref) {
   // Zero the memory so that it is ready for immediate use if needed later.
   char* mem_base = allocator_->GetAsObject<char>(ref, object_type_);
   DCHECK(mem_base);
   memset(mem_base, 0, object_size_);
 
@@ skipping 30 matching lines @@
   }
   activity->call_stack[i - 1] = 0;
 #else
   activity->call_stack[0] = 0;
 #endif
 }
 
 ActivitySnapshot::ActivitySnapshot() {}
 ActivitySnapshot::~ActivitySnapshot() {}
 
+ActivityUserData::ValueInfo::ValueInfo() {}
+ActivityUserData::ValueInfo::ValueInfo(ValueInfo&&) = default;
+ActivityUserData::ValueInfo::~ValueInfo() {}
+
+ActivityUserData::ActivityUserData(void* memory, size_t size)
+    : memory_(static_cast<char*>(memory)), available_(size) {}
+
+ActivityUserData::~ActivityUserData() {}
+
+void ActivityUserData::Set(StringPiece name,
+                           ValueType type,
+                           const void* memory,
+                           size_t size) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK_GE(std::numeric_limits<uint8_t>::max(), name.length());
+  size = std::min(std::numeric_limits<uint16_t>::max() - (kMemoryAlignment - 1),
+                  size);
+
+  // It's possible that no user data is being stored.
+  if (!memory_)
+    return;
+
+  ValueInfo* info;
+  auto existing = values_.find(name);
+  if (existing != values_.end()) {
+    info = &existing->second;
+  } else {
+    // The name size is limited to what can be held in a single byte but
+    // because there are not alignment constraints on strings, it's set tight
+    // against the header. It's extent (the reserved space, even if it's not

[manzagop (departed), 2016/11/11 17:01:42]

typo: It's -> Its

[bcwhite, 2016/11/14 15:08:42]

Actually, that is correct.  "It is not" => "It's not".

[manzagop (departed), 2016/11/14 19:51:17]

Ah, I meant the other one! ;) "It's extent (...) is calculated".

[bcwhite, 2016/11/15 13:37:47]

I knew that...  ;-)

+    // all used) is calculated so that, when pressed against the header, the
+    // following field will be aligned properly.
+    size_t name_size =
+        std::min(name.length(),
+                 static_cast<size_t>(std::numeric_limits<uint8_t>::max()));
+    size_t name_extent = ((sizeof(Header) + name_size + kMemoryAlignment - 1) &
+                          (0 - kMemoryAlignment)) -
+                         sizeof(Header);
+    size_t value_extent =
+        (size + kMemoryAlignment - 1) & (0 - kMemoryAlignment);

[manzagop (departed), 2016/11/11 17:01:42]

An alignment ceiling function might make this more readable since it's used
twice.

[bcwhite, 2016/11/14 15:08:42]

Done.

+
+    // The "basic size" is the minimum size of the record. It's possible that
+    // lengthy values will get truncated but there must be at least some bytes
+    // available.
+    size_t basic_size = sizeof(Header) + name_extent + kMemoryAlignment;
+    if (basic_size > available_)
+      return;  // No space to store even the smallest value.
+
+    // The "full size" is the size for storing the entire value, truncated
+    // to the amount of available memory.
+    size_t full_size =
+        std::min(sizeof(Header) + name_extent + value_extent, available_);
+    size = std::min(full_size - sizeof(Header) - name_extent, size);
+
+    // Allocate a chunk of memory.
+    Header* header = reinterpret_cast<Header*>(memory_);
+    memory_ += full_size;
+    available_ -= full_size;
+
+    // Datafill the header and name records.
+    DCHECK_EQ(EMPTY_VALUE, header->type);  // Memory must be zeroed.
+    header->type = type;
+    header->name_size = static_cast<uint8_t>(name_size);
+    header->value_size = size;
+    header->record_size = full_size;
+    char* name_memory = reinterpret_cast<char*>(header) + sizeof(Header);
+    void* value_memory =
+        reinterpret_cast<char*>(header) + sizeof(Header) + name_extent;
+    memcpy(name_memory, name.data(), name_size);

[manzagop (departed), 2016/11/11 17:01:42]

Does this need to be nul terminated?

[bcwhite, 2016/11/14 15:08:42]

No.  StringPiece doesn't require a NUL terminator.  In fact, it supports
embedded NULs.

[manzagop (departed), 2016/11/14 19:51:17]

I was thinking more in terms of how we read it back. FWIU the header gives the
size and we can implicitly derive the extent knowing the alignment is 64 bits.
So we're fine.

[bcwhite, 2016/11/15 13:37:47]

In most cases yes but if it was a string overwritten with a much shorter one
then that wouldn't be sufficient.

+
+    // Create an entry in |values_| so that this field can be found and changed
+    // later on without having to allocate new entries.
+    StringPiece persistent_name(name_memory, name_size);

[manzagop (departed), 2016/11/11 17:01:42]

I'm not sure how we should handle names that exceed the uint8_t limit. We DCHECK
but I'm not sure if that's safe enough if folks use dynamic names. Maybe we need
to handle it explicitely, which is what this line is doing: truncating the name
for addition to the map. However, l.213 looks for the full name in the map.

[bcwhite, 2016/11/14 15:08:42]

I'll have it pre-truncate before doing a look-up in the map.  If it turns out to
be an actual problem, I'll look at using a 2-byte length but I don't see much
point yet.  A user could always map

  "a" => "long dynamically generated name"
  "b" => "value for that name"

[manzagop (departed), 2016/11/14 19:51:17]

Sgtm!

+    auto inserted =
+        values_.insert(std::make_pair(persistent_name, ValueInfo()));
+    DCHECK(inserted.second);  // True if inserted, false if existed.
+    info = &inserted.first->second;
+    info->name = persistent_name;
+    info->memory = value_memory;
+    info->size = size;
+    info->extent = full_size - sizeof(Header) - name_extent;
+    info->type = type;
+  }
+
+  // Copy the value data to storage.
+  DCHECK_EQ(type, info->type);
+  size = std::min(size, info->extent);
+  memcpy(info->memory, memory, size);

[manzagop (departed), 2016/11/11 17:01:42]

Do we need to update the value's size, or is it always implicit from the type or
a delimiting character (e.g. \0)?

[bcwhite, 2016/11/14 15:08:42]

Done.

+  info->size = size;
+}
+
+void ActivityUserData::SetExternal(StringPiece name,
+                                   ValueType type,
+                                   const void* memory,
+                                   size_t size) {
+  ExternalRecord rec;
+  rec.address = reinterpret_cast<uintptr_t>(memory);
+  rec.size = size;
+  Set(name, static_cast<ValueType>(type - 1), &rec, sizeof(rec));

[manzagop (departed), 2016/11/11 17:01:42]

Why does the type get decremented (EXTERNAL_STRING_VALUE -> STRING_VALUE)? Can
you still tell it's an external record in there?

[bcwhite, 2016/11/14 15:08:42]

Done.

+}
 
 // This information is kept for every thread that is tracked. It is filled
 // the very first time the thread is seen. All fields must be of exact sizes
 // so there is no issue moving between 32 and 64-bit builds.
 struct ThreadActivityTracker::Header {
   // This unique number indicates a valid initialization of the memory.
   std::atomic<uint32_t> cookie;
   uint32_t reserved;  // pad out to 64 bits
 
   // The process-id and thread-id (thread_ref.as_id) to which this data belongs.
@@ skipping 34 matching lines @@
   // is not the current implementation so no parallel snapshots allowed).
   std::atomic<uint32_t> stack_unchanged;
 
   // The name of the thread (up to a maximum length). Dynamic-length names
   // are not practical since the memory has to come from the same persistent
   // allocator that holds this structure and to which this object has no
   // reference.
   char thread_name[32];
 };
 
+ThreadActivityTracker::ScopedActivity::ScopedActivity(
+    ThreadActivityTracker* tracker,
+    const void* origin,
+    Activity::Type type,
+    const ActivityData& data)
+    : tracker_(tracker) {
+  if (tracker_)
+    activity_id_ = tracker_->PushActivity(origin, type, data);
+}
+
+ThreadActivityTracker::ScopedActivity::~ScopedActivity() {
+  if (tracker_)
+    tracker_->PopActivity(activity_id_);
+}
+
+void ThreadActivityTracker::ScopedActivity::ChangeTypeAndData(
+    Activity::Type type,
+    const ActivityData& data) {
+  if (tracker_)
+    tracker_->ChangeActivity(activity_id_, type, data);
+}
+
+ActivityUserData& ThreadActivityTracker::ScopedActivity::user_data() {
+  if (!user_data_) {
+    if (tracker_)
+      user_data_ = tracker_->GetUserData(activity_id_);
+    else
+      user_data_ = MakeUnique<ActivityUserData>(nullptr, 0);
+  }
+  return *user_data_;
+}
+
 ThreadActivityTracker::ThreadActivityTracker(void* base, size_t size)
     : header_(static_cast<Header*>(base)),
       stack_(reinterpret_cast<Activity*>(reinterpret_cast<char*>(base) +
                                          sizeof(Header))),
       stack_slots_(
           static_cast<uint32_t>((size - sizeof(Header)) / sizeof(Activity))) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // Verify the parameters but fail gracefully if they're not valid so that
   // production code based on external inputs will not crash.  IsValid() will
@@ skipping 57 matching lines @@
     DCHECK(IsValid());
   } else {
     // This is a file with existing data. Perform basic consistency checks.
     valid_ = true;
     valid_ = IsValid();
   }
 }
 
 ThreadActivityTracker::~ThreadActivityTracker() {}
 
-void ThreadActivityTracker::PushActivity(const void* origin,
-                                         Activity::Type type,
-                                         const ActivityData& data) {
+ThreadActivityTracker::ActivityId ThreadActivityTracker::PushActivity(
+    const void* origin,
+    Activity::Type type,
+    const ActivityData& data) {
   // A thread-checker creates a lock to check the thread-id which means
   // re-entry into this code if lock acquisitions are being tracked.
   DCHECK(type == Activity::ACT_LOCK_ACQUIRE ||
          thread_checker_.CalledOnValidThread());
 
   // Get the current depth of the stack. No access to other memory guarded
   // by this variable is done here so a "relaxed" load is acceptable.
   uint32_t depth = header_->current_depth.load(std::memory_order_relaxed);
 
   // Handle the case where the stack depth has exceeded the storage capacity.
   // Extra entries will be lost leaving only the base of the stack.
   if (depth >= stack_slots_) {
     // Since no other threads modify the data, no compare/exchange is needed.
     // Since no other memory is being modified, a "relaxed" store is acceptable.
     header_->current_depth.store(depth + 1, std::memory_order_relaxed);
-    return;
+    return depth;
   }
 
   // Get a pointer to the next activity and load it. No atomicity is required
   // here because the memory is known only to this thread. It will be made
   // known to other threads once the depth is incremented.
   Activity::FillFrom(&stack_[depth], origin, type, data);
 
   // Save the incremented depth. Because this guards |activity| memory filled
   // above that may be read by another thread once the recorded depth changes,
   // a "release" store is required.
   header_->current_depth.store(depth + 1, std::memory_order_release);
+
+  // The current depth is used as the activity ID because it simply identifies
+  // an entry. Once an entry is pop'd, it's okay to reuse the ID.
+  return depth;
 }
 
-void ThreadActivityTracker::ChangeActivity(Activity::Type type,
+void ThreadActivityTracker::ChangeActivity(ActivityId id,
+                                           Activity::Type type,
                                            const ActivityData& data) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(type != Activity::ACT_NULL || &data != &kNullActivityData);
-
-  // Get the current depth of the stack and acquire the data held there.
-  uint32_t depth = header_->current_depth.load(std::memory_order_acquire);
-  DCHECK_LT(0U, depth);
+  DCHECK_LT(id, header_->current_depth.load(std::memory_order_acquire));
 
   // Update the information if it is being recorded (i.e. within slot limit).
-  if (depth <= stack_slots_) {
-    Activity* activity = &stack_[depth - 1];
+  if (id < stack_slots_) {
+    Activity* activity = &stack_[id];
 
     if (type != Activity::ACT_NULL) {
       DCHECK_EQ(activity->activity_type & Activity::ACT_CATEGORY_MASK,
                 type & Activity::ACT_CATEGORY_MASK);
       activity->activity_type = type;
     }
 
     if (&data != &kNullActivityData)
       activity->data = data;
   }
 }
 
-void ThreadActivityTracker::PopActivity() {
+void ThreadActivityTracker::PopActivity(ActivityId id) {
   // Do an atomic decrement of the depth. No changes to stack entries guarded
   // by this variable are done here so a "relaxed" operation is acceptable.
-  // |depth| will receive the value BEFORE it was modified.
+  // |depth| will receive the value BEFORE it was modified which means the
+  // return value must also be decremented. The slot will be "free" after
+  // this call but since only a single thread can access this object, the
+  // data will remain valid until this method returns or calls outside.
   uint32_t depth =
-      header_->current_depth.fetch_sub(1, std::memory_order_relaxed);
+      header_->current_depth.fetch_sub(1, std::memory_order_relaxed) - 1;
 
   // Validate that everything is running correctly.
-  DCHECK_LT(0U, depth);
+  DCHECK_EQ(id, depth);
 
   // A thread-checker creates a lock to check the thread-id which means
   // re-entry into this code if lock acquisitions are being tracked.
-  DCHECK(stack_[depth - 1].activity_type == Activity::ACT_LOCK_ACQUIRE ||
+  DCHECK(stack_[depth].activity_type == Activity::ACT_LOCK_ACQUIRE ||
          thread_checker_.CalledOnValidThread());
 
+  // Check if there was any user-data memory. It isn't free'd until later
+  // because the call to release it can push something on the stack.
+  PersistentMemoryAllocator::Reference user_data = stack_[depth].user_data;
+  stack_[depth].user_data = 0;
+
   // The stack has shrunk meaning that some other thread trying to copy the
   // contents for reporting purposes could get bad data. That thread would
   // have written a non-zero value into |stack_unchanged|; clearing it here
   // will let that thread detect that something did change. This needs to
   // happen after the atomic |depth| operation above so a "release" store
   // is required.
   header_->stack_unchanged.store(0, std::memory_order_release);
+
+  // Release resources located above. All stack processing is done so it's
+  // safe if some outside code does another push.
+  if (user_data)
+    GlobalActivityTracker::Get()->ReleaseUserDataMemory(&user_data);
+}
+
+std::unique_ptr<ActivityUserData> ThreadActivityTracker::GetUserData(
+    ActivityId id) {
+  // User-data is only stored for activities actually held in the stack.
+  if (id < stack_slots_) {
+    void* memory =
+        GlobalActivityTracker::Get()->GetUserDataMemory(&stack_[id].user_data);
+    if (memory)
+      return MakeUnique<ActivityUserData>(memory, kUserDataSize);
+  }
+
+  // Return a dummy object that will still accept (but ignore) Set() calls.
+  return MakeUnique<ActivityUserData>(nullptr, 0);
 }
 
 bool ThreadActivityTracker::IsValid() const {
   if (header_->cookie.load(std::memory_order_acquire) != kHeaderCookie ||
       header_->process_id.load(std::memory_order_relaxed) == 0 ||
       header_->thread_ref.as_id == 0 ||
       header_->start_time == 0 ||
       header_->start_ticks == 0 ||
       header_->stack_slots != stack_slots_ ||
       header_->thread_name[sizeof(header_->thread_name) - 1] != '\0') {
@@ skipping 223 matching lines @@
   return tracker;
 }
 
 void GlobalActivityTracker::ReleaseTrackerForCurrentThreadForTesting() {
   ThreadActivityTracker* tracker =
       reinterpret_cast<ThreadActivityTracker*>(this_thread_tracker_.Get());
   if (tracker)
     delete tracker;
 }
 
+void* GlobalActivityTracker::GetUserDataMemory(
+    PersistentMemoryAllocator::Reference* reference) {
+  if (!*reference) {
+    base::AutoLock autolock(user_data_allocator_lock_);
+    *reference = user_data_allocator_.GetObjectReference();
+    if (!*reference)
+      return nullptr;
+  }
+
+  void* memory =
+      allocator_->GetAsObject<char>(*reference, kTypeIdUserDataRecord);
+  DCHECK(memory);
+  return memory;
+}
+
+void GlobalActivityTracker::ReleaseUserDataMemory(
+    PersistentMemoryAllocator::Reference* reference) {
+  DCHECK(*reference);
+  base::AutoLock autolock(user_data_allocator_lock_);
+  user_data_allocator_.ReleaseObjectReference(*reference);
+  *reference = PersistentMemoryAllocator::kReferenceNull;
+}
+
 GlobalActivityTracker::GlobalActivityTracker(
     std::unique_ptr<PersistentMemoryAllocator> allocator,
     int stack_depth)
     : allocator_(std::move(allocator)),
       stack_memory_size_(ThreadActivityTracker::SizeForStackDepth(stack_depth)),
       this_thread_tracker_(&OnTLSDestroy),
       thread_tracker_count_(0),
       thread_tracker_allocator_(allocator_.get(),
                                 kTypeIdActivityTracker,
                                 kTypeIdActivityTrackerFree,
                                 stack_memory_size_,
-                                kCachedThreadMemories) {
+                                kCachedThreadMemories,
+                                /*make_iterable=*/true),
+      user_data_allocator_(allocator_.get(),
+                           kTypeIdUserDataRecord,
+                           kTypeIdUserDataRecordFree,
+                           kUserDataSize,
+                           kCachedUserDataMemories,
+                           /*make_iterable=*/false),
+      user_data_(
+          allocator_->GetAsObject<char>(
+              allocator_->Allocate(kGlobalDataSize, kTypeIdGlobalDataRecord),
+              kTypeIdGlobalDataRecord),
+          kGlobalDataSize) {
   // Ensure the passed memory is valid and empty (iterator finds nothing).
   uint32_t type;
   DCHECK(!PersistentMemoryAllocator::Iterator(allocator_.get()).GetNext(&type));
 
   // Ensure that there is no other global object and then make this one such.
   DCHECK(!g_tracker_);
   g_tracker_ = this;
 }
 
 GlobalActivityTracker::~GlobalActivityTracker() {
@@ skipping 89 matching lines @@
     const base::Process* process)
     : GlobalActivityTracker::ScopedThreadActivity(
           nullptr,
           Activity::ACT_PROCESS_WAIT,
           ActivityData::ForProcess(process->Pid()),
           /*lock_allowed=*/true) {}
 #endif
 
 }  // namespace debug
 }  // namespace base