Multi-Process Tracking Support

base/debug/activity_tracker.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/debug/activity_tracker.h"
 
 #include <algorithm>
 #include <limits>
 #include <utility>
 
@@ skipping 24 matching lines @@
 // A version number is added on so that major structure changes won't try to
 // read an older version (since the cookie won't match).
 const uint32_t kHeaderCookie = 0xC0029B24UL + 2;  // v2
 
 // The minimum depth a stack should support.
 const int kMinStackDepth = 2;
 
 // The amount of memory set aside for holding arbitrary user data (key/value
 // pairs) globally or associated with ActivityData entries.
 const size_t kUserDataSize = 1 << 10;     // 1 KiB
+const size_t kProcessDataSize = 4 << 10;  // 4 KiB
 const size_t kGlobalDataSize = 16 << 10;  // 16 KiB
 const size_t kMaxUserDataNameLength =
     static_cast<size_t>(std::numeric_limits<uint8_t>::max());
 
 // A constant used to indicate that module information is changing.
 const uint32_t kModuleInformationChanging = 0x80000000;
 
 union ThreadRef {
   int64_t as_id;
 #if defined(OS_WIN)
   // On Windows, the handle itself is often a pseudo-handle with a common
   // value meaning "this thread" and so the thread-id is used. The former
   // can be converted to a thread-id with a system call.
   PlatformThreadId as_tid;
 #elif defined(OS_POSIX)
   // On Posix, the handle is always a unique identifier so no conversion
   // needs to be done. However, it's value is officially opaque so there
   // is no one correct way to convert it to a numerical identifier.
   PlatformThreadHandle::Handle as_handle;
 #endif
 };
 
+// Finds and reuses a specific allocation or creates a new one.
+PersistentMemoryAllocator::Reference AllocateFrom(
+    PersistentMemoryAllocator* allocator,
+    uint32_t from_type,
+    size_t size,
+    uint32_t to_type) {
+  PersistentMemoryAllocator::Iterator iter(allocator);
+  PersistentMemoryAllocator::Reference ref;
+  while ((ref = iter.GetNextOfType(from_type)) != 0) {
+    DCHECK_LE(size, allocator->GetAllocSize(ref));
+    // This can fail if a another thread has just taken it. It isassumed that
+    // the memory is cleared during the "free" operation.
+    if (allocator->ChangeType(ref, to_type, from_type, /*clear=*/false))
+      return ref;
+  }
+
+  return allocator->Allocate(size, to_type);
+}
+
 // Determines the previous aligned index.
 size_t RoundDownToAlignment(size_t index, size_t alignment) {
   return index & (0 - alignment);
 }
 
 // Determines the next aligned index.
 size_t RoundUpToAlignment(size_t index, size_t alignment) {
   return (index + (alignment - 1)) & (0 - alignment);
 }
 
@@ skipping 162 matching lines @@
 StringPiece ActivityUserData::TypedValue::GetReference() const {
   DCHECK_EQ(RAW_VALUE_REFERENCE, type_);
   return ref_value_;
 }
 
 StringPiece ActivityUserData::TypedValue::GetStringReference() const {
   DCHECK_EQ(STRING_VALUE_REFERENCE, type_);
   return ref_value_;
 }
 
+// These are required because std::atomic is (currently) not a POD type and
+// thus clang requires explicit out-of-line constructors and destructors even
+// when they do nothing.
 ActivityUserData::ValueInfo::ValueInfo() {}
 ActivityUserData::ValueInfo::ValueInfo(ValueInfo&&) = default;
 ActivityUserData::ValueInfo::~ValueInfo() {}
+ActivityUserData::MemoryHeader::MemoryHeader() {}
+ActivityUserData::MemoryHeader::~MemoryHeader() {}
+ActivityUserData::FieldHeader::FieldHeader() {}
+ActivityUserData::FieldHeader::~FieldHeader() {}
 
 StaticAtomicSequenceNumber ActivityUserData::next_id_;
 
 ActivityUserData::ActivityUserData(void* memory, size_t size)
     : memory_(reinterpret_cast<char*>(memory)),
       available_(RoundDownToAlignment(size, kMemoryAlignment)),
-      id_(reinterpret_cast<std::atomic<uint32_t>*>(memory)) {
+      header_(reinterpret_cast<MemoryHeader*>(memory)) {
   // It's possible that no user data is being stored.
   if (!memory_)
     return;
 
-  DCHECK_LT(kMemoryAlignment, available_);
-  if (id_->load(std::memory_order_relaxed) == 0) {
-    // Generate a new ID and store it in the first 32-bit word of memory_.
-    // |id_| must be non-zero for non-sink instances.
+  static_assert(0 == sizeof(MemoryHeader) % kMemoryAlignment, "invalid header");
+  DCHECK_LT(sizeof(MemoryHeader), available_);
+  if (header_->data_id.load(std::memory_order_acquire) == 0) {
+    // Store the current process ID so analysis can determine which process
+    // generated the data. This is done first so it can be released later.
+    header_->process_id = GetCurrentProcId();
+
+    // Generate a new ID and store it in the header.
+    // |data_id| must be non-zero for non-sink instances.
     uint32_t id;
     while ((id = next_id_.GetNext()) == 0)
       ;
-    id_->store(id, std::memory_order_relaxed);
-    DCHECK_NE(0U, id_->load(std::memory_order_relaxed));
+    header_->data_id.store(id, std::memory_order_release);
   }
-  memory_ += kMemoryAlignment;
-  available_ -= kMemoryAlignment;
+  memory_ += sizeof(MemoryHeader);
+  available_ -= sizeof(MemoryHeader);
 
   // If there is already data present, load that. This allows the same class
   // to be used for analysis through snapshots.
   ImportExistingData();
 }
 
 ActivityUserData::~ActivityUserData() {}
 
+bool ActivityUserData::CreateSnapshot(Snapshot* output_snapshot) const {
+  DCHECK(output_snapshot);
+  DCHECK(output_snapshot->empty());
+
+  // Find any new data that may have been added by an active instance of this
+  // class that is adding records.
+  ImportExistingData();
+
+  for (const auto& entry : values_) {
+    TypedValue value;
+    value.type_ = entry.second.type;
+    DCHECK_GE(entry.second.extent,
+              entry.second.size_ptr->load(std::memory_order_relaxed));
+
+    switch (entry.second.type) {
+      case RAW_VALUE:
+      case STRING_VALUE:
+        value.long_value_ =
+            std::string(reinterpret_cast<char*>(entry.second.memory),
+                        entry.second.size_ptr->load(std::memory_order_relaxed));
+        break;
+      case RAW_VALUE_REFERENCE:
+      case STRING_VALUE_REFERENCE: {
+        ReferenceRecord* ref =
+            reinterpret_cast<ReferenceRecord*>(entry.second.memory);
+        value.ref_value_ = StringPiece(
+            reinterpret_cast<char*>(static_cast<uintptr_t>(ref->address)),
+            static_cast<size_t>(ref->size));
+      } break;
+      case BOOL_VALUE:
+      case CHAR_VALUE:
+        value.short_value_ = *reinterpret_cast<char*>(entry.second.memory);
+        break;
+      case SIGNED_VALUE:
+      case UNSIGNED_VALUE:
+        value.short_value_ = *reinterpret_cast<uint64_t*>(entry.second.memory);
+        break;
+      case END_OF_VALUES:  // Included for completeness purposes.
+        NOTREACHED();
+    }
+    auto inserted = output_snapshot->insert(
+        std::make_pair(entry.second.name.as_string(), std::move(value)));
+    DCHECK(inserted.second);  // True if inserted, false if existed.
+  }
+
+  return true;
+}
+
+const void* ActivityUserData::GetBaseAddress() {
+  // The |memory_| pointer advances as elements are written but the |header_|
+  // value is always at the start of the block so just return that.
+  return header_;
+}
+
+void ActivityUserData::SetOwningProcessIdForTesting(ProcessId id,
+                                                    int64_t stamp) {
+  if (!header_)
+    return;
+  header_->process_id = id;
+  header_->create_stamp = stamp;
+}
+
+// static
+bool ActivityUserData::OwningProcessId(const void* memory,
+                                       ProcessId* out_id,
+                                       int64_t* out_stamp) {
+  const MemoryHeader* header = reinterpret_cast<const MemoryHeader*>(memory);
+  if (header->data_id.load(std::memory_order_acquire) == 0)
+    return false;
+
+  *out_id = static_cast<ProcessId>(header->process_id);
+  *out_stamp = header->create_stamp;
+  return true;
+}
+
 void ActivityUserData::Set(StringPiece name,
                            ValueType type,
                            const void* memory,
                            size_t size) {
   DCHECK_GE(std::numeric_limits<uint8_t>::max(), name.length());
   size = std::min(std::numeric_limits<uint16_t>::max() - (kMemoryAlignment - 1),
                   size);
 
   // It's possible that no user data is being stored.
   if (!memory_)
     return;
 
   // The storage of a name is limited so use that limit during lookup.
   if (name.length() > kMaxUserDataNameLength)
     name.set(name.data(), kMaxUserDataNameLength);
 
   ValueInfo* info;
   auto existing = values_.find(name);
   if (existing != values_.end()) {
     info = &existing->second;
   } else {
     // The name size is limited to what can be held in a single byte but
     // because there are not alignment constraints on strings, it's set tight
     // against the header. Its extent (the reserved space, even if it's not
     // all used) is calculated so that, when pressed against the header, the
     // following field will be aligned properly.
     size_t name_size = name.length();
     size_t name_extent =
-        RoundUpToAlignment(sizeof(Header) + name_size, kMemoryAlignment) -
-        sizeof(Header);
+        RoundUpToAlignment(sizeof(FieldHeader) + name_size, kMemoryAlignment) -
+        sizeof(FieldHeader);
     size_t value_extent = RoundUpToAlignment(size, kMemoryAlignment);
 
     // The "base size" is the size of the header and (padded) string key. Stop
     // now if there's not room enough for even this.
-    size_t base_size = sizeof(Header) + name_extent;
+    size_t base_size = sizeof(FieldHeader) + name_extent;
     if (base_size > available_)
       return;
 
     // The "full size" is the size for storing the entire value.
     size_t full_size = std::min(base_size + value_extent, available_);
 
     // If the value is actually a single byte, see if it can be stuffed at the
     // end of the name extent rather than wasting kMemoryAlignment bytes.
     if (size == 1 && name_extent > name_size) {
       full_size = base_size;
       --name_extent;
       --base_size;
     }
 
     // Truncate the stored size to the amount of available memory. Stop now if
     // there's not any room for even part of the value.
     size = std::min(full_size - base_size, size);
     if (size == 0)
       return;
 
     // Allocate a chunk of memory.
-    Header* header = reinterpret_cast<Header*>(memory_);
+    FieldHeader* header = reinterpret_cast<FieldHeader*>(memory_);
     memory_ += full_size;
     available_ -= full_size;
 
     // Datafill the header and name records. Memory must be zeroed. The |type|
     // is written last, atomically, to release all the other values.
     DCHECK_EQ(END_OF_VALUES, header->type.load(std::memory_order_relaxed));
     DCHECK_EQ(0, header->value_size.load(std::memory_order_relaxed));
     header->name_size = static_cast<uint8_t>(name_size);
     header->record_size = full_size;
-    char* name_memory = reinterpret_cast<char*>(header) + sizeof(Header);
+    char* name_memory = reinterpret_cast<char*>(header) + sizeof(FieldHeader);
     void* value_memory =
-        reinterpret_cast<char*>(header) + sizeof(Header) + name_extent;
+        reinterpret_cast<char*>(header) + sizeof(FieldHeader) + name_extent;
     memcpy(name_memory, name.data(), name_size);
     header->type.store(type, std::memory_order_release);
 
     // Create an entry in |values_| so that this field can be found and changed
     // later on without having to allocate new entries.
     StringPiece persistent_name(name_memory, name_size);
     auto inserted =
         values_.insert(std::make_pair(persistent_name, ValueInfo()));
     DCHECK(inserted.second);  // True if inserted, false if existed.
     info = &inserted.first->second;
     info->name = persistent_name;
     info->memory = value_memory;
     info->size_ptr = &header->value_size;
-    info->extent = full_size - sizeof(Header) - name_extent;
+    info->extent = full_size - sizeof(FieldHeader) - name_extent;
     info->type = type;
   }
 
   // Copy the value data to storage. The |size| is written last, atomically, to
   // release the copied data. Until then, a parallel reader will just ignore
   // records with a zero size.
   DCHECK_EQ(type, info->type);
   size = std::min(size, info->extent);
   info->size_ptr->store(0, std::memory_order_seq_cst);
   memcpy(info->memory, memory, size);
   info->size_ptr->store(size, std::memory_order_release);
 }
 
 void ActivityUserData::SetReference(StringPiece name,
                                     ValueType type,
                                     const void* memory,
                                     size_t size) {
   ReferenceRecord rec;
   rec.address = reinterpret_cast<uintptr_t>(memory);
   rec.size = size;
   Set(name, type, &rec, sizeof(rec));
 }
 
 void ActivityUserData::ImportExistingData() const {
-  while (available_ > sizeof(Header)) {
-    Header* header = reinterpret_cast<Header*>(memory_);
+  while (available_ > sizeof(FieldHeader)) {
+    FieldHeader* header = reinterpret_cast<FieldHeader*>(memory_);
     ValueType type =
         static_cast<ValueType>(header->type.load(std::memory_order_acquire));
     if (type == END_OF_VALUES)
       return;
     if (header->record_size > available_)
       return;
 
-    size_t value_offset = RoundUpToAlignment(sizeof(Header) + header->name_size,
-                                             kMemoryAlignment);
+    size_t value_offset = RoundUpToAlignment(
+        sizeof(FieldHeader) + header->name_size, kMemoryAlignment);
     if (header->record_size == value_offset &&
         header->value_size.load(std::memory_order_relaxed) == 1) {
       value_offset -= 1;
     }
     if (value_offset + header->value_size > header->record_size)
       return;
 
     ValueInfo info;
-    info.name = StringPiece(memory_ + sizeof(Header), header->name_size);
+    info.name = StringPiece(memory_ + sizeof(FieldHeader), header->name_size);
     info.type = type;
     info.memory = memory_ + value_offset;
     info.size_ptr = &header->value_size;
     info.extent = header->record_size - value_offset;
 
     StringPiece key(info.name);
     values_.insert(std::make_pair(key, std::move(info)));
 
     memory_ += header->record_size;
     available_ -= header->record_size;
   }
 }
 
-bool ActivityUserData::CreateSnapshot(Snapshot* output_snapshot) const {
-  DCHECK(output_snapshot);
-  DCHECK(output_snapshot->empty());
-
-  // Find any new data that may have been added by an active instance of this
-  // class that is adding records.
-  ImportExistingData();
-
-  for (const auto& entry : values_) {
-    TypedValue value;
-    value.type_ = entry.second.type;
-    DCHECK_GE(entry.second.extent,
-              entry.second.size_ptr->load(std::memory_order_relaxed));
-
-    switch (entry.second.type) {
-      case RAW_VALUE:
-      case STRING_VALUE:
-        value.long_value_ =
-            std::string(reinterpret_cast<char*>(entry.second.memory),
-                        entry.second.size_ptr->load(std::memory_order_relaxed));
-        break;
-      case RAW_VALUE_REFERENCE:
-      case STRING_VALUE_REFERENCE: {
-        ReferenceRecord* ref =
-            reinterpret_cast<ReferenceRecord*>(entry.second.memory);
-        value.ref_value_ = StringPiece(
-            reinterpret_cast<char*>(static_cast<uintptr_t>(ref->address)),
-            static_cast<size_t>(ref->size));
-      } break;
-      case BOOL_VALUE:
-      case CHAR_VALUE:
-        value.short_value_ = *reinterpret_cast<char*>(entry.second.memory);
-        break;
-      case SIGNED_VALUE:
-      case UNSIGNED_VALUE:
-        value.short_value_ = *reinterpret_cast<uint64_t*>(entry.second.memory);
-        break;
-      case END_OF_VALUES:  // Included for completeness purposes.
-        NOTREACHED();
-    }
-    auto inserted = output_snapshot->insert(
-        std::make_pair(entry.second.name.as_string(), std::move(value)));
-    DCHECK(inserted.second);  // True if inserted, false if existed.
-  }
-
-  return true;
-}
-
-const void* ActivityUserData::GetBaseAddress() {
-  // The |memory_| pointer advances as elements are written but the |id_|
-  // value is always at the start of the block so just return that.
-  return id_;
-}
-
 // This information is kept for every thread that is tracked. It is filled
 // the very first time the thread is seen. All fields must be of exact sizes
 // so there is no issue moving between 32 and 64-bit builds.
 struct ThreadActivityTracker::Header {
   // Defined in .h for analyzer access. Increment this if structure changes!
   static constexpr uint32_t kPersistentTypeId =
       GlobalActivityTracker::kTypeIdActivityTracker;
 
   // Expected size for 32/64-bit check.
   static constexpr size_t kExpectedInstanceSize = 80;
@@ skipping 396 matching lines @@
     }
 
     // Success!
     return true;
   }
 
   // Too many attempts.
   return false;
 }
 
+const void* ThreadActivityTracker::GetBaseAddress() {
+  return header_;
+}
+
+void ThreadActivityTracker::SetOwningProcessIdForTesting(ProcessId id,
+                                                         int64_t stamp) {
+  header_->process_id.store(id, std::memory_order_relaxed);
+  header_->start_time = stamp;
+}
+
+// static
+bool ThreadActivityTracker::OwningProcessId(const void* memory,
+                                            ProcessId* out_id,
+                                            int64_t* out_stamp) {
+  const Header* header = reinterpret_cast<const Header*>(memory);
+  if (header->cookie.load(std::memory_order_acquire) != kHeaderCookie)
+    return false;
+
+  *out_id = static_cast<ProcessId>(
+      header->process_id.load(std::memory_order_relaxed));
+  *out_stamp = header->start_time;
+  return true;
+}
+
 // static
 size_t ThreadActivityTracker::SizeForStackDepth(int stack_depth) {
   return static_cast<size_t>(stack_depth) * sizeof(Activity) + sizeof(Header);
 }
 
 // The instantiation of the GlobalActivityTracker object.
 // The object held here will obviously not be destructed at process exit
 // but that's best since PersistentMemoryAllocator objects (that underlie
 // GlobalActivityTracker objects) are explicitly forbidden from doing anything
 // essential at exit anyway due to the fact that they depend on data managed
@@ skipping 274 matching lines @@
   return tracker;
 }
 
 void GlobalActivityTracker::ReleaseTrackerForCurrentThreadForTesting() {
   ThreadActivityTracker* tracker =
       reinterpret_cast<ThreadActivityTracker*>(this_thread_tracker_.Get());
   if (tracker)
     delete tracker;
 }
 
+void GlobalActivityTracker::SetBackgroundTaskRunner(
+    const scoped_refptr<TaskRunner>& runner) {
+  AutoLock lock(global_tracker_lock_);
+  background_task_runner_ = runner;
+}
+
+void GlobalActivityTracker::RecordProcessLaunch(ProcessId process_id) {}
+
+void GlobalActivityTracker::RecordProcessExit(ProcessId process_id,
+                                              int exit_code) {
+  DCHECK_NE(GetCurrentProcId(), process_id);
+
+  int64_t now_stamp = Time::Now().ToInternalValue();
+
+  // The persistent allocator is thread-safe so run the iteration and
+  // adjustments on a worker thread if one was provided.
+  {
+    AutoLock lock(global_tracker_lock_);
+    if (background_task_runner_ &&
+        !background_task_runner_->RunsTasksOnCurrentThread()) {
+      background_task_runner_->PostTask(
+          FROM_HERE, Bind(&GlobalActivityTracker::RecordProcessExitImpl,
+                          Unretained(this), process_id, exit_code, now_stamp));
+      return;
+    }
+  }
+
+  RecordProcessExitImpl(process_id, exit_code, now_stamp);
+}
+
+void GlobalActivityTracker::RecordProcessExitImpl(ProcessId process_id,
+                                                  int exit_code,
+                                                  int64_t exit_stamp) {
+  // The process may not have exited cleanly so its necessary to go through
+  // all the data structures it may have allocated in the persistent memory
+  // segment and mark them as "released". This will allow them to be reused
+  // later on. Memory is cleared here, rather than when it's needed, so as to
+  // limit the impact at that critical time.
+  PersistentMemoryAllocator::Iterator iter(allocator_.get());
+  PersistentMemoryAllocator::Reference ref;
+  uint32_t type;
+  while ((ref = iter.GetNext(&type)) != 0) {
+    const void* memory = allocator_->GetAsArray<char>(
+        ref, type, PersistentMemoryAllocator::kSizeAny);
+    ProcessId found_id;
+    int64_t create_stamp;
+
+    switch (type) {
+      case kTypeIdActivityTracker:
+        if (ThreadActivityTracker::OwningProcessId(memory, &found_id,
+                                                   &create_stamp)) {
+          break;
+        }
+        continue;
+
+      case kTypeIdUserDataRecord:
+      case kTypeIdProcessDataRecord:
+        if (ActivityUserData::OwningProcessId(memory, &found_id,
+                                              &create_stamp)) {
+          break;
+        }
+        continue;
+
+      default:
+        continue;
+    }
+
+    // Only change the type to be "free" if the process ID matches and the
+    // creation time is before the exit time (so PID re-use doesn't cause
+    // the erasure of something that is in-use).
+    if (found_id == process_id && create_stamp < exit_stamp)
+      allocator_->ChangeType(ref, ~type, type, /*clear=*/true);
+  }
+}
+
 void GlobalActivityTracker::RecordLogMessage(StringPiece message) {
   // Allocate at least one extra byte so the string is NUL terminated. All
   // memory returned by the allocator is guaranteed to be zeroed.
   PersistentMemoryAllocator::Reference ref =
       allocator_->Allocate(message.size() + 1, kTypeIdGlobalLogMessage);
   char* memory = allocator_->GetAsArray<char>(ref, kTypeIdGlobalLogMessage,
                                               message.size() + 1);
   if (memory) {
     memcpy(memory, message.data(), message.size());
     allocator_->MakeIterable(ref);
@@ skipping 43 matching lines @@
                                 kTypeIdActivityTracker,
                                 kTypeIdActivityTrackerFree,
                                 stack_memory_size_,
                                 kCachedThreadMemories,
                                 /*make_iterable=*/true),
       user_data_allocator_(allocator_.get(),
                            kTypeIdUserDataRecord,
                            kTypeIdUserDataRecordFree,
                            kUserDataSize,
                            kCachedUserDataMemories,
-                           /*make_iterable=*/false),
+                           /*make_iterable=*/true),
+      process_data_(allocator_->GetAsArray<char>(
+                        AllocateFrom(allocator_.get(),
+                                     kTypeIdProcessDataRecordFree,
+                                     kProcessDataSize,
+                                     kTypeIdProcessDataRecord),
+                        kTypeIdProcessDataRecord,
+                        kProcessDataSize),
+                    kProcessDataSize),
       global_data_(
           allocator_->GetAsArray<char>(
               allocator_->Allocate(kGlobalDataSize, kTypeIdGlobalDataRecord),
               kTypeIdGlobalDataRecord,
-              PersistentMemoryAllocator::kSizeAny),
+              kGlobalDataSize),
           kGlobalDataSize) {
   // Ensure the passed memory is valid and empty (iterator finds nothing).
   uint32_t type;
   DCHECK(!PersistentMemoryAllocator::Iterator(allocator_.get()).GetNext(&type));
 
   // Ensure that there is no other global object and then make this one such.
   DCHECK(!g_tracker_);
   subtle::NoBarrier_Store(&g_tracker_, reinterpret_cast<uintptr_t>(this));
 
-  // The global records must be iterable in order to be found by an analyzer.
+  // The data records must be iterable in order to be found by an analyzer.
+  allocator_->MakeIterable(allocator_->GetAsReference(
+      process_data_.GetBaseAddress(), kTypeIdProcessDataRecord));
   allocator_->MakeIterable(allocator_->GetAsReference(
       global_data_.GetBaseAddress(), kTypeIdGlobalDataRecord));
 
   // Fetch and record all activated field trials.
   FieldTrial::ActiveGroups active_groups;
   FieldTrialList::GetActiveFieldTrialGroups(&active_groups);
   for (auto& group : active_groups)
     RecordFieldTrial(group.trial_name, group.group_name);
 }
 
@@ skipping 102 matching lines @@
     : GlobalActivityTracker::ScopedThreadActivity(
           program_counter,
           nullptr,
           Activity::ACT_PROCESS_WAIT,
           ActivityData::ForProcess(process->Pid()),
           /*lock_allowed=*/true) {}
 #endif
 
 }  // namespace debug
 }  // namespace base