Support "shared" histograms between processes.

base/metrics/histogram_persistence.cc

+// Copyright (c) 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/metrics/histogram_persistence.h"
+
+#include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_base.h"
+#include "base/metrics/histogram_samples.h"
+#include "base/metrics/statistics_recorder.h"
+#include "base/synchronization/lock.h"
+
+namespace base {
+
+// Type identifiers used when storing in persistent memory so they can be
+// identified during extraction. A "version number" is added to the base
+// so that, if the structure of that object changes, stored older versions
+// will be safely ignored.
+enum : uint32_t {
+  kTypeIdHistogram   = 0xF1645910 + 1,  // SHA1(Histogram) v1
+  kTypeIdRangesArray = 0xBCEA225A + 1,  // SHA1(RangesArray) v1
+  kTypeIdCountsArray = 0x53215530 + 1,  // SHA1(CountsArray) v1
+  kTypeIdNameString  = 0x117DF971 + 1,  // SHA1(NameString) v1
+};
+
+// This data must be held in persistent memory in order for processes to
+// locate and use histograms created elsewhere.
+struct PersistentHistogramData {
+  PersistentMemoryAllocator::Reference name_ref;
+  size_t name_length;
+  int histogram_type;
+  int flags;
+  int minimum;
+  int maximum;
+  size_t bucket_count;
+  PersistentMemoryAllocator::Reference ranges_ref;
+  uint32_t ranges_checksum;
+  PersistentMemoryAllocator::Reference counts_ref;
+  HistogramSamples::Metadata samples_metadata;
+};
+
+scoped_ptr<PersistentMemoryAllocator> allocator_;
+
+void SetPersistentHistogramMemoryAllocator(
+    PersistentMemoryAllocator* allocator) {
+  // Releasing or changing an allocator is extremely dangerous because it
+  // likely has histograms stored within it. If the backing memory is also
+  // also released, future accesses to those histograms will seg-fault.
+  // It's not a fatal CHECK() because tests do this knowing that all
+  // such persistent histograms have already been forgotten.
+  if (allocator_) {
+    LOG(WARNING) << "Active PersistentMemoryAllocator has been released."
+                 << " Some existing histogram pointers may be invalid.";
+  }
+  allocator_.reset(allocator);
+}
+
+PersistentMemoryAllocator* GetPersistentHistogramMemoryAllocator() {
+  return allocator_.get();
+}
+
+PersistentMemoryAllocator* ReleasePersistentHistogramMemoryAllocator() {
+  return allocator_.release();
+};
+
+// Extract a histogram from persistent memory. Unfortunately, the above "pickle"
+// methods cannot be used as part of the persistance because the deserialization
+// methods always create local count data (these must referenced the persistent
+// counts) and always add it to the local list of known histograms (these may
+// be simple references to histograms in other processes).
+HistogramBase* GetPersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    int32_t ref) {
+  PersistentHistogramData* histogram_data =
+      allocator->GetAsObject<PersistentHistogramData>(ref, kTypeIdHistogram);
+  return CreatePersistentHistogram(allocator, histogram_data);
+}
+
+HistogramBase* GetNextPersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    PersistentMemoryAllocator::Iterator* iter) {
+  PersistentMemoryAllocator::Reference ref;
+  uint32_t type_id;
+  while ((ref = allocator->GetNextIterable(iter, &type_id)) != 0) {
+    if (type_id == kTypeIdHistogram)
+      return GetPersistentHistogram(allocator, ref);
+  }
+  return nullptr;
+}
+
+HistogramBase* CreatePersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    PersistentHistogramData* histogram_data_ptr) {
+  if (!histogram_data_ptr) {
+    LOG(WARNING) << "Persistent histogram data was not valid; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+
+  // Copy the histogram_data to local storage because anything in persistent
+  // memory cannot be trusted as it could be changed at any moment by a
+  // malicious actor that shares access. The contents of histogram_data are
+  // validated below; the local copy is to ensure that the contents cannot
+  // be externally changed between validation and use.
+  PersistentHistogramData histogram_data = *histogram_data_ptr;
+
+  char* name_data =
+      allocator->GetAsObject<char>(histogram_data.name_ref, kTypeIdNameString);
+  size_t name_length = histogram_data.name_length;
+  if (!name_data ||
+      name_length >= allocator->GetAllocSize(histogram_data.name_ref) ||
+      name_data[name_length] != 0) {
+    LOG(WARNING) << "Persistent histogram referenced invalid name string"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  std::string name(name_data, name_length);
+
+  HistogramBase::Sample* ranges_data =
+      allocator->GetAsObject<HistogramBase::Sample>(histogram_data.ranges_ref,
+                                                    kTypeIdRangesArray);
+  if (!ranges_data || histogram_data.bucket_count < 2 ||
+      histogram_data.bucket_count + 1 >
+          std::numeric_limits<size_t>::max() / sizeof(HistogramBase::Sample) ||
+      allocator->GetAllocSize(histogram_data.ranges_ref) <
+          (histogram_data.bucket_count + 1) * sizeof(HistogramBase::Sample)) {
+    LOG(WARNING) << "Persistent histogram referenced invalid ranges array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  // To avoid racy destruction at shutdown, the following will be leaked.
+  BucketRanges* ranges = new BucketRanges(histogram_data.bucket_count + 1);
+  bool bad_ranges = false;
+  for (size_t i = 0; i < ranges->size(); ++i) {
+    if (i > 0 && ranges_data[i] <= ranges_data[i - 1])
+      bad_ranges = true;
+    ranges->set_range(i, ranges_data[i]);
+  }
+  ranges->ResetChecksum();
+  if (bad_ranges || ranges->checksum() != histogram_data.ranges_checksum) {
+    LOG(WARNING) << "Persistent histogram referenced invalid ranges array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  const BucketRanges* registered_ranges =
+      StatisticsRecorder::RegisterOrDeleteDuplicateRanges(ranges);
+
+  HistogramBase::AtomicCount* counts_data =
+      allocator->GetAsObject<HistogramBase::AtomicCount>(
+          histogram_data.counts_ref, kTypeIdCountsArray);
+  if (!counts_data ||
+      allocator->GetAllocSize(histogram_data.counts_ref) <
+          histogram_data.bucket_count * sizeof(HistogramBase::AtomicCount)) {
+    LOG(WARNING) << "Persistent histogram referenced invalid counts array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+
+  HistogramBase* histogram = nullptr;
+  switch (histogram_data.histogram_type) {
+  case HISTOGRAM:
+    histogram = new Histogram(
+        name,
+        histogram_data.minimum,
+        histogram_data.maximum,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case LINEAR_HISTOGRAM:
+    histogram = new LinearHistogram(
+        name,
+        histogram_data.minimum,
+        histogram_data.maximum,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case BOOLEAN_HISTOGRAM:
+    histogram = new BooleanHistogram(
+        name,
+        registered_ranges,
+        counts_data,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case CUSTOM_HISTOGRAM:
+    histogram = new CustomHistogram(
+        name,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  }
+
+  if (histogram) {
+    DCHECK_EQ(histogram_data.histogram_type, histogram->GetHistogramType());
+    histogram->SetFlags(histogram_data.flags);
+  }
+
+  return histogram;
+}
+
+HistogramBase* AllocatePersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    HistogramType histogram_type,
+    const std::string& name,
+    int minimum,
+    int maximum,
+    const BucketRanges* bucket_ranges,
+    int32 flags,
+    PersistentMemoryAllocator::Reference* ref_ptr) {
+  if (allocator) {
+    size_t bucket_count = bucket_ranges->bucket_count();
+    CHECK(bucket_count <= std::numeric_limits<int32_t>::max() /
+                              sizeof(HistogramBase::AtomicCount));
+    size_t counts_memory = bucket_count * sizeof(HistogramBase::AtomicCount);
+    size_t ranges_memory = (bucket_count + 1) * sizeof(HistogramBase::Sample);
+    PersistentMemoryAllocator::Reference name_ref =
+        allocator->Allocate(name.size() + 1, kTypeIdNameString);
+    PersistentMemoryAllocator::Reference ranges_ref =
+        allocator->Allocate(ranges_memory, kTypeIdRangesArray);
+    PersistentMemoryAllocator::Reference counts_ref =
+        allocator->Allocate(counts_memory, kTypeIdCountsArray);
+    PersistentMemoryAllocator::Reference histogram_ref =
+        allocator->Allocate(sizeof(PersistentHistogramData), kTypeIdHistogram);
+    char* name_data = allocator->GetAsObject<char>(name_ref, kTypeIdNameString);
+    HistogramBase::Sample* ranges_data =
+        allocator->GetAsObject<HistogramBase::Sample>(ranges_ref,
+                                                      kTypeIdRangesArray);
+    PersistentHistogramData* histogram_data =
+        allocator->GetAsObject<PersistentHistogramData>(histogram_ref,
+                                                        kTypeIdHistogram);
+
+    // Only continue here if all allocations were successful.

[Alexei Svitkine (slow), 2015/12/15 22:54:03]

If only some of them were successful, what handles the freeing?

Another option would be to allocate it all in one chunk and have the different
parts be one after another in the structure. Possibly with offsets/pointers to
them from the struct head. This reduces the failure cases during allocation.

[bcwhite, 2015/12/16 13:54:29]

It's not possible to free anything.  The memory would just be lost, but that's
not unreasonable since the segment is too full for any further allocations
anyway.
Yes but that introduces complexities such as alignment into this code which
seemed better left to lower-level components.  I can easily allocate the name
string directly after the histogram_data structure since, as a char*, it has no
alignment requirements.

[Alexei Svitkine (slow), 2015/12/16 16:33:49]

Okay, that's fair. Please add a comment about that to the code.

[bcwhite, 2015/12/17 15:55:48]

Done.  Though I think I'm going to look at embedding the histogram name in the
main data rather than do separate allocations.

+    if (counts_ref && name_data && ranges_data && histogram_data) {
+      strcpy(name_data, name.c_str());
+      for (size_t i = 0; i < bucket_ranges->size(); ++i)
+        ranges_data[i] = bucket_ranges->range(i);
+
+      histogram_data->name_ref = name_ref;
+      histogram_data->name_length = name.size();
+      histogram_data->histogram_type = histogram_type;
+      histogram_data->flags = flags;
+      histogram_data->minimum = minimum;
+      histogram_data->maximum = maximum;
+      histogram_data->bucket_count = bucket_count;
+      histogram_data->ranges_ref = ranges_ref;
+      histogram_data->ranges_checksum = bucket_ranges->checksum();
+      histogram_data->counts_ref = counts_ref;
+
+      // Create the histogram using resources in persistent memory. This ends up
+      // resolving the "ref" values stored in histogram_data instad of just
+      // using what is already known above but avoids duplicating the switch
+      // statement here and serves as a double-check that everything is
+      // correct before commiting the new histogram to persistent space.
+      HistogramBase* histogram =
+          CreatePersistentHistogram(allocator, histogram_data);
+      DCHECK(histogram);
+      if (ref_ptr != nullptr)
+        *ref_ptr = histogram_ref;
+      return histogram;
+    }
+
+    LOG(WARNING) << "Could not create histogram \"" << name
+                 << "\" in persistent memory (full=" << allocator->IsFull()
+                 << ", corrupt=" << allocator->IsCorrupt() << ")";
+  }
+
+  return nullptr;
+}
+
+void ImportPersistentHistograms() {
+  // Each call resumes from where it last left off so need persistant iterator.
+  // The lock protects against concurrent access to the iterator.
+  static PersistentMemoryAllocator::Iterator iter;
+  static base::Lock lock;
+
+  if (allocator_) {
+    base::AutoLock auto_lock(lock);
+    if (iter.is_clear())
+      allocator_->CreateIterator(&iter);
+
+    for (;;) {
+      HistogramBase* h = GetNextPersistentHistogram(allocator_.get(), &iter);
+      if (!h)
+        break;
+      StatisticsRecorder::RegisterOrDeleteDuplicate(h);
+    }
+  }
+}
+
+}  // namespace base