Support "shared" histograms between processes.

base/metrics/histogram_persistence.cc

Index: base/metrics/histogram_persistence.cc
diff --git a/base/metrics/histogram_persistence.cc b/base/metrics/histogram_persistence.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3d290af59ca61b1263a8b24b0212e6aa9415cc9a
--- /dev/null
+++ b/base/metrics/histogram_persistence.cc
@@ -0,0 +1,326 @@
+// Copyright (c) 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/metrics/histogram_persistence.h"
+
+#include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_base.h"
+#include "base/metrics/histogram_samples.h"
+#include "base/metrics/statistics_recorder.h"
+#include "base/synchronization/lock.h"
+
+namespace {
+
+// Type identifiers used when storing in persistent memory so they can be
+// identified during extraction. A "version number" is added to the base
+// so that, if the structure of that object changes, stored older versions
+// will be safely ignored.
+enum : uint32_t {
+  kTypeIdHistogram   = 0xF1645910 + 1,  // SHA1(Histogram) v1

[Alexei Svitkine (slow), 2016/01/12 21:54:44]

SHA1 identifiers are longer than uint32_t. Do you mean
HashMetricName("Histogram")?

[bcwhite, 2016/01/13 13:38:29]

No.  The number is the first 4 bytes of a sha1 hash.  It could be any arbitrary
integer so long as it's unique.  Trying to do sequential numbering didn't seem
feasible and this seemed better than using a RNG.

+  kTypeIdRangesArray = 0xBCEA225A + 1,  // SHA1(RangesArray) v1
+  kTypeIdCountsArray = 0x53215530 + 1,  // SHA1(CountsArray) v1
+};
+
+// This data must be held in persistent memory in order for processes to
+// locate and use histograms created elsewhere.
+struct PersistentHistogramData {
+  int histogram_type;
+  int flags;
+  int minimum;
+  int maximum;
+  size_t bucket_count;
+  base::PersistentMemoryAllocator::Reference ranges_ref;

[Alexei Svitkine (slow), 2016/01/12 21:54:45]

Nit: The anon namespace can still be inside the base namespace, in which case
you can remove the base:: here.

[bcwhite, 2016/01/13 13:38:29]

Done.

+  uint32_t ranges_checksum;
+  base::PersistentMemoryAllocator::Reference counts_ref;
+  base::HistogramSamples::Metadata samples_metadata;
+
+  // Space for the histogram name will be added during the actual allocation
+  // request. This must be the last field of the structure. A zero-size array
+  // or a "flexible" array would be preferred but is not (yet) valid C++.
+  char name[1];
+};
+
+// The object held here will obviously not be destructed at process exit
+// but that's okay since PersistentMemoryAllocator objects are explicitly
+// forbidden from doing anything essential at exit anyway due to the fact
+// that they depend on data managed elsewhere and which could be destructed
+// first.
+base::PersistentMemoryAllocator* g_allocator = nullptr;
+
+}  // namespace
+
+namespace base {
+
+const Feature kPersistentHistogramsFeature = {

[Alexei Svitkine (slow), 2016/01/12 21:54:45]

Nit: No = needed.

[bcwhite, 2016/01/13 13:38:29]

Done.  It's actually done both ways elsewhere, sometimes within the same file:
https://code.google.com/p/chromium/codesearch#chromium/src/content/public/com...

+  "PersistentMetrics", FEATURE_DISABLED_BY_DEFAULT

[Alexei Svitkine (slow), 2016/01/12 21:54:44]

Nit: PersistentHistograms

[bcwhite, 2016/01/13 13:38:29]

Done.

+};
+
+void SetPersistentHistogramMemoryAllocator(
+    PersistentMemoryAllocator* allocator) {
+  // Releasing or changing an allocator is extremely dangerous because it
+  // likely has histograms stored within it. If the backing memory is also
+  // also released, future accesses to those histograms will seg-fault.
+  // It's not a fatal CHECK() because tests do this knowing that all
+  // such persistent histograms have already been forgotten.

[Alexei Svitkine (slow), 2016/01/12 21:54:45]

I think it should be a DCHECK here but we should have a separate
SerPsersistentHistogramMemoryAllocatorForTesting() function that doesn't have
the DCHECK - only the comment about why only the test version allows it.

+  if (g_allocator) {
+    LOG(WARNING) << "Active PersistentMemoryAllocator has been released."
+                 << " Some existing histogram pointers may be invalid.";
+    delete g_allocator;
+  }
+  g_allocator = allocator;
+}
+
+PersistentMemoryAllocator* GetPersistentHistogramMemoryAllocator() {
+  return g_allocator;
+}
+
+PersistentMemoryAllocator* ReleasePersistentHistogramMemoryAllocator() {
+  PersistentMemoryAllocator* allocator = g_allocator;
+  g_allocator = nullptr;
+  return allocator;
+};
+
+HistogramBase* CreatePersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    PersistentHistogramData* histogram_data_ptr) {
+  if (!histogram_data_ptr) {
+    LOG(WARNING) << "Persistent histogram data was not valid; skipped.";

[Alexei Svitkine (slow), 2016/01/12 21:54:44]

Nit: You can pipe the message into the NOTREACHED().

Same for the other places below.

[bcwhite, 2016/01/13 13:38:29]

But that would omit the warning on Release builds, would it not?  Would you
prefer the message not appear on Release builds?

[Alexei Svitkine (slow), 2016/01/18 19:25:35]

I think it's best practice to have release builds not output messages as it
bloats the binary - and these things are not useful to end users. If you want
monitoring from the wild, then it's better to add an enum histogram with an enum
for each error case.

[bcwhite, 2016/01/20 15:26:14]

Should that histogram have only the errors or should it also include a "success"
entry?

[Alexei Svitkine (slow), 2016/01/20 16:26:41]

Yep, that's usually the best practice.

+    NOTREACHED();
+    return nullptr;
+  }
+
+  // Copy the histogram_data to local storage because anything in persistent
+  // memory cannot be trusted as it could be changed at any moment by a
+  // malicious actor that shares access. The contents of histogram_data are
+  // validated below; the local copy is to ensure that the contents cannot
+  // be externally changed between validation and use.
+  PersistentHistogramData histogram_data = *histogram_data_ptr;
+
+  HistogramBase::Sample* ranges_data =
+      allocator->GetAsObject<HistogramBase::Sample>(histogram_data.ranges_ref,
+                                                    kTypeIdRangesArray);
+  if (!ranges_data || histogram_data.bucket_count < 2 ||
+      histogram_data.bucket_count + 1 >
+          std::numeric_limits<size_t>::max() / sizeof(HistogramBase::Sample) ||
+      allocator->GetAllocSize(histogram_data.ranges_ref) <
+          (histogram_data.bucket_count + 1) * sizeof(HistogramBase::Sample)) {
+    LOG(WARNING) << "Persistent histogram referenced invalid ranges array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  // To avoid racy destruction at shutdown, the following will be leaked.
+  BucketRanges* ranges = new BucketRanges(histogram_data.bucket_count + 1);
+  bool bad_ranges = false;
+  for (size_t i = 0; i < ranges->size(); ++i) {
+    if (i > 0 && ranges_data[i] <= ranges_data[i - 1])
+      bad_ranges = true;
+    ranges->set_range(i, ranges_data[i]);
+  }
+  ranges->ResetChecksum();
+  if (bad_ranges || ranges->checksum() != histogram_data.ranges_checksum) {
+    LOG(WARNING) << "Persistent histogram referenced invalid ranges array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  const BucketRanges* registered_ranges =
+      StatisticsRecorder::RegisterOrDeleteDuplicateRanges(ranges);
+
+  HistogramBase::AtomicCount* counts_data =
+      allocator->GetAsObject<HistogramBase::AtomicCount>(
+          histogram_data.counts_ref, kTypeIdCountsArray);
+  if (!counts_data ||
+      allocator->GetAllocSize(histogram_data.counts_ref) <
+          histogram_data.bucket_count * sizeof(HistogramBase::AtomicCount)) {
+    LOG(WARNING) << "Persistent histogram referenced invalid counts array"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+
+  HistogramBase* histogram = nullptr;
+  switch (histogram_data.histogram_type) {
+  case HISTOGRAM:

[Alexei Svitkine (slow), 2016/01/12 21:54:44]

Nit: indent these 2 more.

[bcwhite, 2016/01/13 13:38:29]

Done.

+    histogram = Histogram::PersistentGet(
+        histogram_data_ptr->name,
+        histogram_data.minimum,
+        histogram_data.maximum,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case LINEAR_HISTOGRAM:
+    histogram = LinearHistogram::PersistentGet(
+        histogram_data_ptr->name,
+        histogram_data.minimum,
+        histogram_data.maximum,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case BOOLEAN_HISTOGRAM:
+    histogram = BooleanHistogram::PersistentGet(
+        histogram_data_ptr->name,
+        registered_ranges,
+        counts_data,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  case CUSTOM_HISTOGRAM:
+    histogram = CustomHistogram::PersistentGet(
+        histogram_data_ptr->name,
+        registered_ranges,
+        counts_data,
+        histogram_data.bucket_count,
+        &histogram_data_ptr->samples_metadata);
+    break;
+  }
+
+  if (histogram) {
+    DCHECK_EQ(histogram_data.histogram_type, histogram->GetHistogramType());
+    histogram->SetFlags(histogram_data.flags);
+  }
+
+  return histogram;
+}
+
+HistogramBase* GetPersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    int32_t ref) {
+  // Unfortunately, the above "pickle" methods cannot be used as part of the
+  // persistance because the deserialization methods always create local
+  // count data (these must referenced the persistent counts) and always add
+  // it to the local list of known histograms (these may be simple references
+  // to histograms in other processes).
+  PersistentHistogramData* histogram_data =
+      allocator->GetAsObject<PersistentHistogramData>(ref, kTypeIdHistogram);
+  size_t length = allocator->GetAllocSize(ref);
+  if (!histogram_data ||
+      reinterpret_cast<char*>(histogram_data)[length - 1] != '\0') {
+    LOG(WARNING) << "Persistent histogram data was invalid or had invalid name"
+                 << "; skipped.";
+    NOTREACHED();
+    return nullptr;
+  }
+  return CreatePersistentHistogram(allocator, histogram_data);
+}
+
+HistogramBase* GetNextPersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    PersistentMemoryAllocator::Iterator* iter) {
+  PersistentMemoryAllocator::Reference ref;
+  uint32_t type_id;
+  while ((ref = allocator->GetNextIterable(iter, &type_id)) != 0) {
+    if (type_id == kTypeIdHistogram)
+      return GetPersistentHistogram(allocator, ref);
+  }
+  return nullptr;
+}
+
+void FinalizePersistentHistogram(PersistentMemoryAllocator::Reference ref,
+                                 bool registered) {
+  // If the created persistent histogram was registered then it needs to
+  // be marked as "iterable" in order to be found by other processes.
+  if (registered)
+    GetPersistentHistogramMemoryAllocator()->MakeIterable(ref);
+  // If it wasn't registered then a race condition must have caused
+  // two to be created. The allocator does not support releasing the
+  // acquired memory so just change the type to be empty.
+  else
+    GetPersistentHistogramMemoryAllocator()->SetType(ref, 0);
+}
+
+HistogramBase* AllocatePersistentHistogram(
+    PersistentMemoryAllocator* allocator,
+    HistogramType histogram_type,
+    const std::string& name,
+    int minimum,
+    int maximum,
+    const BucketRanges* bucket_ranges,
+    int32_t flags,
+    PersistentMemoryAllocator::Reference* ref_ptr) {
+  if (allocator) {
+    size_t bucket_count = bucket_ranges->bucket_count();
+    CHECK(bucket_count <= std::numeric_limits<int32_t>::max() /
+                              sizeof(HistogramBase::AtomicCount));
+    size_t counts_memory = bucket_count * sizeof(HistogramBase::AtomicCount);
+    size_t ranges_memory = (bucket_count + 1) * sizeof(HistogramBase::Sample);
+    PersistentMemoryAllocator::Reference ranges_ref =
+        allocator->Allocate(ranges_memory, kTypeIdRangesArray);
+    PersistentMemoryAllocator::Reference counts_ref =
+        allocator->Allocate(counts_memory, kTypeIdCountsArray);
+    PersistentMemoryAllocator::Reference histogram_ref =
+        allocator->Allocate(offsetof(PersistentHistogramData, name) +
+                            name.length() + 1, kTypeIdHistogram);
+    HistogramBase::Sample* ranges_data =
+        allocator->GetAsObject<HistogramBase::Sample>(ranges_ref,
+                                                      kTypeIdRangesArray);
+    PersistentHistogramData* histogram_data =
+        allocator->GetAsObject<PersistentHistogramData>(histogram_ref,
+                                                        kTypeIdHistogram);
+
+    // Only continue here if all allocations were successful. If they weren't
+    // there is no way to free the space but that's not really a problem since
+    // the allocations only fail because the space is full and so any future
+    // attempts will also fail.
+    if (counts_ref && ranges_data && histogram_data) {
+      strcpy(histogram_data->name, name.c_str());
+      for (size_t i = 0; i < bucket_ranges->size(); ++i)
+        ranges_data[i] = bucket_ranges->range(i);
+
+      histogram_data->histogram_type = histogram_type;
+      histogram_data->flags = flags;
+      histogram_data->minimum = minimum;
+      histogram_data->maximum = maximum;
+      histogram_data->bucket_count = bucket_count;
+      histogram_data->ranges_ref = ranges_ref;
+      histogram_data->ranges_checksum = bucket_ranges->checksum();
+      histogram_data->counts_ref = counts_ref;
+
+      // Create the histogram using resources in persistent memory. This ends up
+      // resolving the "ref" values stored in histogram_data instad of just
+      // using what is already known above but avoids duplicating the switch
+      // statement here and serves as a double-check that everything is
+      // correct before commiting the new histogram to persistent space.
+      HistogramBase* histogram =
+          CreatePersistentHistogram(allocator, histogram_data);
+      DCHECK(histogram);
+      if (ref_ptr != nullptr)
+        *ref_ptr = histogram_ref;
+      return histogram;
+    }
+
+    LOG(WARNING) << "Could not create histogram \"" << name
+                 << "\" in persistent memory (full=" << allocator->IsFull()
+                 << ", corrupt=" << allocator->IsCorrupt() << ")";
+  }
+
+  return nullptr;
+}
+
+void ImportPersistentHistograms() {
+  // Each call resumes from where it last left off so need persistant iterator.
+  // The lock protects against concurrent access to the iterator.
+  static PersistentMemoryAllocator::Iterator iter;
+  static base::Lock lock;
+
+  if (g_allocator) {
+    base::AutoLock auto_lock(lock);
+    if (iter.is_clear())
+      g_allocator->CreateIterator(&iter);
+
+    for (;;) {
+      HistogramBase* h = GetNextPersistentHistogram(g_allocator, &iter);
+      if (!h)
+        break;
+      StatisticsRecorder::RegisterOrDeleteDuplicate(h);
+    }
+  }
+}
+
+}  // namespace base