memory-infra: port peak detection logic to MemoryPeakDetector

base/trace_event/memory_peak_detector.cc

 // Copyright 2017 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/trace_event/memory_peak_detector.h"
 
-#include <stdint.h>
-
 #include "base/bind.h"
 #include "base/logging.h"
+#include "base/sys_info.h"
 #include "base/threading/sequenced_task_runner_handle.h"
 #include "base/time/time.h"
+#include "base/trace_event/memory_dump_manager.h"
 #include "base/trace_event/memory_dump_provider_info.h"
+#include "base/trace_event/trace_event.h"
+#include "build/build_config.h"
 
 namespace base {
 namespace trace_event {
 
 // static
 MemoryPeakDetector* MemoryPeakDetector::GetInstance() {
   static MemoryPeakDetector* instance = new MemoryPeakDetector();
   return instance;
 }
 
 MemoryPeakDetector::MemoryPeakDetector()
     : generation_(0),
       state_(NOT_INITIALIZED),
-      polling_interval_ms_(0),
       poll_tasks_count_for_testing_(0) {}
 
 MemoryPeakDetector::~MemoryPeakDetector() {
   // This is hit only in tests, in which case the test is expected to TearDown()
   // cleanly and not leave the peak detector running.
   DCHECK_EQ(NOT_INITIALIZED, state_);
 }
 
 void MemoryPeakDetector::Setup(
     const GetDumpProvidersFunction& get_dump_providers_function,
     const scoped_refptr<SequencedTaskRunner>& task_runner,
     const OnPeakDetectedCallback& on_peak_detected_callback) {
   DCHECK(!get_dump_providers_function.is_null());
   DCHECK(task_runner);
   DCHECK(!on_peak_detected_callback.is_null());
   DCHECK(state_ == NOT_INITIALIZED || state_ == DISABLED);
   DCHECK(dump_providers_.empty());
   get_dump_providers_function_ = get_dump_providers_function;
   task_runner_ = task_runner;
   on_peak_detected_callback_ = on_peak_detected_callback;
   state_ = DISABLED;
+  config_ = {};
+  ResetPollHistory();
+
+#if !defined(OS_NACL)

[hjd, 2017/04/06 11:33:48]

The previous check also excluded iOS I don't know if that is important:
https://cs.chromium.org/chromium/src/base/trace_event/memory_dump_scheduler.c...

[Primiano Tucci (use gerrit), 2017/04/06 20:07:10]

This one seems defined for ios (base/sys_info_ios.mm)

+  // Set threshold to 1% of total system memory.
+  static_threshold_bytes_ =
+      static_cast<uint64_t>(SysInfo::AmountOfPhysicalMemory()) / 100;
+#else
+  static_threshold_bytes_ = 0;
+#endif
 }
 
 void MemoryPeakDetector::TearDown() {
   if (task_runner_) {
     task_runner_->PostTask(
         FROM_HERE,
         Bind(&MemoryPeakDetector::TearDownInternal, Unretained(this)));
   }
   task_runner_ = nullptr;
 }
 
-void MemoryPeakDetector::Start() {
-  task_runner_->PostTask(
-      FROM_HERE, Bind(&MemoryPeakDetector::StartInternal, Unretained(this)));
+void MemoryPeakDetector::Start(MemoryPeakDetector::Config config) {
+  if (!config.polling_interval_ms) {
+    NOTREACHED();
+    return;
+  }
+  task_runner_->PostTask(FROM_HERE, Bind(&MemoryPeakDetector::StartInternal,
+                                         Unretained(this), config));
 }
 
 void MemoryPeakDetector::Stop() {
   task_runner_->PostTask(
       FROM_HERE, Bind(&MemoryPeakDetector::StopInternal, Unretained(this)));
 }
 
+void MemoryPeakDetector::Clear() {
+  if (!task_runner_)
+    return;  // Can be called before Setup().
+  task_runner_->PostTask(
+      FROM_HERE, Bind(&MemoryPeakDetector::ResetPollHistory, Unretained(this)));
+}
+
 void MemoryPeakDetector::NotifyMemoryDumpProvidersChanged() {
-  // It is possible to call this before the first Setup() call, in which case
-  // we want to just make this a noop. The next Start() will fetch the MDP list.
   if (!task_runner_)
-    return;
+    return;  // Can be called before Setup().
   task_runner_->PostTask(
       FROM_HERE,
       Bind(&MemoryPeakDetector::ReloadDumpProvidersAndStartPollingIfNeeded,
            Unretained(this)));
 }
 
-void MemoryPeakDetector::StartInternal() {
+void MemoryPeakDetector::StartInternal(MemoryPeakDetector::Config config) {
   DCHECK_EQ(DISABLED, state_);
   state_ = ENABLED;
-  polling_interval_ms_ = 1;  // TODO(primiano): temporary until next CL.
+  config_ = config;
+  ResetPollHistory();
 
-  // If there are any dump providers available, NotifyMemoryDumpProvidersChanged
-  // will fetch them and start the polling. Otherwise this will remain in the
-  // ENABLED state and the actual polling will start on the next call to
+  // If there are any dump providers available,
+  // NotifyMemoryDumpProvidersChanged will fetch them and start the polling.
+  // Otherwise this will remain in the ENABLED state and the actual polling
+  // will start on the next call to
   // ReloadDumpProvidersAndStartPollingIfNeeded().
-  // Depending on the sandbox model, it is possible that no polling-capable dump
-  // providers will be ever available.
+  // Depending on the sandbox model, it is possible that no polling-capable
+  // dump providers will be ever available.
   ReloadDumpProvidersAndStartPollingIfNeeded();
 }
 
 void MemoryPeakDetector::StopInternal() {
   DCHECK_NE(NOT_INITIALIZED, state_);
   state_ = DISABLED;
   ++generation_;
   dump_providers_.clear();
 }
 
@@ skipping 23 matching lines @@
                            Bind(&MemoryPeakDetector::PollMemoryAndDetectPeak,
                                 Unretained(this), ++generation_));
   } else if (state_ == RUNNING && dump_providers_.empty()) {
     // Will cause the next PollMemoryAndDetectPeak() task to early return.
     state_ = ENABLED;
     ++generation_;
   }
 }
 
 void MemoryPeakDetector::PollMemoryAndDetectPeak(uint32_t expected_generation) {
-  if (state_ != RUNNING || expected_generation != generation_)
+  if (state_ != RUNNING || generation_ != expected_generation)
     return;
 
   // We should never end up in a situation where state_ == RUNNING but all dump
   // providers are gone.
   DCHECK(!dump_providers_.empty());
 
   poll_tasks_count_for_testing_++;
-  uint64_t memory_total = 0;
+  uint64_t polled_mem_bytes = 0;
   for (const scoped_refptr<MemoryDumpProviderInfo>& mdp_info :
        dump_providers_) {
     DCHECK(mdp_info->options.is_fast_polling_supported);
     uint64_t value = 0;
     mdp_info->dump_provider->PollFastMemoryTotal(&value);
-    memory_total += value;
+    polled_mem_bytes += value;
   }
-  ignore_result(memory_total);  // TODO(primiano): temporary until next CL.
+  if (config_.enable_verbose_poll_tracing) {
+    TRACE_COUNTER1(MemoryDumpManager::kTraceCategory, "PolledMemoryKB",
+                   polled_mem_bytes / 1024);
+  }
 
-  // TODO(primiano): Move actual peak detection logic from the
-  // MemoryDumpScheduler in next CLs.
+  // Peak detection logic. Design doc: https://goo.gl/0kOU4A .
+  bool is_peak = false;
+  if (skip_polls_ > 0) {
+    skip_polls_--;
+  } else if (last_dump_memory_total_ == 0) {
+    last_dump_memory_total_ = polled_mem_bytes;
+  } else if (polled_mem_bytes > 0) {
+    int64_t diff_from_last_dump = polled_mem_bytes - last_dump_memory_total_;
+
+    DCHECK_GT(static_threshold_bytes_, 0u);
+    is_peak = static_threshold_bytes_ &&
+              (diff_from_last_dump > static_threshold_bytes_);
+
+    if (!is_peak)
+      is_peak = DetectPeakUsingSlidingWindowStddev(polled_mem_bytes);
+  }
+
+  if (is_peak) {
+    TRACE_EVENT_INSTANT1(MemoryDumpManager::kTraceCategory,
+                         "Peak memory dump detected", TRACE_EVENT_SCOPE_PROCESS,
+                         "PolledMemoryKB", polled_mem_bytes / 1024);
+    on_peak_detected_callback_.Run();
+    ResetPollHistory();
+    last_dump_memory_total_ = polled_mem_bytes;
+  }
 
   SequencedTaskRunnerHandle::Get()->PostDelayedTask(
       FROM_HERE,
       Bind(&MemoryPeakDetector::PollMemoryAndDetectPeak, Unretained(this),
            expected_generation),
-      TimeDelta::FromMilliseconds(polling_interval_ms_));
+      TimeDelta::FromMilliseconds(config_.polling_interval_ms));
+}
+
+bool MemoryPeakDetector::DetectPeakUsingSlidingWindowStddev(
+    uint64_t polled_mem_bytes) {
+  DCHECK(polled_mem_bytes);
+  samples_bytes_[samples_index_] = polled_mem_bytes;
+  samples_index_ = (samples_index_ + 1) % kSlidingWindowNumSamples;
+  uint64_t mean = 0;
+  for (uint32_t i = 0; i < kSlidingWindowNumSamples; ++i) {
+    if (samples_bytes_[i] == 0)

[hjd, 2017/04/06 11:33:48]

We won't be able to detect a spike from 0 with this code which I guess is
probably fine.

[Primiano Tucci (use gerrit), 2017/04/06 20:07:10]

Yeah. 0 memory usage is quite rare :)

+      return false;  // Not enough samples to detect peaks.
+    mean += samples_bytes_[i];
+  }
+  mean /= kSlidingWindowNumSamples;
+  uint64_t variance = 0;
+  for (uint32_t i = 0; i < kSlidingWindowNumSamples; ++i) {
+    const uint64_t deviation = samples_bytes_[i] - mean;
+    variance += deviation * deviation;
+  }
+  variance /= kSlidingWindowNumSamples;
+
+  // If stddev is less than 0.2% then we consider that the process is inactive.
+  if (variance < (mean / 500) * (mean / 500))
+    return false;
+
+  // (mean + 3.69 * stddev) corresponds to a value that is higher than current
+  // sample with 99.99% probability.
+  const uint64_t cur_sample_deviation = polled_mem_bytes - mean;
+  return cur_sample_deviation * cur_sample_deviation > (3.69 * 3.69 * variance);
+}
+
+void MemoryPeakDetector::ResetPollHistory() {
+  skip_polls_ = 0;
+  last_dump_memory_total_ = 0;
+  memset(samples_bytes_, 0, sizeof(samples_bytes_));
+  samples_index_ = 0;
+  if (config_.polling_interval_ms > 0) {
+    skip_polls_ =
+        (config_.min_time_between_peaks_ms + config_.polling_interval_ms - 1) /
+        config_.polling_interval_ms;
+  }
+}
+
+void MemoryPeakDetector::SetStaticThresholdForTesting(
+    uint64_t static_threshold_bytes) {
+  DCHECK_EQ(DISABLED, state_);
+  static_threshold_bytes_ = static_threshold_bytes;
 }
 
 }  // namespace trace_event
 }  // namespace base