Reporting top cpu and memory consumers via rappor on chromeos

chrome/browser/chromeos/resource_reporter/resource_reporter.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/resource_reporter/resource_reporter.h"
+
+#include <stdint.h>
+
+#include <queue>
+
+#include "base/bind.h"
+#include "base/memory/singleton.h"
+#include "base/rand_util.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/sys_info.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/task_management/task_manager_interface.h"
+#include "components/rappor/rappor_service.h"
+#include "content/public/browser/browser_thread.h"
+
+namespace chromeos {
+
+namespace {
+
+// The task manager refresh interval, currently at 1 minute.
+const int64_t kRefreshIntervalSeconds = 60;
+
+// 1 GB in bytes.
+const int64_t kMemory1GB = 1024 * 1024 * 1024;
+
+// 800 MB in bytes.
+const int64_t kMemory800MB = 800 * 1024 * 1024;
+
+// 600 MB in bytes.
+const int64_t kMemory600MB = 600 * 1024 * 1024;
+
+// 400 MB in bytes.
+const int64_t kMemory400MB = 400 * 1024 * 1024;
+
+// 200 MB in bytes.
+const int64_t kMemory200MB = 200 * 1024 * 1024;
+
+// The name of the Rappor metric to report the CPU usage.
+const char kCpuRapporMetric[] = "ResourceReporter.Cpu";
+
+// The name of the Rappor metric to report the memory usage.
+const char kMemoryRapporMetric[] = "ResourceReporter.Memory";
+
+// The name of the string field of the Rappor metrics in which we'll record the
+// task's Rappor sample name.
+const char kRapporTaskStringField[] = "task";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// priority of the process on which the task is running.
+const char kRapporPriorityFlagsField[] = "priority";
+
+// The name of the flags field of the CPU usage Rappor metrics in which we'll
+// record the number of cores in the current system.
+const char kRapporNumCoresRangeFlagsField[] = "num_cores_range";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// CPU / memory usage ranges.
+const char kRapporUsageRangeFlagsField[] = "usage_range";
+
+// Currently set to be one day.
+const int kMinimumTimeBetweenReportsInMS = 1 * 24 * 60 * 60 * 1000;
+
+// A functor to sort the TaskRecords by their |cpu|.
+struct TaskRecordByCpuSorter {
+  bool operator()(ResourceReporter::TaskRecord* const& lhs,
+                  ResourceReporter::TaskRecord* const& rhs) const {
+    if (lhs->cpu == rhs->cpu)
+      return lhs->id < rhs->id;
+    return lhs->cpu < rhs->cpu;
+  }
+};
+
+// A functor to sort the TaskRecords by their |memory|.
+struct TaskRecordByMemorySorter {
+  bool operator()(ResourceReporter::TaskRecord* const& lhs,
+                  ResourceReporter::TaskRecord* const& rhs) const {
+    if (lhs->memory == rhs->memory)
+      return lhs->id < rhs->id;
+    return lhs->memory < rhs->memory;
+  }
+};
+
+}  // namespace
+
+ResourceReporter::TaskRecord::TaskRecord(task_management::TaskId the_id)
+    : id(the_id), cpu(0.0), memory(0), is_background(false) {
+}
+
+ResourceReporter::TaskRecord::TaskRecord(task_management::TaskId the_id,
+                                         const std::string& task_name,
+                                         double the_cpu,
+                                         int64_t the_memory,
+                                         bool background)
+    : id(the_id),
+      task_name_for_rappor(task_name),
+      cpu(the_cpu),
+      memory(the_memory),
+      is_background(background) {
+}
+
+ResourceReporter::~ResourceReporter() {
+}
+
+// static
+ResourceReporter* ResourceReporter::GetInstance() {
+  return base::Singleton<ResourceReporter>::get();
+}
+
+void ResourceReporter::StartMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (is_monitoring_)
+    return;
+
+  is_monitoring_ = true;
+  task_management::TaskManagerInterface::GetTaskManager()->AddObserver(this);
+  memory_pressure_listener_.reset(new base::MemoryPressureListener(
+      base::Bind(&ResourceReporter::OnMemoryPressure, base::Unretained(this))));
+}
+
+void ResourceReporter::StopMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (!is_monitoring_)
+    return;
+
+  is_monitoring_ = false;
+  memory_pressure_listener_.reset();
+  task_management::TaskManagerInterface::GetTaskManager()->RemoveObserver(this);
+}
+
+void ResourceReporter::OnTaskAdded(task_management::TaskId id) {
+  // Ignore this event.
+}
+
+void ResourceReporter::OnTaskToBeRemoved(task_management::TaskId id) {
+  auto itr = task_records_.find(id);
+  if (itr == task_records_.end())
+    return;
+
+  // Must be erased from the sorted set first.
+  // Note: this could mean that the sorted records are now less than
+  // |kTopConsumerCount| with other records in |task_records_| that can be
+  // added now. That's ok, we ignore this case.
+  auto cpu_itr = std::find(task_records_by_cpu_.begin(),
+                           task_records_by_cpu_.end(),
+                           itr->second.get());
+  if (cpu_itr != task_records_by_cpu_.end())
+    task_records_by_cpu_.erase(cpu_itr);
+
+  auto memory_itr = std::find(task_records_by_memory_.begin(),
+                              task_records_by_memory_.end(),
+                              itr->second.get());
+  if (memory_itr != task_records_by_memory_.end())
+    task_records_by_memory_.erase(memory_itr);
+
+  task_records_.erase(itr);
+}
+
+void ResourceReporter::OnTasksRefreshed(
+    const task_management::TaskIdList& task_ids) {
+  // A priority queue to sort the task records by their |cpu|. Greatest |cpu|
+  // first.
+  std::priority_queue<TaskRecord*,
+                      std::vector<TaskRecord*>,
+                      TaskRecordByCpuSorter> records_by_cpu_queue;
+  // A priority queue to sort the task records by their |memory|. Greatest
+  // |memory| first.
+  std::priority_queue<TaskRecord*,
+                      std::vector<TaskRecord*>,
+                      TaskRecordByMemorySorter> records_by_memory_queue;
+  task_records_by_cpu_.clear();
+  task_records_by_cpu_.reserve(kTopConsumersCount);
+  task_records_by_memory_.clear();
+  task_records_by_memory_.reserve(kTopConsumersCount);
+
+  for (const auto& id : task_ids) {
+    const double cpu_usage = observed_task_manager()->GetCpuUsage(id);
+    const int64_t memory_usage =
+        observed_task_manager()->GetPhysicalMemoryUsage(id);
+
+    // Browser and GPU processes are reported later using UMA histograms as they
+    // don't have any privacy issues.
+    const auto task_type = observed_task_manager()->GetType(id);
+    switch (task_type) {
+      case task_management::Task::UNKNOWN:
+      case task_management::Task::ZYGOTE:
+        break;
+
+      case task_management::Task::BROWSER:
+        last_browser_process_cpu_ = cpu_usage;
+        last_browser_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      case task_management::Task::GPU:
+        last_gpu_process_cpu_ = cpu_usage;
+        last_gpu_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      default:
+        // Other tasks types will be reported using Rappor.
+        TaskRecord* task_data = nullptr;
+        auto itr = task_records_.find(id);
+        if (itr == task_records_.end()) {
+          task_data = new TaskRecord(id);
+          task_records_[id] = make_scoped_ptr(task_data);
+        } else {
+          task_data = itr->second.get();
+        }
+
+        DCHECK_EQ(task_data->id, id);
+        task_data->task_name_for_rappor =
+            observed_task_manager()->GetTaskNameForRappor(id);
+        task_data->cpu = cpu_usage;
+        task_data->memory = memory_usage;
+        task_data->is_background =
+            observed_task_manager()->IsTaskOnBackgroundedProcess(id);
+
+        // Push only valid or useful data to both priority queues. They might
+        // end up having more records than |kTopConsumerCount|, that's fine.
+        // We'll take care of that next.
+        if (task_data->cpu > 0)
+          records_by_cpu_queue.push(task_data);
+        if (task_data->memory > 0)
+          records_by_memory_queue.push(task_data);
+    }
+  }
+
+  // Sort the |kTopConsumersCount| task records by their CPU and memory usage
+  // and assign weights for each of them based on their order.
+  int cpu_current_weight =
+      std::min(records_by_cpu_queue.size(), kTopConsumersCount);
+  while (!records_by_cpu_queue.empty() &&
+         task_records_by_cpu_.size() < kTopConsumersCount) {
+    task_records_by_cpu_.push_back(records_by_cpu_queue.top());
+    task_records_by_cpu_.back()->cpu_weight = cpu_current_weight--;

[ncarter (slow), 2015/11/18 22:17:15]

This seems like a totally arbitrary assignment of weight. Why do it this way,
versus weighting by the record's |cpu| value?

[afakhry, 2015/11/19 01:36:55]

Bad judgement from my side. Removed.

+    records_by_cpu_queue.pop();
+  }
+
+  int memory_current_weight =
+      std::min(records_by_memory_queue.size(), kTopConsumersCount);
+  while (!records_by_memory_queue.empty() &&
+         task_records_by_memory_.size() < kTopConsumersCount) {
+    task_records_by_memory_.push_back(records_by_memory_queue.top());
+    task_records_by_memory_.back()->memory_weight = memory_current_weight--;
+    records_by_memory_queue.pop();
+  }
+}
+
+// static
+const size_t ResourceReporter::kTopConsumersCount = 10U;
+
+ResourceReporter::ResourceReporter()
+    : TaskManagerObserver(base::TimeDelta::FromSeconds(kRefreshIntervalSeconds),
+                          task_management::REFRESH_TYPE_CPU |
+                              task_management::REFRESH_TYPE_MEMORY |
+                                  task_management::REFRESH_TYPE_PRIORITY),
+      system_cpu_cores_range_(GetCurrentSystemCpuCoresRange()) {
+}
+
+// static
+scoped_ptr<rappor::Sample> ResourceReporter::CreateRapporSample(
+    rappor::RapporService* rappor_service,
+    const ResourceReporter::TaskRecord& task_record) {
+  scoped_ptr<rappor::Sample> sample(rappor_service->CreateSample(
+      rappor::UMA_RAPPOR_TYPE));
+  sample->SetStringField(kRapporTaskStringField,
+                         task_record.task_name_for_rappor);
+  sample->SetFlagsField(kRapporPriorityFlagsField,
+                        task_record.is_background ? BACKGROUND : FOREGROUND,
+                        PRIORITIES_NUM);
+  return sample.Pass();
+}
+
+// static
+ResourceReporter::CpuUsageRange
+ResourceReporter::GetCpuUsageRange(double cpu) {
+  if (cpu > 60.0)
+    return RANGE_ABOVE_60_PERCENT;
+  if (cpu > 30.0)
+    return RANGE_30_TO_60_PERCENT;
+  if (cpu > 10.0)
+    return RANGE_10_TO_30_PERCENT;
+
+  return RANGE_0_TO_10_PERCENT;
+}
+
+// static
+ResourceReporter::MemoryUsageRange
+ResourceReporter::GetMemoryUsageRange(int64_t memory_in_bytes) {
+  if (memory_in_bytes > kMemory1GB)
+    return RANGE_ABOVE_1_GB;
+  if (memory_in_bytes > kMemory800MB)
+    return RANGE_800_TO_1_GB;
+  if (memory_in_bytes > kMemory600MB)
+    return RANGE_600_TO_800_MB;
+  if (memory_in_bytes > kMemory400MB)
+      return RANGE_400_TO_600_MB;
+  if (memory_in_bytes > kMemory200MB)
+      return RANGE_200_TO_400_MB;
+
+  return RANGE_0_TO_200_MB;
+}
+
+// static
+ResourceReporter::CpuCoresNumberRange
+ResourceReporter::GetCurrentSystemCpuCoresRange() {
+  const int cpus = base::SysInfo::NumberOfProcessors();
+
+  if (cpus > 16)
+    return RANGE_CORES_ABOVE_16_CORES;
+  if (cpus > 8)
+    return RANGE_CORES_9_TO_16_CORES;
+  if (cpus > 4)
+    return RANGE_CORES_5_TO_8_CORES;
+  if (cpus > 2)
+    return RANGE_CORES_3_TO_4_CORES;
+  if (cpus == 2)
+    return RANGE_CORES_2_CORES;
+  if (cpus == 1)
+    return RANGE_CORES_1_CORE;
+
+  NOTREACHED();
+  return RANGE_CORES_NA;
+}
+
+bool ResourceReporter::ShouldRecordSamples() {
+  if (is_first_memory_pressure_event_) {
+    is_first_memory_pressure_event_ = false;
+    return true;
+  }
+
+  return (base::TimeTicks::Now() - last_memory_pressure_event_time_) >=
+      base::TimeDelta::FromMilliseconds(kMinimumTimeBetweenReportsInMS);
+}
+
+const ResourceReporter::TaskRecord* ResourceReporter::SampleTaskByCpu() const {
+  TaskRecord* sampled_task = nullptr;
+  int cpu_weights_sum = 0;
+  for (const auto& task_data : task_records_by_cpu_) {
+    if ((base::RandDouble() * (cpu_weights_sum + task_data->cpu_weight)) >=
+        cpu_weights_sum) {
+      sampled_task = task_data;
+    }
+    cpu_weights_sum += task_data->cpu_weight;
+  }
+
+  return sampled_task;
+}
+
+const ResourceReporter::TaskRecord*
+ResourceReporter::SampleTaskByMemory() const {
+  TaskRecord* sampled_task = nullptr;
+  int memory_weights_sum = 0;
+  for (const auto& task_data : task_records_by_memory_) {
+    if ((base::RandDouble() *
+         (memory_weights_sum + task_data->memory_weight)) >=
+          memory_weights_sum) {
+      sampled_task = task_data;
+    }
+    memory_weights_sum += task_data->memory_weight;
+  }
+
+  return sampled_task;
+}
+
+void ResourceReporter::OnMemoryPressure(
+    MemoryPressureLevel memory_pressure_level) {
+  if (memory_pressure_level >=
+      MemoryPressureLevel::MEMORY_PRESSURE_LEVEL_MODERATE) {
+    // Report browser and GPU processes usage using UMA histograms.
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.BrowserProcess.CpuUsage",
+                              GetCpuUsageRange(last_browser_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.BrowserProcess.MemoryUsage",
+                              GetMemoryUsageRange(last_browser_process_memory_),
+                              MEMORY_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.GpuProcess.CpuUsage",
+                              GetCpuUsageRange(last_gpu_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.GpuProcess.MemoryUsage",
+                              GetMemoryUsageRange(last_gpu_process_memory_),
+                              MEMORY_RANGES_NUM);
+
+    // For the rest of tasks, report them using Rappor.
+    auto rappor_service = g_browser_process->rappor_service();
+    if (!rappor_service)
+      return;
+
+    if (!ShouldRecordSamples())
+      return;
+    last_memory_pressure_event_time_ = base::TimeTicks::Now();
+
+    // Use weighted random sampling to select a task to report in the CPU
+    // metric.
+    const TaskRecord* sampled_cpu_task = SampleTaskByCpu();
+    if (sampled_cpu_task) {
+      scoped_ptr<rappor::Sample> cpu_sample(
+          CreateRapporSample(rappor_service, *sampled_cpu_task));
+      cpu_sample->SetFlagsField(kRapporNumCoresRangeFlagsField,
+                                system_cpu_cores_range_,
+                                CORES_RANGES_NUM);
+      cpu_sample->SetFlagsField(kRapporUsageRangeFlagsField,
+                                GetCpuUsageRange(sampled_cpu_task->cpu),

[ncarter (slow), 2015/11/18 22:17:15]

If we're already doing a weighted choice of the task, what are the implications
for logging the weight (which influenced the choice) as a field? Naively, I
would expect that when aggregating this field (e.g. if we wanted to estimate the
average CPU/memory usage for a particular site) we'd need to take the sample
weight into account. Is this something commonly encountered by other reppor use
cases, or do we need to do something special?

E.g., suppose we're sampling two tab scenario, where tab1.com is 100MB 50% of
the time and 200MB 50% of the time, and tab2.com is 100MB 100% of the time, then
tab1.com's average memory usage is 150MB. But we'll see samples according to the
following distribution (ignoring memory pressure events):
  25% 100MB tab1.com
  33% 200MB tab1.com
  42% 100MB tab2.com 

So, assuming that we're ultimately going to see a dashboard that tells us
"tab1.com appears in XX% of memory pressure scenarios, and consumes an average
of YYY megabytes," YYY ought to be 150MB (the true average) and not 157MB (which
is the average of the samples).

[afakhry, 2015/11/19 01:36:55]

We're doing something special here to account for (as far as I understood) a
limitation in the Rappor dashboard, which currently doesn't support the analysis
of multi-dimentional Rappor metrics we're using here (ones with different flag
fields and string fields).

That's why we're randomly selecting one task to report in each metric, cpu and
memory. What we are interested in is seeing the most frequent offenders when the
system is under moderate or critical memory pressure. Hence, the sampling to
select a single task to report via Rappor out of the top 10 ones, based on the
most recent measurements done by the task manager's Refresh() right before the
memory pressure event happens.

Ideally, we'd want to see which usage range bucket the reported task falls into,
but that currently is not supported by the dashboard. holte@ can correct me here
if I'm wrong.

I honestly didn't get the relevance of the example you gave above to what we're
doing here, as we're not trying to average anything over time. Just the most
recent measurements before the memory pressure event is hit are what we're
interested in.

[Steven Holte, 2015/11/19 22:05:32]

You are correct that this sampling scheme doesn't let us compute the average
memory consumed by a particular process, and that in order to compute that we
would need to know the weights that each sample was sampled with.

However, I don't know if that stat is actually that interesting of metric for
this use case, or if including a Rappor-noised indicator of the weight would
give enough data to estimate it with any accuracy.

For more detail about why it might not be interesting metric, consider a task
which is ubiquitous, but has some pathological bad state where it uses way too
much memory.  We might expect that for many reports this task would be very
small say 5MB, but in 1% of cases it's ballooned to 505MB.  Knowing that it uses
10MB on average doesn't tell you much about whether or not it's a problematic
task.

That said, I wouldn't be against including the total amount of memory / CPU used
by all tasks in the reports as an extra field.  If we start sending the other
fields as non-noised, that would probably be enough to compute the averages.

[afakhry, 2015/11/21 01:32:58]

What's the difference between sending a Rappor-noised weight and a Rappor-noised
memory usage range that we already do, considering that we're not interested in
the average usage, but rather the use @ bad memory pressure?

+                                CPU_RANGES_NUM);
+      rappor_service->RecordSampleObj(kCpuRapporMetric, cpu_sample.Pass());
+    }
+
+    // Use weighted random sampling to select a task to report in the memory
+    // metric.
+    const TaskRecord* sampled_memory_task = SampleTaskByMemory();
+    if (sampled_memory_task) {
+      scoped_ptr<rappor::Sample> memory_sample(
+          CreateRapporSample(rappor_service, *sampled_memory_task));
+      memory_sample->SetFlagsField(
+          kRapporUsageRangeFlagsField,
+          GetMemoryUsageRange(sampled_memory_task->memory),
+          MEMORY_RANGES_NUM);
+      rappor_service->RecordSampleObj(kMemoryRapporMetric,
+                                      memory_sample.Pass());
+    }
+  }
+}
+
+}  // namespace chromeos