Reporting top cpu and memory consumers via rappor on chromeos

chrome/browser/chromeos/resource_reporter/resource_reporter.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/resource_reporter/resource_reporter.h"
+
+#include <stdint.h>
+
+#include <queue>
+#include <string>
+
+#include "base/bind.h"
+#include "base/memory/singleton.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/sys_info.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/task_management/task_manager_interface.h"
+#include "components/rappor/rappor_service.h"
+#include "content/public/browser/browser_thread.h"
+
+namespace chromeos {
+
+namespace {
+
+// The task manager refresh interval, currently at 1 minute.
+const int64_t kRefreshIntervalSeconds = 60;
+
+// 1 GB in bytes.
+const int64_t kMemory1GB = 1024 * 1024 * 1024;
+
+// 800 MB in bytes.
+const int64_t kMemory800MB = 800 * 1024 * 1024;
+
+// 600 MB in bytes.
+const int64_t kMemory600MB = 600 * 1024 * 1024;
+
+// 400 MB in bytes.
+const int64_t kMemory400MB = 400 * 1024 * 1024;
+
+// 200 MB in bytes.
+const int64_t kMemory200MB = 200 * 1024 * 1024;
+
+// The name of the Rappor metric to report the CPU usage.
+const char kCpuRapporMetric[] = "ResourceReporter.Cpu";
+
+// The name of the Rappor metric to report the memory usage.
+const char kMemoryRapporMetric[] = "ResourceReporter.Memory";
+
+// The name of the string field of the Rappor metrics in which we'll record the
+// task's Rappor sample name.
+const char kRapporTaskStringField[] = "task";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// priority of the process on which the task is running.
+const char kRapporPriorityFlagsField[] = "priority";
+
+// The name of the flags field of the CPU usage Rappor metrics in which we'll
+// record the number of cores in the current system.
+const char kRapporNumCoresRangeFlagsField[] = "num_cores_range";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// CPU / memory usage ranges.
+const char kRapporUsageRangeFlagsField[] = "usage_range";
+
+// A functor to sort the TaskRecords by their |cpu|.
+struct TaskRecordByCpuSorter {
+  bool operator()(ResourceReporter::TaskRecord* const& lhs,
+                  ResourceReporter::TaskRecord* const& rhs) const {
+    if (lhs->cpu == rhs->cpu)
+      return lhs->id < rhs->id;
+    return lhs->cpu < rhs->cpu;
+  }
+};
+
+// A functor to sort the TaskRecords by their |memory|.
+struct TaskRecordByMemorySorter {
+  bool operator()(ResourceReporter::TaskRecord* const& lhs,
+                  ResourceReporter::TaskRecord* const& rhs) const {
+    if (lhs->memory == rhs->memory)
+      return lhs->id < rhs->id;
+    return lhs->memory < rhs->memory;
+  }
+};
+
+}  // namespace
+
+ResourceReporter::TaskRecord::TaskRecord(task_management::TaskId the_id)
+    : id(the_id), cpu(0.0), memory(0), is_background(false) {
+}
+
+ResourceReporter::TaskRecord::TaskRecord(task_management::TaskId the_id,
+                                         const std::string& the_sample,
+                                         double the_cpu,
+                                         int64_t the_memory,
+                                         bool background)
+    : id(the_id),
+      rappor_sample(the_sample),
+      cpu(the_cpu),
+      memory(the_memory),
+      is_background(background) {
+}
+
+ResourceReporter::~ResourceReporter() {
+}
+
+// static
+ResourceReporter* ResourceReporter::GetInstance() {
+  return base::Singleton<ResourceReporter>::get();
+}
+
+// static
+void ResourceReporter::StartObservingMetricsService() {
+  auto metrics_service = g_browser_process->metrics_service();
+  DCHECK(metrics_service);
+  metrics_service->AddObserver(ResourceReporter::GetInstance());
+}
+
+// static
+void ResourceReporter::StopObservingMetricsService() {
+  // In case we were still monitoring the task manager, we must stop monitoring
+  // here, since we stop observing the MetricService we won't be able to
+  // receive any notifications from it when it actually stops.
+  ResourceReporter::GetInstance()->StopMonitoring();
+  auto metrics_service = g_browser_process->metrics_service();
+  DCHECK(metrics_service);
+  metrics_service->RemoveObserver(ResourceReporter::GetInstance());
+}
+
+void ResourceReporter::OnTaskAdded(task_management::TaskId id) {
+  // Ignore this event.
+}
+
+void ResourceReporter::OnTaskToBeRemoved(task_management::TaskId id) {
+  auto itr = task_records_.find(id);
+  if (itr == task_records_.end())
+    return;
+
+  // Must be erased from the sorted set first.
+  // Note: this could mean that the sorted records are now less than
+  // |kTopConsumerCount| with other records in |task_records_| that can be
+  // added now. That's ok, we ignore this case.
+  auto cpu_itr = std::find(task_records_by_cpu_.begin(),
+                           task_records_by_cpu_.end(),
+                           itr->second.get());
+  if (cpu_itr != task_records_by_cpu_.end())
+    task_records_by_cpu_.erase(cpu_itr);
+
+  auto memory_itr = std::find(task_records_by_memory_.begin(),
+                              task_records_by_memory_.end(),
+                              itr->second.get());
+  if (memory_itr != task_records_by_memory_.end())
+    task_records_by_memory_.erase(memory_itr);
+
+  task_records_.erase(itr);
+}
+
+void ResourceReporter::OnTasksRefreshed(
+    const task_management::TaskIdList& task_ids) {
+  // A priority queue to sort the task records by their |cpu|. Greatest |cpu|
+  // first.
+  std::priority_queue<TaskRecord*,
+                      std::vector<TaskRecord*>,
+                      TaskRecordByCpuSorter> records_by_cpu_queue;
+  // A priority queue to sort the task records by their |memory|. Greatest
+  // |memory| first.
+  std::priority_queue<TaskRecord*,
+                      std::vector<TaskRecord*>,
+                      TaskRecordByMemorySorter> records_by_memory_queue;
+  task_records_by_cpu_.clear();
+  task_records_by_cpu_.reserve(kTopConsumersCount);
+  task_records_by_memory_.clear();
+  task_records_by_memory_.reserve(kTopConsumersCount);
+
+  for (const auto& id : task_ids) {
+    const double cpu_usage = observed_task_manager()->GetCpuUsage(id);
+    const int64_t memory_usage =
+        observed_task_manager()->GetPhysicalMemoryUsage(id);
+
+    // Browser and GPU processes are reported later using UMA histograms as they
+    // don't have any privacy issues.
+    const auto task_type = observed_task_manager()->GetType(id);
+    switch (task_type) {
+      case task_management::Task::BROWSER:
+        last_browser_process_cpu_ = cpu_usage;
+        last_browser_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      case task_management::Task::GPU:
+        last_gpu_process_cpu_ = cpu_usage;
+        last_gpu_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      default:

[ncarter (slow), 2015/11/13 22:44:32]

Are you sure you want to sample all task types?

I imagine that Zygotes, for example, will show up when there are fewer than 10
renderer processes.

[afakhry, 2015/11/16 18:44:19]

Done.

+        // Other tasks types will be reported using Rappor.
+        TaskRecord* task_data = nullptr;
+        auto itr = task_records_.find(id);
+        if (itr == task_records_.end()) {
+          task_data = new TaskRecord(id);
+          task_records_[id] = make_scoped_ptr(task_data);
+        } else {
+          task_data = itr->second.get();
+        }
+
+        DCHECK_EQ(task_data->id, id);
+        task_data->rappor_sample =
+            observed_task_manager()->GetRapporSampleName(id);
+        task_data->cpu = cpu_usage;
+        task_data->memory = memory_usage;
+        task_data->is_background =
+            observed_task_manager()->IsTaskOnBackgroundedProcess(id);
+
+        // Push only valid or useful data to both priority queues. They might
+        // end up having more records than |kTopConsumerCount|, that's fine.
+        // We'll take care of that next.
+        if (task_data->cpu > 0)
+          records_by_cpu_queue.push(task_data);
+        if (task_data->memory > 0)
+          records_by_memory_queue.push(task_data);
+    }
+  }
+
+  while (!records_by_cpu_queue.empty() &&
+         task_records_by_cpu_.size() < kTopConsumersCount) {
+    task_records_by_cpu_.push_back(records_by_cpu_queue.top());
+    records_by_cpu_queue.pop();
+  }
+
+  while (!records_by_memory_queue.empty() &&
+         task_records_by_memory_.size() < kTopConsumersCount) {
+    task_records_by_memory_.push_back(records_by_memory_queue.top());
+    records_by_memory_queue.pop();
+  }
+}
+
+void ResourceReporter::OnMetricsServiceStart() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);

[Steven Holte, 2015/11/13 20:32:25]

Why are you observing the metrics service?  I would discourage tieing a
collection activities to the state of the service.

[afakhry, 2015/11/13 20:56:06]

The collection activities is done only for the purpose of reporting via Rappor.
If Rappor reporting is disabled, then there's no need at all for doing this
work.

[afakhry, 2015/11/16 18:44:19]

There are two things here. First thing is the stats calculations done by the
task manager for the various tasks running on the system, and the second being
the processing of these stats we do here in this system (ResourceReporter).
The task manager has been recently rewritten from scratch optimized specifically
for such use cases. Stats calculations will not introduce any jank for users.
The processing we do here is trivial.
Despite of that, I strongly believe that if this system won't be doing the task
which is the only reason for its existence (reporting samples), then there is no
need for running it altogether.

+
+  StartMonitoring();
+}
+
+void ResourceReporter::OnMetricsServiceStop() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  StopMonitoring();
+}
+
+// static
+const size_t ResourceReporter::kTopConsumersCount = 10U;
+
+ResourceReporter::ResourceReporter()
+    : TaskManagerObserver(base::TimeDelta::FromSeconds(kRefreshIntervalSeconds),
+                          task_management::REFRESH_TYPE_CPU |
+                              task_management::REFRESH_TYPE_MEMORY),
+      system_cpu_cores_range_(GetCurrentSystemCpuCoresRange()),
+      is_monitoring_(false) {
+}
+
+// static
+scoped_ptr<rappor::Sample> ResourceReporter::CreateRapporSample(
+    rappor::RapporService* rappor_service,
+    const ResourceReporter::TaskRecord& task_record) {
+  scoped_ptr<rappor::Sample> sample(rappor_service->CreateSample(
+      rappor::UMA_RAPPOR_TYPE));
+  sample->SetStringField(kRapporTaskStringField, task_record.rappor_sample);
+  sample->SetFlagsField(kRapporPriorityFlagsField,
+                        task_record.is_background ? BACKGROUND : FOREGROUND,
+                        PRIORITIES_NUM);
+  return sample.Pass();
+}
+
+// static
+ResourceReporter::CpuUsageRange
+ResourceReporter::GetCpuUsageRange(double cpu) {
+  if (cpu > 60.0)
+    return RANGE_ABOVE_60_PERCENT;
+  if (cpu > 30.0)
+    return RANGE_30_TO_60_PERCENT;
+  if (cpu > 10.0)
+    return RANGE_10_TO_30_PERCENT;
+
+  return RANGE_0_TO_10_PERCENT;
+}
+
+// static
+ResourceReporter::MemoryUsageRange
+ResourceReporter::GetMemoryUsageRange(int64_t memory_in_bytes) {
+  if (memory_in_bytes > kMemory1GB)
+    return RANGE_ABOVE_1_GB;
+  if (memory_in_bytes > kMemory800MB)
+    return RANGE_800_TO_1_GB;
+  if (memory_in_bytes > kMemory600MB)
+    return RANGE_600_TO_800_MB;
+  if (memory_in_bytes > kMemory400MB)
+      return RANGE_400_TO_600_MB;
+  if (memory_in_bytes > kMemory200MB)
+      return RANGE_200_TO_400_MB;
+
+  return RANGE_0_TO_200_MB;
+}
+
+// static
+ResourceReporter::CpuCoresNumberRange
+ResourceReporter::GetCurrentSystemCpuCoresRange() {
+  const int cpus = base::SysInfo::NumberOfProcessors();
+
+  if (cpus > 16)
+    return RANGE_CORES_ABOVE_16_CORES;
+  if (cpus > 8)
+    return RANGE_CORES_9_TO_16_CORES;
+  if (cpus > 4)
+    return RANGE_CORES_5_TO_8_CORES;
+  if (cpus > 2)
+    return RANGE_CORES_3_TO_4_CORES;
+  if (cpus == 2)
+    return RANGE_CORES_2_CORES;
+  if (cpus == 1)
+    return RANGE_CORES_1_CORE;
+
+  NOTREACHED();
+  return RANGE_CORES_NA;
+}
+
+void ResourceReporter::StartMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (is_monitoring_)
+    return;
+
+  is_monitoring_ = true;
+  task_management::TaskManagerInterface::GetTaskManager()->AddObserver(this);
+  memory_pressure_listener_.reset(new base::MemoryPressureListener(
+      base::Bind(&ResourceReporter::OnMemoryPressure, base::Unretained(this))));
+}
+
+void ResourceReporter::StopMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (!is_monitoring_)
+    return;
+
+  is_monitoring_ = false;
+  memory_pressure_listener_.reset();
+  task_management::TaskManagerInterface::GetTaskManager()->RemoveObserver(this);
+}
+
+void ResourceReporter::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  // TODO(afakhry): Double check if we will ever receive a notification when
+  // we don't have memory pressure at all. The target is to report only once
+  // per each level value.
+  if (memory_pressure_level >=
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_MODERATE &&
+      memory_pressure_level > previous_memory_pressure_level_) {
+    // Report browser and GPU processes usage using UMA histograms.
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.BrowserProcess.CpuUsage",
+                              GetCpuUsageRange(last_browser_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.BrowserProcess.MemoryUsage",
+                              GetMemoryUsageRange(last_browser_process_memory_),
+                              MEMORY_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.GpuProcess.CpuUsage",
+                              GetCpuUsageRange(last_gpu_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION("ResourceReporter.GpuProcess.MemoryUsage",
+                              GetMemoryUsageRange(last_gpu_process_memory_),
+                              MEMORY_RANGES_NUM);
+
+    // For the rest of tasks, report them using Rappor.
+    auto rappor_service = g_browser_process->rappor_service();
+
+    for (const auto& task_data : task_records_by_cpu_) {

[Steven Holte, 2015/11/13 21:42:18]

The code for creating the samples looks fine, but I'm not sure I understand the
implications of your sampling strategy.  I think it's probably worth setting up
a meeting with Ananth/Ilya to discuss how you would actually be able to analyze
this data.

[ncarter (slow), 2015/11/13 22:44:32]

Feel free to include me in this meeting, if it happens.

+      scoped_ptr<rappor::Sample> sample(CreateRapporSample(rappor_service,
+                                                           *task_data));
+      sample->SetFlagsField(kRapporNumCoresRangeFlagsField,
+                            system_cpu_cores_range_,
+                            CORES_RANGES_NUM);
+      sample->SetFlagsField(kRapporUsageRangeFlagsField,
+                            GetCpuUsageRange(task_data->cpu),
+                            CPU_RANGES_NUM);

[ncarter (slow), 2015/11/13 22:44:32]

Dumb question: would it make sense to record the memory in the cpu sample as
well (and vice versa)?

[afakhry, 2015/11/16 18:44:19]

I thought about that, but since top CPU tasks are probably different than top
memory tasks, I preferred to keep separate rappor metric for each.

+      rappor_service->RecordSampleObj(kCpuRapporMetric, sample.Pass());
+    }
+
+    for (const auto& task_data : task_records_by_memory_) {
+      scoped_ptr<rappor::Sample> sample(CreateRapporSample(rappor_service,
+                                                           *task_data));
+      sample->SetFlagsField(kRapporUsageRangeFlagsField,
+                            GetMemoryUsageRange(task_data->memory),
+                            MEMORY_RANGES_NUM);
+      rappor_service->RecordSampleObj(kMemoryRapporMetric, sample.Pass());
+    }
+  }
+
+  previous_memory_pressure_level_ = memory_pressure_level;
+}
+
+}  // namespace chromeos