Reporting top cpu and memory consumers via rappor on chromeos

chrome/browser/chromeos/resource_reporter/resource_reporter.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/resource_reporter/resource_reporter.h"
+
+#include <stdint.h>
+#include <string>
+
+#include "base/bind.h"
+#include "base/memory/singleton.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/sys_info.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/task_management/task_manager_interface.h"
+#include "components/rappor/rappor_service.h"
+#include "content/public/browser/browser_thread.h"
+
+namespace chromeos {
+
+namespace {
+
+// The task manager refresh interval, currently at 1 minute.
+const int64_t kRefreshIntervalSeconds = 60;
+
+// 1 GB in bytes.
+const int64_t kMemory1GB = 1024 * 1024 * 1024;
+
+// 800 MB in bytes.
+const int64_t kMemory800MB = 800 * 1024 * 1024;
+
+// 600 MB in bytes.
+const int64_t kMemory600MB = 600 * 1024 * 1024;
+
+// 400 MB in bytes.
+const int64_t kMemory400MB = 400 * 1024 * 1024;
+
+// 200 MB in bytes.
+const int64_t kMemory200MB = 200 * 1024 * 1024;
+
+// The name of the Rappor metric to report the CPU usage.
+const char kCpuRapporMetric[] = "ResourceReporter.Cpu";
+
+// The name of the Rappor metric to report the memory usage.
+const char kMemoryRapporMetric[] = "ResourceReporter.Memory";
+
+// The name of the string field of the Rappor metrics in which we'll record the
+// task's Rappor sample name.
+const char kRapporTaskStringField[] = "task";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// priority of the process on which the task is running.
+const char kRapporPriorityFlagsField[] = "priority";
+
+// The name of the flags field of the CPU usage Rappor metrics in which we'll
+// record the number of cores in the current system.
+const char kRapporNumCoresRangeFlagsField[] = "num_cores_range";
+
+// The name of the flags field of the Rappor metrics in which we'll store the
+// CPU / memory usage ranges.
+const char kRapporUsageRangeFlagsField[] = "usage_range";
+
+// The names of UMA histograms to report the CPU and memory usages of both the
+// browser and GPU processes.
+const char kBrowserProcessCpuUsageHistogramName[] =
+    "ResourceReporter.BrowserProcess.CpuUsage";
+const char kBrowserProcessMemoryUsageHistogramName[] =
+    "ResourceReporter.BrowserProcess.MemoryUsage";
+const char kGpuProcessCpuUsageHistogramName[] =
+    "ResourceReporter.GpuProcess.CpuUsage";
+const char kGpuProcessMemoryUsageHistogramName[] =
+    "ResourceReporter.GpuProcess.MemoryUsage";
+
+}  // namespace
+
+ResourceReporter::~ResourceReporter() {
+}
+
+// static
+ResourceReporter* ResourceReporter::GetInstance() {
+  return base::Singleton<ResourceReporter>::get();
+}
+
+// static
+void ResourceReporter::StartObservingMetricsService() {
+  auto metrics_service = g_browser_process->metrics_service();
+  DCHECK(metrics_service);
+  metrics_service->AddObserver(ResourceReporter::GetInstance());
+}
+
+// static
+void ResourceReporter::StopObservingMetricsService() {
+  auto metrics_service = g_browser_process->metrics_service();
+  DCHECK(metrics_service);
+  metrics_service->RemoveObserver(ResourceReporter::GetInstance());
+}
+
+void ResourceReporter::OnTaskAdded(task_management::TaskId id) {
+  // Ignore this event.
+}
+
+void ResourceReporter::OnTaskToBeRemoved(task_management::TaskId id) {
+  auto itr = task_records_.find(id);
+  if (itr == task_records_.end())
+    return;
+
+  // Must be erased from the sorted set first.
+  // Note: this could mean that the sorted records are now less than
+  // |kTopConsumerCount| with other records in |task_records_| that can be
+  // added now. That's ok, we ignore this case.
+  task_records_by_cpu_.erase(itr->second.get());
+  task_records_by_memory_.erase(itr->second.get());
+
+  task_records_.erase(itr);
+}
+
+void ResourceReporter::OnTasksRefreshed(
+    const task_management::TaskIdList& task_ids) {
+  task_records_by_cpu_.clear();
+  task_records_by_memory_.clear();
+
+  for (const auto& id : task_ids) {
+    const double cpu_usage = observed_task_manager()->GetCpuUsage(id);
+    const int64_t memory_usage =
+        observed_task_manager()->GetPhysicalMemoryUsage(id);
+
+    // Browser and GPU processes are reported later using UMA histograms as they
+    // don't have any privacy issues.
+    const auto task_type = observed_task_manager()->GetType(id);
+    switch (task_type) {
+      case task_management::Task::BROWSER:
+        last_browser_process_cpu_ = cpu_usage;
+        last_browser_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      case task_management::Task::GPU:
+        last_gpu_process_cpu_ = cpu_usage;
+        last_gpu_process_memory_ = memory_usage != -1 ? memory_usage : 0;
+        break;
+
+      default:
+        // Other tasks types will be reported using Rappor.
+        TaskRecord* task_data = nullptr;
+        auto itr = task_records_.find(id);
+        if (itr == task_records_.end()) {
+          task_data = new TaskRecord(id);
+          task_records_[id] = make_scoped_ptr(task_data);
+        } else {
+          task_data = itr->second.get();
+        }
+
+        DCHECK_EQ(task_data->id, id);
+        task_data->rappor_sample =
+            observed_task_manager()->GetRapporSampleName(id);
+        task_data->cpu = cpu_usage;
+        task_data->memory = memory_usage;
+        task_data->is_background =
+            observed_task_manager()->IsTaskOnBackgroundedProcess(id);
+
+        if (task_records_by_cpu_.size() < kTopConsumersCount &&
+            task_data->cpu > 0) {
+          task_records_by_cpu_.insert(task_data);
+        }
+
+        if (task_records_by_memory_.size() < kTopConsumersCount &&
+            task_data->memory > 0) {
+          task_records_by_memory_.insert(task_data);

[ncarter (slow), 2015/11/10 23:29:56]

It looks like this logic means we just count the first 10 items, rather than the
top 10 items (or 10 random items, or 10 cost-weighted random items)? Is that
right?

Assuming we were trying to sample the top 10: you can insert the item
unconditionally, and then pop the lowest one off if it put us over
kTopConsumersCount. I think a std::priority_queue ought to be the ideal
container for that.

[afakhry, 2015/11/12 00:21:27]

Thank you so much for catching that nasty bug! Indeed a priority queue is the
perfect container for this use case.

+        }
+    }
+  }
+}
+
+void ResourceReporter::OnMetricsServiceStart() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  StartMonitoring();
+}
+
+void ResourceReporter::OnMetricsServiceStop() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  StopMonitoring();
+}
+
+// static
+const size_t ResourceReporter::kTopConsumersCount = 10U;
+
+ResourceReporter::ResourceReporter()
+    : TaskManagerObserver(base::TimeDelta::FromSeconds(kRefreshIntervalSeconds),
+                          task_management::REFRESH_TYPE_CPU |
+                              task_management::REFRESH_TYPE_MEMORY),
+      system_cpu_cores_range_(GetCurrentSystemCpuCoresRange()),
+      is_monitoring_(false) {
+}
+
+// static
+scoped_ptr<rappor::Sample> ResourceReporter::CreateRapporSample(
+    rappor::RapporService* rappor_service,
+    const ResourceReporter::TaskRecord& task_record) {
+  scoped_ptr<rappor::Sample> sample(rappor_service->CreateSample(
+      rappor::UMA_RAPPOR_TYPE));
+  sample->SetStringField(kRapporTaskStringField, task_record.rappor_sample);
+  sample->SetFlagsField(kRapporPriorityFlagsField,
+                        task_record.is_background ? BACKGROUND : FOREGROUND,
+                        PRIORITIES_NUM);
+  return sample.Pass();
+}
+
+// static
+ResourceReporter::CpuUsageRange
+ResourceReporter::GetCpuUsageRange(double cpu) {
+  if (cpu > 60.0)
+    return RANGE_ABOVE_60_PERCENT;
+  if (cpu > 30.0)
+    return RANGE_30_TO_60_PERCENT;
+  if (cpu > 10.0)
+    return RANGE_10_TO_30_PERCENT;
+
+  return RANGE_0_TO_10_PERCENT;
+}
+
+// static
+ResourceReporter::MemoryUsageRange
+ResourceReporter::GetMemoryUsageRange(int64_t memory_in_bytes) {
+  if (memory_in_bytes > kMemory1GB)
+    return RANGE_ABOVE_1_GB;
+  if (memory_in_bytes > kMemory800MB)
+    return RANGE_800_TO_1_GB;
+  if (memory_in_bytes > kMemory600MB)
+    return RANGE_600_TO_800_MB;
+  if (memory_in_bytes > kMemory400MB)
+      return RANGE_400_TO_600_MB;
+  if (memory_in_bytes > kMemory200MB)
+      return RANGE_200_TO_400_MB;
+
+  return RANGE_0_TO_200_MB;
+}
+
+// static
+ResourceReporter::CpuCoresNumberRange
+ResourceReporter::GetCurrentSystemCpuCoresRange() {
+  const int cpus = base::SysInfo::NumberOfProcessors();
+
+  if (cpus > 16)
+    return RANGE_CORES_ABOVE_16_CORES;
+  if (cpus > 8)
+    return RANGE_CORES_8_TO_16_CORES;
+  if (cpus > 4)
+    return RANGE_CORES_4_TO_8_CORES;
+  if (cpus > 2)
+    return RANGE_CORES_2_TO_4_CORES;
+  if (cpus >= 1)
+    return RANGE_CORES_1_TO_2_CORES;
+
+  NOTREACHED();
+  return RANGE_CORES_0_CORES;
+}
+
+void ResourceReporter::StartMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (is_monitoring_)
+    return;
+
+  is_monitoring_ = true;
+  task_management::TaskManagerInterface::GetTaskManager()->AddObserver(this);
+  memory_pressure_listener_.reset(new base::MemoryPressureListener(
+      base::Bind(&ResourceReporter::OnMemoryPressure, base::Unretained(this))));
+}
+
+void ResourceReporter::StopMonitoring() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (!is_monitoring_)
+    return;
+
+  is_monitoring_ = false;
+  memory_pressure_listener_.reset();
+  task_management::TaskManagerInterface::GetTaskManager()->RemoveObserver(this);
+}
+
+void ResourceReporter::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  // TODO(afakhry): Double check if we will ever receive a notification when
+  // we don't have memory pressure at all. The target is to report only once
+  // per each level value.
+  if (memory_pressure_level >=
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_MODERATE &&
+      memory_pressure_level > previous_memory_pressure_level_) {
+    // Report browser and GPU processes usage using UMA histograms.
+    UMA_HISTOGRAM_ENUMERATION(kBrowserProcessCpuUsageHistogramName,

[ncarter (slow), 2015/11/10 23:29:56]

Don't use a kConstant for the histogram name, just inline literal. I think
otherwise you'll fail the presubmit check below?

https://code.google.com/p/chromium/codesearch#chromium/src/PRESUBMIT.py&q=UMA...

[afakhry, 2015/11/12 00:21:28]

That's weird! I didn't fail any presubmit checks at all! Is that a new check?

[ncarter (slow), 2015/11/12 19:33:27]

Ah, I looked at it more closely. The regex includes a literal double-quote
character, so if you use an indirect name, the check doesn't run at all.

I still think it's a good idea to use the literal notation (unless you really
need to increment the histogram in multiple places, which is very rare) -- it's
a tad easier to grep for when you're looking at histogram data and want to cross
reference the code.

[afakhry, 2015/11/13 19:41:13]

Done.

+                              GetCpuUsageRange(last_browser_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION(kBrowserProcessMemoryUsageHistogramName,
+                              GetMemoryUsageRange(last_browser_process_memory_),
+                              MEMORY_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION(kGpuProcessCpuUsageHistogramName,
+                              GetCpuUsageRange(last_gpu_process_cpu_),
+                              CPU_RANGES_NUM);
+    UMA_HISTOGRAM_ENUMERATION(kGpuProcessMemoryUsageHistogramName,
+                              GetMemoryUsageRange(last_gpu_process_memory_),
+                              MEMORY_RANGES_NUM);
+
+    // For the rest of tasks, report them using Rappor.
+    auto rappor_service = g_browser_process->rappor_service();
+
+    for (const auto& task_data : task_records_by_cpu_) {
+      scoped_ptr<rappor::Sample> sample(CreateRapporSample(rappor_service,
+                                                           *task_data));
+      sample->SetFlagsField(kRapporNumCoresRangeFlagsField,
+                            system_cpu_cores_range_,
+                            CORES_RANGES_NUM);
+      sample->SetFlagsField(kRapporUsageRangeFlagsField,
+                            GetCpuUsageRange(task_data->cpu),
+                            CPU_RANGES_NUM);
+      rappor_service->RecordSampleObj(kCpuRapporMetric, sample.Pass());

[ncarter (slow), 2015/11/10 23:29:56]

Doesn't this oversample processes with multiple tasks?

[afakhry, 2015/11/12 00:21:27]

I'm not sure I understand what you need to know, but I'll try to explain. When
multiple samples map to the same Rappor metric during the same reporting
interval, Rappor will pick only a single one randomly for generating the report.
We sample all the tasks but we done't know which one rappor will be selecting at
the end.

[ncarter (slow), 2015/11/12 19:33:27]

By "oversampling", I mean that now a 100MB process with two tasks in it is twice
as likely to be sampled as a 100MB process with one task in it. That'll bias
your overall population estimate toward tasks that are actually consuming less
memory (since the two tasks average 50MB each, though it could be a 90/10
split).

It's still a little fuzzy to me what we're trying to estimate. Even if you
assume each task has its own process, logging an unweighted random element
within the top 10 consumers doesn't really agree with my mathematical intuition.
What value does the top-10 cutoff really provide, if we're just picking one of
the top ten at random?

[afakhry, 2015/11/13 19:41:12]

The top 10 limit is merely for us to limit our attention. When there are a huge
number of samples from many users, It's easy to say that these are the tasks
that show up in the top 10 list of consumers when the system is under pressure.

[ncarter (slow), 2015/11/13 22:44:32]

The current strategy still seems a little heuristic to me -- maybe that's fine,
or maybe we'll need a couple of iterations of this data pipeline
before we get data that we really understand how to interpret. Either way,
it seems like launching & iterating is the way to go, so I'm not going
to hold this CL up on my concerns.

+    }
+
+    for (const auto& task_data : task_records_by_memory_) {
+      scoped_ptr<rappor::Sample> sample(CreateRapporSample(rappor_service,
+                                                           *task_data));
+      sample->SetFlagsField(kRapporUsageRangeFlagsField,
+                            GetMemoryUsageRange(task_data->memory),
+                            MEMORY_RANGES_NUM);
+      rappor_service->RecordSampleObj(kMemoryRapporMetric, sample.Pass());

[ncarter (slow), 2015/11/10 23:29:56]

Same question as above: doesn't this oversample groups with >1 task?

[afakhry, 2015/11/12 00:21:27]

See above.

+    }
+  }
+
+  previous_memory_pressure_level_ = memory_pressure_level;
+}
+
+}  // namespace chromeos