[telemetry] Implement first version of timeline based measurement

tools/perf/measurements/timeline_based_measurement.py

Index: tools/perf/measurements/timeline_based_measurement.py
diff --git a/tools/perf/measurements/timeline_based_measurement.py b/tools/perf/measurements/timeline_based_measurement.py
new file mode 100644
index 0000000000000000000000000000000000000000..eab9e635a655f8e9b59f2b1f4f8852df7b1d30fc
--- /dev/null
+++ b/tools/perf/measurements/timeline_based_measurement.py
@@ -0,0 +1,96 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from telemetry.page import page_measurement

[tonyg, 2014/02/26 17:20:04]

uber nit: alphabetize

+from metrics import timeline as timeline_module
+from metrics import timeline_based_metric_request as tbmr_module
+
+
+SMOOTHNESS_METRIC_TYPE = 'smoothness'
+
+
+# TimelineBasedMeasurement considers all instrumentation as producing a single
+# timeline. But, depending on the amount of instrumentation that is enabled,
+# overhead increases. The user of the measurement must therefore chose between
+# a few levels of instrumentation.
+MINIMUM_INSTRUMENTATION_LEVEL = 'minimum'
+FIVE_PERCENT_OVERHEAD_INSTRUMENTATION_LEVEL = 'five-percent-overhead'
+DEBUG_INSTRUMENTATION_LEVEL = 'debug'
+
+ALL_INSTRUMENTATION_LEVELS = [

[tonyg, 2014/02/26 17:20:04]

These levels are an awesome idea :)

I'm wondering if this would be even better. WDYT?

NO_OVERHEAD_LEVEL => Things like measuring page load time, memory consumption,
power consumption, framegrabber video capture, etc where you just checkpoint
before and after the benchmark without running anything extra during the
benchmark.

MINIMAL_OVERHEAD_LEVEL => Things like tracing, devtools timeline and android
video capture which have minimal overhead, but we will shy away from listing an
exact percent.

DEBUG_OVERHEAD_LEVEL => Things like recording skpictures, running perf profiler,
etc which definitely have non-negligible overhead.

+  MINIMUM_INSTRUMENTATION_LEVEL,
+  FIVE_PERCENT_OVERHEAD_INSTRUMENTATION_LEVEL,
+  DEBUG_INSTRUMENTATION_LEVEL
+]
+
+
+class _TimelineBasedMetrics(object):
+  def __init__(self, model, renderer_thread):
+    self._model = model
+    self._renderer_thread = renderer_thread
+
+  def FindTimelineMetricRequests(self):
+    # TODO(nduca): Add support for page-load metric request.
+    return [tbmr_module.TimelineBasedMetricRequest(event) for
+            event in self._renderer_thread.IterAllAsyncSlices()
+            if tbmr_module.IsTimelineMetricRequest(event.name)]
+
+  def CreateMetricsForTimelineMetricRequest(self, request):
+    if request.metric_type == SMOOTHNESS_METRIC_TYPE:
+      return [] # TODO(nduca): Hook up a real metric.
+    raise Exception('Unrecognized metric type: %s' % request.metric_type)

[nednguyen, 2014/02/26 01:00:10]

I think eventually we will want this to be configurable instead of hard coded. I
can imagine the scenario that GWS, bigtop, and chromium are interested in
different config of metric_type->metrics.
Can you add a TODO so this can read the mapping from metric_type to metric lists
from an external file?

+
+  def AddResults(self, results):
+    requests = self.FindTimelineMetricRequests()
+    if len(requests) == 0:
+      raise Exception('Expected at least one request from the page')
+    for request in requests:
+      metrics = self.CreateMetricsForTimelineMetricRequest(request)
+      for m in metrics:
+        m.AddResults(self._model, self._renderer_thread,
+                     request, results)
+
+
+class TimelineBasedMeasurement(page_measurement.PageMeasurement):
+  """ A meta measurement shifts the burden of what metrics to collect onto
+  the page under test. Instead of the measurement having a fixed set of values
+  it collects about the page, the page being tested issues a set of standard
+  calls to the user timing API specifying time spans of interest for which it
+  wants metrics, and what kind of metrics are requested for that time range. The
+  meta measurement object collects a trace and these metric requests, then
+  generates result values based on matching those requests with the appropriate
+  timeline metrics. """
+  def __init__(self):
+    super(TimelineBasedMeasurement, self).__init__('smoothness')
+
+  def AddCommandLineOptions(self, parser):
+    parser.add_option(
+        '--instrumentation-level', type='choice',
+        choices=ALL_INSTRUMENTATION_LEVELS,
+        default=MINIMUM_INSTRUMENTATION_LEVEL,
+        help='How much instrumentation to enable in the measurement.')
+
+  def CanRunForPage(self, page):
+    return hasattr(page, 'smoothness')
+
+  def WillNavigateToPage(self, page, tab):
+    if not tab.browser.supports_tracing:
+      raise Exception('Not supported')
+    assert self.options.instrumentation_level in ALL_INSTRUMENTATION_LEVELS
+    if self.options.instrumentation_level == MINIMUM_INSTRUMENTATION_LEVEL:
+      categories = timeline_module.MINIMAL_TRACE_CATEGORIES
+    elif self.options.instrumentation_level == \
+        FIVE_PERCENT_OVERHEAD_INSTRUMENTATION_LEVEL:
+      categories = ''
+    else:
+      categories = '*,disabled-by-default-cc.debug'
+    tab.browser.StartTracing(categories)
+
+  def MeasurePage(self, page, tab, results):
+    """ Collect all possible metrics and added them to results. """
+    trace_result = tab.browser.StopTracing()
+    model = trace_result.AsTimelineModel()
+    renderer_thread = model.GetRendererThreadFromTab(tab)
+    meta_metrics = _TimelineBasedMetrics(model, renderer_thread)
+    meta_metrics.AddResults(results)