Set up a parallel memory.memory_health_quick_tbmv2 benchmark

tools/perf/benchmarks/memory_infra.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 
 from core import perf_benchmark
 
 from telemetry import benchmark
 from telemetry.timeline import tracing_category_filter
@@ skipping 11 matching lines @@
   is part of chrome tracing, and extracts it using timeline-based measurements.
   """
 
   def SetExtraBrowserOptions(self, options):
     options.AppendExtraBrowserArgs([
         # TODO(perezju): Temporary workaround to disable periodic memory dumps.
         # See: http://crbug.com/513692
         '--enable-memory-benchmarking',
     ])
 
+
+class _TBMv1MemoryInfra(_MemoryInfra):

[perezju, 2016/04/25 09:03:49]

As per the discussion in
https://github.com/catapult-project/catapult/issues/2137#issuecomment-197954711
we're planning to get rid of these "TraceRerun" methods; so let's take the
chance to do a bit of clean up here.

I would suggest.
- Get rid of HasTraceRerunDebugOption.
- Get rid of SetupBenchmarkDefaultTraceRerunOptions, its code can be moved to
the bottom of CreateTimelineBasedMeasurementOptions.

After doing this, the change between v1 and v2 is so small (just calling
SetLegacyTimelineBasedMetrics vs SeTimelineBasedMetrics), that  probably isn't
worth anymore to have two different classes.

Here is a suggestion:
- Just add a class variable, something like 'USE_LEGACY_TBM' that defaults to
True; the class uses this to decide which tbm setting method to call.
- Then TBMv2MemoryHealthQuick can just subclass MemoryHealthQuick, flip the
value of that variable, and adjust the name. This also avoids the awkward bit
below where one of the classes appears to be "hijacking" methods from the other;
now it just would be "super".

[petrcermak, 2016/04/25 10:00:35]

Done.

[perezju, 2016/04/25 11:24:00]

Awesome! Looks much better.

+  """Base class for memory benchmarks based on TBMv1 memory-infra.
+
+  See
+  third_party/catapult/telemetry/telemetry/web_perf/metrics/memory_timeline.py
+  for more details about the Timeline Based Measurement v1 memory metric.
+  """
+
   def CreateTimelineBasedMeasurementOptions(self):
     # Enable only memory-infra, to get memory dumps, and blink.console, to get
     # the timeline markers used for mapping threads to tabs.
     trace_memory = tracing_category_filter.TracingCategoryFilter(
         filter_string='-*,blink.console,disabled-by-default-memory-infra')
     tbm_options = timeline_based_measurement.Options(
         overhead_level=trace_memory)
     tbm_options.config.enable_android_graphics_memtrack = True
     return tbm_options
 
   @classmethod
   def HasTraceRerunDebugOption(cls):
     return True
 
   def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
     tbm_options.SetLegacyTimelineBasedMetrics((
         memory_timeline.MemoryTimelineMetric(),
     ))
 
 
+class _TBMv2MemoryInfra(_MemoryInfra):
+  """Base class for memory benchmarks based on TBMv2 memory-infra.
+
+  See
+  third_party/catapult/tracing/tracing/metrics/system_health/memory_metric.html
+  for more details about the Timeline Based Measurement v2 memory metric.
+  """
+
+  def CreateTimelineBasedMeasurementOptions(self):
+    tbm_options = super(
+        _TBMv2MemoryInfra, self).CreateTimelineBasedMeasurementOptions()
+    tbm_options.SetTimelineBasedMetric('memoryMetric')
+    return tbm_options
+
+
 # TODO(bashi): Workaround for http://crbug.com/532075
 # @benchmark.Enabled('android') shouldn't be needed.
 @benchmark.Enabled('android')
-class MemoryHealthQuick(_MemoryInfra):
+class MemoryHealthQuick(_TBMv1MemoryInfra):
   """Timeline based benchmark for the Memory Health Plan (1 iteration)."""
   page_set = page_sets.MemoryHealthStory
 
   @classmethod
   def Name(cls):
     return 'memory.memory_health_quick'
 
   @classmethod
   def ShouldDisable(cls, possible_browser):
     # Benchmark requires DeskClock app only available on Nexus devices.
     # See http://crbug.com/546842
     return 'nexus' not in possible_browser.platform.GetDeviceTypeName().lower()
 
 
 # Benchmark is disabled by default because it takes too long to run.
 @benchmark.Disabled('all')
 class MemoryHealthPlan(MemoryHealthQuick):
   """Timeline based benchmark for the Memory Health Plan (5 iterations)."""
   options = {'pageset_repeat': 5}
 
   @classmethod
   def Name(cls):
     return 'memory.memory_health_plan'
 
 
+@benchmark.Enabled('android')
+class TBMv2MemoryHealthQuick(_TBMv2MemoryInfra):
+  """Timeline based benchmark for the Memory Health Plan based on TBMv2.
+
+  This is a temporary benchmark to compare the new TBMv2 memory metric
+  (memory_metric.html) with the existing TBMv1 one (memory_timeline.py). Once
+  all issues associated with the TBMv2 metric are resolved, all memory
+  benchmarks (including the ones in this file) will switch to use it instead
+  of the TBMv1 metric and this temporary benchmark will be removed.
+  """
+  page_set = MemoryHealthQuick.page_set
+
+  @classmethod
+  def Name(cls):
+    return MemoryHealthQuick.Name() + '_tbmv2'
+
+  @classmethod
+  def ShouldDisable(cls, possible_browser):
+    return MemoryHealthQuick.ShouldDisable(possible_browser)
+
+
 # TODO(bashi): Workaround for http://crbug.com/532075
 # @benchmark.Enabled('android') shouldn't be needed.
 @benchmark.Enabled('android')
-class RendererMemoryBlinkMemoryMobile(_MemoryInfra):
+class RendererMemoryBlinkMemoryMobile(_TBMv1MemoryInfra):
   """Timeline based benchmark for measuring memory consumption on mobile
   sites on which blink's memory consumption is relatively high."""
 
   _RE_RENDERER_VALUES = re.compile('memory_.+_renderer')
 
   page_set = page_sets.BlinkMemoryMobilePageSet
 
   def SetExtraBrowserOptions(self, options):
     super(RendererMemoryBlinkMemoryMobile, self).SetExtraBrowserOptions(
         options)
     options.AppendExtraBrowserArgs([
         # Ignore certs errors because record_wpr cannot handle certs correctly
         # in some cases (e.g. WordPress).
         '--ignore-certificate-errors',
     ])
 
   @classmethod
   def Name(cls):
     return 'memory.blink_memory_mobile'
 
   @classmethod
   def ValueCanBeAddedPredicate(cls, value, is_first_result):
     return bool(cls._RE_RENDERER_VALUES.match(value.name))
 
 
 # Disabled on reference builds because they don't support the new
 # Tracing.requestMemoryDump DevTools API. See http://crbug.com/540022.
 @benchmark.Disabled('reference')
-class MemoryBenchmarkTop10Mobile(_MemoryInfra):
+class MemoryBenchmarkTop10Mobile(_TBMv1MemoryInfra):
   """Timeline based benchmark for measuring memory on top 10 mobile sites."""
 
   page_set = page_sets.MemoryInfraTop10MobilePageSet
 
   @classmethod
   def Name(cls):
     return 'memory.top_10_mobile'
 
 
 # Disabled on reference builds because they don't support the new
 # Tracing.requestMemoryDump DevTools API.
 # For 'reference' see http://crbug.com/540022.
 # For 'android' see http://crbug.com/579546.
 @benchmark.Disabled('reference', 'android')
-class MemoryLongRunningIdleGmailTBM(_MemoryInfra):
+class MemoryLongRunningIdleGmailTBM(_TBMv1MemoryInfra):
   """Use (recorded) real world web sites and measure memory consumption
   of long running idle Gmail page """
   page_set = page_sets.LongRunningIdleGmailPageSet
 
   def CreateTimelineBasedMeasurementOptions(self):
     v8_categories = [
         'blink.console', 'renderer.scheduler', 'v8', 'webkit.console']
     memory_categories = 'blink.console,disabled-by-default-memory-infra'
     category_filter = tracing_category_filter.TracingCategoryFilter(
         memory_categories)
     for category in v8_categories:
       category_filter.AddIncludedCategory(category)
     options = timeline_based_measurement.Options(category_filter)
     return options
 
   def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
     tbm_options.SetLegacyTimelineBasedMetrics((
         v8_gc_latency.V8GCLatency(),
         memory_timeline.MemoryTimelineMetric(),
     ))
 
   @classmethod
   def Name(cls):
     return 'memory.long_running_idle_gmail_tbm'
 
   @classmethod
   def ShouldTearDownStateAfterEachStoryRun(cls):
     return True