Cleanup of page_measurement_results object

tools/telemetry/telemetry/page/value.py

Index: tools/telemetry/telemetry/page/value.py
diff --git a/tools/telemetry/telemetry/page/value.py b/tools/telemetry/telemetry/page/value.py
new file mode 100644
index 0000000000000000000000000000000000000000..46d94d96c1349288b1a5e62eca26b3a4efda32ab
--- /dev/null
+++ b/tools/telemetry/telemetry/page/value.py
@@ -0,0 +1,505 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+The Value hierarchy provide a way of representing the values measurements
+produce such that they can be merged across runs, grouped by page, and output
+to different targets.
+
+The core Value concept provides the basic functionality:
+- association with a page, may be none
+- naming and units
+- importance tracking [whether a value will show up on a waterfall or output
+  file by default]
+- default conversion to scalar and string
+- merging properties
+
+A page may actually run a few times during a single telemetry session.
+Downstream consumers of test results typically want to group these runs
+together, then compute summary statistics across runs. Value provides the
+Merge* family of methods for this kind of aggregation.
+"""
+
+import json
+
+from telemetry.page import perf_tests_helper
+
+# When combining a pair of Values togehter, it is sometimes ambiguous whether
+# the values should be concatenated, or one should be picked as representative.
+# The possible merging policies are listed here.
+CONCATENATE = 'concatenate'
+PICK_FIRST = 'pick-first'
+
+# When converting a Value to its buildbot equivalent, the context in which the
+# value is being interpreted actually affects the conversion. This is insane,
+# but there you have it. There are three contexts in which Values are converted
+# for use by buildbot, represented by these output-intent values.
+PER_PAGE_RESULT_OUTPUT_CONTEXT = 'per-page-result-output-context'
+MERGED_PAGES_RESULT_OUTPUT_CONTEXT = 'merged-pages-result-output-context'
+SUMMARY_RESULT_OUTPUT_CONTEXT = 'summary-result-output-context'
+
+def _Mean(values):
+  return float(sum(values)) / len(values) if len(values) > 0 else 0.0
+
+class Value(object):
+  """An abstract value produced by a telemetry page test.
+  """
+  def __init__(self, page, name, units, important):
+    """A generic Value object.
+
+    Note: page may be given as None to indicate that the value represents
+    results multiple pages.
+    """
+    self.page = page
+    self.name = name
+    self.units = units
+    self.important = important
+
+  def IsMergableWith(self, that):
+    return (self.units == that.units and
+            type(self) == type(that) and
+            self.important == that.important)
+
+  @classmethod
+  def MergeLikeValuesFromSamePage(cls, values):
+    """Combines the provided list of values into a single compound value.
+
+    When a page runs multiple times, it may produce multiple values. This
+    function is given the same-named values across the multiple runs, and has
+    the responsibility of producing a single result.
+
+    It must return a single Value. If merging does not make sense, the
+    implementation must pick a representative value from one of the runs.
+
+    For instance, it may be given
+        [ScalarValue(page, 'a', 1), ScalarValue(page, 'a', 2)]
+    and it might produce
+        ListOfScalarValues(page, 'a', [1, 2])
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def MergeLikeValuesFromDifferentPages(cls, values):
+    """Combines the provided values into a single compound value.
+
+    When a full pageset runs, a single value_name will usually end up getting
+    collected for multiple pages. For instance, we may end up with
+       [ScalarValue(page1, 'a',  1),
+        ScalarValue(page2, 'a',  2)]
+
+    This function takes in the values of the same name, but across multiple
+    pages, and produces a single summary result value. In this instance, it
+    could produce a ScalarValue(None, 'a', 1.5) to indicate averaging, or even
+    ListOfScalarValues(None, 'a', [1, 2]) if concatenated output was desired.
+
+    Some results are so specific to a page that they make no sense when
+    aggregated across pages. If merging values of this type across pages is
+    non-sensical, this method may return None.
+    """
+    raise NotImplementedError()
+
+  def _IsImportantGivenOutputIntent(self, output_context):
+    if output_context == PER_PAGE_RESULT_OUTPUT_CONTEXT:
+      return False
+    elif output_context == MERGED_PAGES_RESULT_OUTPUT_CONTEXT:
+      return self.important
+    elif output_context == SUMMARY_RESULT_OUTPUT_CONTEXT:
+      return self.important
+
+  def GetBuildbotDataType(self, output_context):
+    """Returns the buildbot's equivalent data_type.
+
+    This should be one of the values accepted by perf_tests_results_helper.py.
+    """
+    raise NotImplementedError()
+
+  def GetBuildbotValue(self):
+    """Returns the buildbot's equivalent value."""
+    raise NotImplementedError()
+
+  def GetBuildbotMeasurementAndTraceNameForPerPageResult(self):
+    measurement, _ = (
+       _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name))
+    return measurement + '_by_url', self.page.display_name
+
+  def GetBuildbotMeasurementAndTraceNameForMergedPagesResult(self, trace_tag):
+    measurement, bb_trace_name = (
+        _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name))
+    if trace_tag:
+      return measurement, bb_trace_name + trace_tag
+    else:
+      return measurement, bb_trace_name
+
+  def GetRepresentativeNumberValue(self):
+    """Gets a single scalar value that best-represents this value.
+
+    Returns None if not possible.
+    """
+    raise NotImplementedError()
+
+  def GetRepresentativeStringValue(self):

[dtu, 2013/11/01 00:16:31]

nit: GetRepresentativeString, to avoid confusion with our new definition of
"Value." Same with GetRepresentativeNumber.

+    """Gets a string value that best-represents this value.
+
+    Returns None if not possible.
+    """
+    raise NotImplementedError()
+
+class ScalarValue(Value):
+  def __init__(self, page, name, units, value, important=True):
+    """A single value (float or integer) result from a test.
+
+    A test that counts the number of DOM elements in a page might produce a
+    scalar value:
+       ScalarValue(page, 'num_dom_elements', 'count', num_elements)
+    """
+    super(ScalarValue, self).__init__(page, name, units, important)
+    assert isinstance(value, int) or isinstance(value, float)
+    self.value = value
+
+  def __repr__(self):
+    if self.page:
+      page_name = self.page.url
+    else:
+      page_name = None
+    return 'ScalarValue(%s, %s, %s, %s, important=%s)' % (
+      page_name,
+      self.name, self.units,
+      self.value,
+      self.important)
+
+  def GetBuildbotDataType(self, output_context):
+    if self._IsImportantGivenOutputIntent(output_context):
+      return 'default'
+    return 'unimportant'
+
+  def GetBuildbotValue(self):
+    # Buildbot's print_perf_results method likes to get lists for all values,
+    # even when they are scalar, so list-ize the return value.
+    return [self.value]
+
+  def GetRepresentativeNumberValue(self):
+    return self.value
+
+  def GetRepresentativeStringValue(self):
+    return str(self.value)
+
+  @classmethod
+  def MergeLikeValuesFromSamePage(cls, values):
+    assert len(values) > 0
+    v0 = values[0]
+    return ListOfScalarValues(
+        v0.page, v0.name, v0.units,
+        [v.value for v in values],
+        important=v0.important)
+
+  @classmethod
+  def MergeLikeValuesFromDifferentPages(cls, values):
+    assert len(values) > 0
+    v0 = values[0]
+    return ListOfScalarValues(
+        None, v0.name, v0.units,
+        [v.value for v in values],
+        important=v0.important)
+
+
+class ListOfScalarValues(Value):
+  def __init__(self, page, name, units, values,
+               important=True, same_page_merge_policy=CONCATENATE):
+    super(ListOfScalarValues, self).__init__(page, name, units, important)
+    assert len(values) > 0
+    assert isinstance(values, list)
+    for v in values:
+      assert isinstance(v, int) or isinstance(v, float)
+    self.values = values
+    self.same_page_merge_policy = same_page_merge_policy
+
+  def __repr__(self):
+    if self.page:
+      page_name = self.page.url
+    else:
+      page_name = None
+    if self.same_page_merge_policy == CONCATENATE:
+      merge_policy = 'CONCATENATE'
+    else:
+      merge_policy = 'PICK_FIRST'
+    return ('ListOfScalarValues(%s, %s, %s, %s, ' +
+            'important=%s, same_page_merge_policy=%s)') % (
+              page_name,
+              self.name, self.units,
+              repr(self.values),
+              self.important,
+              merge_policy)
+
+  def GetBuildbotDataType(self, output_context):
+    if self._IsImportantGivenOutputIntent(output_context):
+      return 'default'
+    return 'unimportant'
+
+  def GetBuildbotValue(self):
+    return self.values
+
+  def GetRepresentativeNumberValue(self):
+    return _Mean(self.values)
+
+  def GetRepresentativeStringValue(self):
+    return repr(self.values)
+
+  def IsMergableWith(self, that):
+    return (super(ListOfScalarValues, self).IsMergableWith(that) and
+            self.same_page_merge_policy == that.same_page_merge_policy)
+
+  @classmethod
+  def MergeLikeValuesFromSamePage(cls, values):
+    assert len(values) > 0
+    v0 = values[0]
+
+    if v0.same_page_merge_policy == PICK_FIRST:
+      return ListOfScalarValues(
+          v0.page, v0.name, v0.units,
+          values[0].values,
+          important=v0.important,
+          same_page_merge_policy=v0.same_page_merge_policy)
+
+    assert v0.same_page_merge_policy == CONCATENATE
+    all_values = []
+    for v in values:
+      all_values.extend(v.values)
+    return ListOfScalarValues(
+        v0.page, v0.name, v0.units,
+        all_values,
+        important=v0.important,
+        same_page_merge_policy=v0.same_page_merge_policy)
+
+  @classmethod
+  def MergeLikeValuesFromDifferentPages(cls, values):
+    assert len(values) > 0
+    v0 = values[0]
+    all_values = []
+    for v in values:
+      all_values.extend(v.values)
+    return ListOfScalarValues(
+        None, v0.name, v0.units,
+        all_values,
+        important=v0.important,
+        same_page_merge_policy=v0.same_page_merge_policy)
+
+
+class HistogramValueBucket(object):
+  def __init__(self, low, high, count=0):
+    self.low = low
+    self.high = high
+    self.count = count
+
+  def ToJSONString(self):
+    return '{%s}' % ', '.join([
+      '"low": %i' % self.low,
+      '"high": %i' % self.high,
+      '"count": %i' % self.count])
+
+class HistogramValue(Value):
+  def __init__(self, page, name, units,
+               raw_value=None, raw_value_json=None, important=True):
+    super(HistogramValue, self).__init__(page, name, units, important)
+    if raw_value_json:
+      assert raw_value == None, 'Dont specify both raw_value and raw_value_json'
+      raw_value = json.loads(raw_value_json)
+    if raw_value:
+      assert 'buckets' in raw_value
+      assert isinstance(raw_value['buckets'], list)
+      self.buckets = []
+      for bucket in raw_value['buckets']:
+        self.buckets.append(HistogramValueBucket(
+          low=bucket['low'],
+          high=bucket['high'],
+          count=bucket['count']))
+    else:
+      self.buckets = []
+
+  def __repr__(self):
+    if self.page:
+      page_name = self.page.url
+    else:
+      page_name = None
+    return 'HistogramValue(%s, %s, %s, raw_json_string="%s", important=%s)' % (
+      page_name,
+      self.name, self.units,
+      self.ToJSONString(),
+      self.important)
+
+  def GetBuildbotDataType(self, output_context):
+    if self._IsImportantGivenOutputIntent(output_context):
+      return 'histogram'
+    return 'unimportant-histogram'
+
+  def GetBuildbotValue(self):
+    # More buildbot insanity: perf_tests_results_helper requires the histogram
+    # to be an array of size one.
+    return [self.ToJSONString()]
+
+  def ToJSONString(self):
+    # This has to hand-JSONify the histogram to ensure the order of keys
+    # produced is stable across different systems.
+    #
+    # Sigh, buildbot, Y U gotta be that way.
+    return '{"buckets": [%s]}' % (
+      ', '.join([b.ToJSONString() for b in self.buckets]))
+
+  def GetRepresentativeNumberValue(self):
+    (mean, _) = perf_tests_helper.GeomMeanAndStdDevFromHistogram(
+        self.ToJSONString())
+    return mean
+
+  def GetRepresentativeStringValue(self):
+    return self.GetBuildbotValue()
+
+  @classmethod
+  def MergeLikeValuesFromSamePage(cls, values):
+    assert len(values) > 0
+    v0 = values[0]
+    return HistogramValue(
+        v0.page, v0.name, v0.units,
+        raw_value_json=v0.ToJSONString(),
+        important=v0.important)
+
+  @classmethod
+  def MergeLikeValuesFromDifferentPages(cls, values):
+    # Histograms cannot be merged across pages, at least for now. It should be
+    # theoretically possible, just requires more work.
+    return None
+
+
+def MergeLikeValuesFromSamePage(all_values):
+  """Merges values that measure the same thing on the same page.
+
+  A page may end up being measured multiple times, meaning that we may end up
+  with something like this:
+       ScalarValue(page1, 'x', 1)
+       ScalarValue(page2, 'x', 4)
+       ScalarValue(page1, 'x', 2)
+       ScalarValue(page2, 'x', 5)
+
+  This function will produce:
+       ListOfScalarValues(page1, 'x', [1, 2])
+       ListOfScalarValues(page2, 'x', [4, 5])
+
+  The workhorse of this code is Value.MergeLikeValuesFromSamePage.
+
+  This requires (but assumes) that the values passed in with the same grouping
+  key pass the Value.IsMergableWith test. If this is not obeyed, the
+  results will be undefined.
+  """
+  return _MergeLikeValuesCommon(
+      all_values,
+      lambda x: (x.page, x.name),
+      lambda v0, merge_group: v0.MergeLikeValuesFromSamePage(merge_group))
+
+
+def MergeLikeValuesFromDifferentPages(all_values):
+  """Merges values that measure the same thing on different pages.
+
+  After using MergeLikeValuesFromSamePage, one still ends up with values from
+  different pages:
+       ScalarValue(page1, 'x', 1)
+       ScalarValue(page1, 'y', 30)
+       ScalarValue(page2, 'x', 2)
+       ScalarValue(page2, 'y', 40)
+
+  This function will group the values of the same value_name together:
+       ListOfScalarValues(None, 'x', [1, 2])
+       ListOfScalarValues(None, 'y', [30, 40])
+
+  The workhorse of this code is Value.MergeLikeValuesFromDifferentPages.
+
+  Not all values that go into this function will come out: not every value can
+  be merged across pages. Values whose MergeLikeValuesFromDifferentPages returns
+  None will be omitted from the results.
+
+  This requires (but assumes) that the values passed in with the same name pass
+  the Value.IsMergableWith test. If this is not obeyed, the results
+  will be undefined.
+  """
+  return _MergeLikeValuesCommon(
+      all_values,
+      lambda x: x.name,
+      lambda v0, merge_group: v0.MergeLikeValuesFromDifferentPages(merge_group))
+
+def _MergeLikeValuesCommon(all_values, key_func, merge_func):
+  """Groups all_values by key_func then applies merge_func to the groups.
+
+  This takes the all_values list and groups each item in that using the key
+  provided by key_func. This produces groups of values with like keys. Thes are
+  then handed to the merge_func to produce a new key. If merge_func produces a
+  non-None return, it is added to the list of returned values.
+  """
+  # When merging, we want to merge values in a consistent order, e.g. so that
+  # Scalar(1), Scalar(2) predictably produces ListOfScalarValues([1,2]) rather
+  # than 2,1.
+  #
+  # To do this, the values are sorted by key up front. Then, grouping is
+  # performed using a dictionary, but as new groups are found, the order in
+  # which they were found is also noted.
+  #
+  # Merging is then performed on groups in group-creation-order. This ensures
+  # that the returned array is in a stable order, group by group.
+  #
+  # Within a group, the order is stable because of the original sort.
+  all_values = list(all_values)
+  merge_groups = GroupStably(all_values, key_func)
+
+  res = []
+  for merge_group in merge_groups:
+    v0 = merge_group[0]
+    vM = merge_func(v0, merge_group)
+    if vM:
+      res.append(vM)
+  return res
+
+def GroupStably(all_values, key_func):
+  """Groups an array by key_func, with the groups returned in a stable order.
+
+  Returns a list of groups.
+  """
+  all_values = list(all_values)
+  all_values.sort(key=key_func)
+
+  merge_groups = {}
+  merge_groups_in_creation_order = []
+  for value in all_values:
+    key = key_func(value)
+    if key not in merge_groups:
+      merge_groups[key] = []
+      merge_groups_in_creation_order.append(merge_groups[key])
+    merge_groups[key].append(value)
+  return merge_groups_in_creation_order
+
+
+def ValueNameFromTraceAndChartName(trace_name, chart_name=None):
+  """Mangles a trace name plus optional chart name into a standard string.
+
+  A value might just be a bareword name, e.g. numPixels. In that case, its
+  chart may be None.
+
+  But, a value might also be intended for display with other values, in which
+  case the chart name indicates that grouping. So, you might have
+  screen.numPixels, screen.resolution, where chartName='screen'.
+  """
+  assert trace_name != 'url', 'The name url cannot be used'
+  if chart_name:
+    return '%s.%s' % (chart_name, trace_name)
+  else:
+    return trace_name
+
+def _ConvertValueNameToBuildbotMeasurementAndTraceName(value_name):
+  """Converts a value_name into the buildbot equivalent name pair.
+
+  Buildbot represents values by the measurement name and an optional trace name,
+  whereas telemetry represents values with a chart_name.trace_name convention,
+  where chart_name is optional.
+
+  This converts from the telemetry convention to the buildbot convention,
+  returning a 2-tuple (measurement_name, trace_name).
+  """
+  if '.' in value_name:
+    return value_name.split('.', 1)
+  else:
+    return value_name, value_name