Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(230)

Unified Diff: tools/telemetry/telemetry/page/value.py

Issue 27486002: Cleanup of page_measurement_results object (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: 500d Created 7 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/telemetry/page/value.py
diff --git a/tools/telemetry/telemetry/page/value.py b/tools/telemetry/telemetry/page/value.py
new file mode 100644
index 0000000000000000000000000000000000000000..46d94d96c1349288b1a5e62eca26b3a4efda32ab
--- /dev/null
+++ b/tools/telemetry/telemetry/page/value.py
@@ -0,0 +1,505 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+The Value hierarchy provide a way of representing the values measurements
+produce such that they can be merged across runs, grouped by page, and output
+to different targets.
+
+The core Value concept provides the basic functionality:
+- association with a page, may be none
+- naming and units
+- importance tracking [whether a value will show up on a waterfall or output
+ file by default]
+- default conversion to scalar and string
+- merging properties
+
+A page may actually run a few times during a single telemetry session.
+Downstream consumers of test results typically want to group these runs
+together, then compute summary statistics across runs. Value provides the
+Merge* family of methods for this kind of aggregation.
+"""
+
+import json
+
+from telemetry.page import perf_tests_helper
+
+# When combining a pair of Values togehter, it is sometimes ambiguous whether
+# the values should be concatenated, or one should be picked as representative.
+# The possible merging policies are listed here.
+CONCATENATE = 'concatenate'
+PICK_FIRST = 'pick-first'
+
+# When converting a Value to its buildbot equivalent, the context in which the
+# value is being interpreted actually affects the conversion. This is insane,
+# but there you have it. There are three contexts in which Values are converted
+# for use by buildbot, represented by these output-intent values.
+PER_PAGE_RESULT_OUTPUT_CONTEXT = 'per-page-result-output-context'
+MERGED_PAGES_RESULT_OUTPUT_CONTEXT = 'merged-pages-result-output-context'
+SUMMARY_RESULT_OUTPUT_CONTEXT = 'summary-result-output-context'
+
+def _Mean(values):
+ return float(sum(values)) / len(values) if len(values) > 0 else 0.0
+
+class Value(object):
+ """An abstract value produced by a telemetry page test.
+ """
+ def __init__(self, page, name, units, important):
+ """A generic Value object.
+
+ Note: page may be given as None to indicate that the value represents
+ results multiple pages.
+ """
+ self.page = page
+ self.name = name
+ self.units = units
+ self.important = important
+
+ def IsMergableWith(self, that):
+ return (self.units == that.units and
+ type(self) == type(that) and
+ self.important == that.important)
+
+ @classmethod
+ def MergeLikeValuesFromSamePage(cls, values):
+ """Combines the provided list of values into a single compound value.
+
+ When a page runs multiple times, it may produce multiple values. This
+ function is given the same-named values across the multiple runs, and has
+ the responsibility of producing a single result.
+
+ It must return a single Value. If merging does not make sense, the
+ implementation must pick a representative value from one of the runs.
+
+ For instance, it may be given
+ [ScalarValue(page, 'a', 1), ScalarValue(page, 'a', 2)]
+ and it might produce
+ ListOfScalarValues(page, 'a', [1, 2])
+ """
+ raise NotImplementedError()
+
+ @classmethod
+ def MergeLikeValuesFromDifferentPages(cls, values):
+ """Combines the provided values into a single compound value.
+
+ When a full pageset runs, a single value_name will usually end up getting
+ collected for multiple pages. For instance, we may end up with
+ [ScalarValue(page1, 'a', 1),
+ ScalarValue(page2, 'a', 2)]
+
+ This function takes in the values of the same name, but across multiple
+ pages, and produces a single summary result value. In this instance, it
+ could produce a ScalarValue(None, 'a', 1.5) to indicate averaging, or even
+ ListOfScalarValues(None, 'a', [1, 2]) if concatenated output was desired.
+
+ Some results are so specific to a page that they make no sense when
+ aggregated across pages. If merging values of this type across pages is
+ non-sensical, this method may return None.
+ """
+ raise NotImplementedError()
+
+ def _IsImportantGivenOutputIntent(self, output_context):
+ if output_context == PER_PAGE_RESULT_OUTPUT_CONTEXT:
+ return False
+ elif output_context == MERGED_PAGES_RESULT_OUTPUT_CONTEXT:
+ return self.important
+ elif output_context == SUMMARY_RESULT_OUTPUT_CONTEXT:
+ return self.important
+
+ def GetBuildbotDataType(self, output_context):
+ """Returns the buildbot's equivalent data_type.
+
+ This should be one of the values accepted by perf_tests_results_helper.py.
+ """
+ raise NotImplementedError()
+
+ def GetBuildbotValue(self):
+ """Returns the buildbot's equivalent value."""
+ raise NotImplementedError()
+
+ def GetBuildbotMeasurementAndTraceNameForPerPageResult(self):
+ measurement, _ = (
+ _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name))
+ return measurement + '_by_url', self.page.display_name
+
+ def GetBuildbotMeasurementAndTraceNameForMergedPagesResult(self, trace_tag):
+ measurement, bb_trace_name = (
+ _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name))
+ if trace_tag:
+ return measurement, bb_trace_name + trace_tag
+ else:
+ return measurement, bb_trace_name
+
+ def GetRepresentativeNumberValue(self):
+ """Gets a single scalar value that best-represents this value.
+
+ Returns None if not possible.
+ """
+ raise NotImplementedError()
+
+ def GetRepresentativeStringValue(self):
dtu 2013/11/01 00:16:31 nit: GetRepresentativeString, to avoid confusion w
+ """Gets a string value that best-represents this value.
+
+ Returns None if not possible.
+ """
+ raise NotImplementedError()
+
+class ScalarValue(Value):
+ def __init__(self, page, name, units, value, important=True):
+ """A single value (float or integer) result from a test.
+
+ A test that counts the number of DOM elements in a page might produce a
+ scalar value:
+ ScalarValue(page, 'num_dom_elements', 'count', num_elements)
+ """
+ super(ScalarValue, self).__init__(page, name, units, important)
+ assert isinstance(value, int) or isinstance(value, float)
+ self.value = value
+
+ def __repr__(self):
+ if self.page:
+ page_name = self.page.url
+ else:
+ page_name = None
+ return 'ScalarValue(%s, %s, %s, %s, important=%s)' % (
+ page_name,
+ self.name, self.units,
+ self.value,
+ self.important)
+
+ def GetBuildbotDataType(self, output_context):
+ if self._IsImportantGivenOutputIntent(output_context):
+ return 'default'
+ return 'unimportant'
+
+ def GetBuildbotValue(self):
+ # Buildbot's print_perf_results method likes to get lists for all values,
+ # even when they are scalar, so list-ize the return value.
+ return [self.value]
+
+ def GetRepresentativeNumberValue(self):
+ return self.value
+
+ def GetRepresentativeStringValue(self):
+ return str(self.value)
+
+ @classmethod
+ def MergeLikeValuesFromSamePage(cls, values):
+ assert len(values) > 0
+ v0 = values[0]
+ return ListOfScalarValues(
+ v0.page, v0.name, v0.units,
+ [v.value for v in values],
+ important=v0.important)
+
+ @classmethod
+ def MergeLikeValuesFromDifferentPages(cls, values):
+ assert len(values) > 0
+ v0 = values[0]
+ return ListOfScalarValues(
+ None, v0.name, v0.units,
+ [v.value for v in values],
+ important=v0.important)
+
+
+class ListOfScalarValues(Value):
+ def __init__(self, page, name, units, values,
+ important=True, same_page_merge_policy=CONCATENATE):
+ super(ListOfScalarValues, self).__init__(page, name, units, important)
+ assert len(values) > 0
+ assert isinstance(values, list)
+ for v in values:
+ assert isinstance(v, int) or isinstance(v, float)
+ self.values = values
+ self.same_page_merge_policy = same_page_merge_policy
+
+ def __repr__(self):
+ if self.page:
+ page_name = self.page.url
+ else:
+ page_name = None
+ if self.same_page_merge_policy == CONCATENATE:
+ merge_policy = 'CONCATENATE'
+ else:
+ merge_policy = 'PICK_FIRST'
+ return ('ListOfScalarValues(%s, %s, %s, %s, ' +
+ 'important=%s, same_page_merge_policy=%s)') % (
+ page_name,
+ self.name, self.units,
+ repr(self.values),
+ self.important,
+ merge_policy)
+
+ def GetBuildbotDataType(self, output_context):
+ if self._IsImportantGivenOutputIntent(output_context):
+ return 'default'
+ return 'unimportant'
+
+ def GetBuildbotValue(self):
+ return self.values
+
+ def GetRepresentativeNumberValue(self):
+ return _Mean(self.values)
+
+ def GetRepresentativeStringValue(self):
+ return repr(self.values)
+
+ def IsMergableWith(self, that):
+ return (super(ListOfScalarValues, self).IsMergableWith(that) and
+ self.same_page_merge_policy == that.same_page_merge_policy)
+
+ @classmethod
+ def MergeLikeValuesFromSamePage(cls, values):
+ assert len(values) > 0
+ v0 = values[0]
+
+ if v0.same_page_merge_policy == PICK_FIRST:
+ return ListOfScalarValues(
+ v0.page, v0.name, v0.units,
+ values[0].values,
+ important=v0.important,
+ same_page_merge_policy=v0.same_page_merge_policy)
+
+ assert v0.same_page_merge_policy == CONCATENATE
+ all_values = []
+ for v in values:
+ all_values.extend(v.values)
+ return ListOfScalarValues(
+ v0.page, v0.name, v0.units,
+ all_values,
+ important=v0.important,
+ same_page_merge_policy=v0.same_page_merge_policy)
+
+ @classmethod
+ def MergeLikeValuesFromDifferentPages(cls, values):
+ assert len(values) > 0
+ v0 = values[0]
+ all_values = []
+ for v in values:
+ all_values.extend(v.values)
+ return ListOfScalarValues(
+ None, v0.name, v0.units,
+ all_values,
+ important=v0.important,
+ same_page_merge_policy=v0.same_page_merge_policy)
+
+
+class HistogramValueBucket(object):
+ def __init__(self, low, high, count=0):
+ self.low = low
+ self.high = high
+ self.count = count
+
+ def ToJSONString(self):
+ return '{%s}' % ', '.join([
+ '"low": %i' % self.low,
+ '"high": %i' % self.high,
+ '"count": %i' % self.count])
+
+class HistogramValue(Value):
+ def __init__(self, page, name, units,
+ raw_value=None, raw_value_json=None, important=True):
+ super(HistogramValue, self).__init__(page, name, units, important)
+ if raw_value_json:
+ assert raw_value == None, 'Dont specify both raw_value and raw_value_json'
+ raw_value = json.loads(raw_value_json)
+ if raw_value:
+ assert 'buckets' in raw_value
+ assert isinstance(raw_value['buckets'], list)
+ self.buckets = []
+ for bucket in raw_value['buckets']:
+ self.buckets.append(HistogramValueBucket(
+ low=bucket['low'],
+ high=bucket['high'],
+ count=bucket['count']))
+ else:
+ self.buckets = []
+
+ def __repr__(self):
+ if self.page:
+ page_name = self.page.url
+ else:
+ page_name = None
+ return 'HistogramValue(%s, %s, %s, raw_json_string="%s", important=%s)' % (
+ page_name,
+ self.name, self.units,
+ self.ToJSONString(),
+ self.important)
+
+ def GetBuildbotDataType(self, output_context):
+ if self._IsImportantGivenOutputIntent(output_context):
+ return 'histogram'
+ return 'unimportant-histogram'
+
+ def GetBuildbotValue(self):
+ # More buildbot insanity: perf_tests_results_helper requires the histogram
+ # to be an array of size one.
+ return [self.ToJSONString()]
+
+ def ToJSONString(self):
+ # This has to hand-JSONify the histogram to ensure the order of keys
+ # produced is stable across different systems.
+ #
+ # Sigh, buildbot, Y U gotta be that way.
+ return '{"buckets": [%s]}' % (
+ ', '.join([b.ToJSONString() for b in self.buckets]))
+
+ def GetRepresentativeNumberValue(self):
+ (mean, _) = perf_tests_helper.GeomMeanAndStdDevFromHistogram(
+ self.ToJSONString())
+ return mean
+
+ def GetRepresentativeStringValue(self):
+ return self.GetBuildbotValue()
+
+ @classmethod
+ def MergeLikeValuesFromSamePage(cls, values):
+ assert len(values) > 0
+ v0 = values[0]
+ return HistogramValue(
+ v0.page, v0.name, v0.units,
+ raw_value_json=v0.ToJSONString(),
+ important=v0.important)
+
+ @classmethod
+ def MergeLikeValuesFromDifferentPages(cls, values):
+ # Histograms cannot be merged across pages, at least for now. It should be
+ # theoretically possible, just requires more work.
+ return None
+
+
+def MergeLikeValuesFromSamePage(all_values):
+ """Merges values that measure the same thing on the same page.
+
+ A page may end up being measured multiple times, meaning that we may end up
+ with something like this:
+ ScalarValue(page1, 'x', 1)
+ ScalarValue(page2, 'x', 4)
+ ScalarValue(page1, 'x', 2)
+ ScalarValue(page2, 'x', 5)
+
+ This function will produce:
+ ListOfScalarValues(page1, 'x', [1, 2])
+ ListOfScalarValues(page2, 'x', [4, 5])
+
+ The workhorse of this code is Value.MergeLikeValuesFromSamePage.
+
+ This requires (but assumes) that the values passed in with the same grouping
+ key pass the Value.IsMergableWith test. If this is not obeyed, the
+ results will be undefined.
+ """
+ return _MergeLikeValuesCommon(
+ all_values,
+ lambda x: (x.page, x.name),
+ lambda v0, merge_group: v0.MergeLikeValuesFromSamePage(merge_group))
+
+
+def MergeLikeValuesFromDifferentPages(all_values):
+ """Merges values that measure the same thing on different pages.
+
+ After using MergeLikeValuesFromSamePage, one still ends up with values from
+ different pages:
+ ScalarValue(page1, 'x', 1)
+ ScalarValue(page1, 'y', 30)
+ ScalarValue(page2, 'x', 2)
+ ScalarValue(page2, 'y', 40)
+
+ This function will group the values of the same value_name together:
+ ListOfScalarValues(None, 'x', [1, 2])
+ ListOfScalarValues(None, 'y', [30, 40])
+
+ The workhorse of this code is Value.MergeLikeValuesFromDifferentPages.
+
+ Not all values that go into this function will come out: not every value can
+ be merged across pages. Values whose MergeLikeValuesFromDifferentPages returns
+ None will be omitted from the results.
+
+ This requires (but assumes) that the values passed in with the same name pass
+ the Value.IsMergableWith test. If this is not obeyed, the results
+ will be undefined.
+ """
+ return _MergeLikeValuesCommon(
+ all_values,
+ lambda x: x.name,
+ lambda v0, merge_group: v0.MergeLikeValuesFromDifferentPages(merge_group))
+
+def _MergeLikeValuesCommon(all_values, key_func, merge_func):
+ """Groups all_values by key_func then applies merge_func to the groups.
+
+ This takes the all_values list and groups each item in that using the key
+ provided by key_func. This produces groups of values with like keys. Thes are
+ then handed to the merge_func to produce a new key. If merge_func produces a
+ non-None return, it is added to the list of returned values.
+ """
+ # When merging, we want to merge values in a consistent order, e.g. so that
+ # Scalar(1), Scalar(2) predictably produces ListOfScalarValues([1,2]) rather
+ # than 2,1.
+ #
+ # To do this, the values are sorted by key up front. Then, grouping is
+ # performed using a dictionary, but as new groups are found, the order in
+ # which they were found is also noted.
+ #
+ # Merging is then performed on groups in group-creation-order. This ensures
+ # that the returned array is in a stable order, group by group.
+ #
+ # Within a group, the order is stable because of the original sort.
+ all_values = list(all_values)
+ merge_groups = GroupStably(all_values, key_func)
+
+ res = []
+ for merge_group in merge_groups:
+ v0 = merge_group[0]
+ vM = merge_func(v0, merge_group)
+ if vM:
+ res.append(vM)
+ return res
+
+def GroupStably(all_values, key_func):
+ """Groups an array by key_func, with the groups returned in a stable order.
+
+ Returns a list of groups.
+ """
+ all_values = list(all_values)
+ all_values.sort(key=key_func)
+
+ merge_groups = {}
+ merge_groups_in_creation_order = []
+ for value in all_values:
+ key = key_func(value)
+ if key not in merge_groups:
+ merge_groups[key] = []
+ merge_groups_in_creation_order.append(merge_groups[key])
+ merge_groups[key].append(value)
+ return merge_groups_in_creation_order
+
+
+def ValueNameFromTraceAndChartName(trace_name, chart_name=None):
+ """Mangles a trace name plus optional chart name into a standard string.
+
+ A value might just be a bareword name, e.g. numPixels. In that case, its
+ chart may be None.
+
+ But, a value might also be intended for display with other values, in which
+ case the chart name indicates that grouping. So, you might have
+ screen.numPixels, screen.resolution, where chartName='screen'.
+ """
+ assert trace_name != 'url', 'The name url cannot be used'
+ if chart_name:
+ return '%s.%s' % (chart_name, trace_name)
+ else:
+ return trace_name
+
+def _ConvertValueNameToBuildbotMeasurementAndTraceName(value_name):
+ """Converts a value_name into the buildbot equivalent name pair.
+
+ Buildbot represents values by the measurement name and an optional trace name,
+ whereas telemetry represents values with a chart_name.trace_name convention,
+ where chart_name is optional.
+
+ This converts from the telemetry convention to the buildbot convention,
+ returning a 2-tuple (measurement_name, trace_name).
+ """
+ if '.' in value_name:
+ return value_name.split('.', 1)
+ else:
+ return value_name, value_name

Powered by Google App Engine
This is Rietveld 408576698