Index: tools/telemetry/telemetry/page/value.py |
diff --git a/tools/telemetry/telemetry/page/value.py b/tools/telemetry/telemetry/page/value.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..46d94d96c1349288b1a5e62eca26b3a4efda32ab |
--- /dev/null |
+++ b/tools/telemetry/telemetry/page/value.py |
@@ -0,0 +1,505 @@ |
+# Copyright (c) 2013 The Chromium Authors. All rights reserved. |
+# Use of this source code is governed by a BSD-style license that can be |
+# found in the LICENSE file. |
+""" |
+The Value hierarchy provide a way of representing the values measurements |
+produce such that they can be merged across runs, grouped by page, and output |
+to different targets. |
+ |
+The core Value concept provides the basic functionality: |
+- association with a page, may be none |
+- naming and units |
+- importance tracking [whether a value will show up on a waterfall or output |
+ file by default] |
+- default conversion to scalar and string |
+- merging properties |
+ |
+A page may actually run a few times during a single telemetry session. |
+Downstream consumers of test results typically want to group these runs |
+together, then compute summary statistics across runs. Value provides the |
+Merge* family of methods for this kind of aggregation. |
+""" |
+ |
+import json |
+ |
+from telemetry.page import perf_tests_helper |
+ |
+# When combining a pair of Values togehter, it is sometimes ambiguous whether |
+# the values should be concatenated, or one should be picked as representative. |
+# The possible merging policies are listed here. |
+CONCATENATE = 'concatenate' |
+PICK_FIRST = 'pick-first' |
+ |
+# When converting a Value to its buildbot equivalent, the context in which the |
+# value is being interpreted actually affects the conversion. This is insane, |
+# but there you have it. There are three contexts in which Values are converted |
+# for use by buildbot, represented by these output-intent values. |
+PER_PAGE_RESULT_OUTPUT_CONTEXT = 'per-page-result-output-context' |
+MERGED_PAGES_RESULT_OUTPUT_CONTEXT = 'merged-pages-result-output-context' |
+SUMMARY_RESULT_OUTPUT_CONTEXT = 'summary-result-output-context' |
+ |
+def _Mean(values): |
+ return float(sum(values)) / len(values) if len(values) > 0 else 0.0 |
+ |
+class Value(object): |
+ """An abstract value produced by a telemetry page test. |
+ """ |
+ def __init__(self, page, name, units, important): |
+ """A generic Value object. |
+ |
+ Note: page may be given as None to indicate that the value represents |
+ results multiple pages. |
+ """ |
+ self.page = page |
+ self.name = name |
+ self.units = units |
+ self.important = important |
+ |
+ def IsMergableWith(self, that): |
+ return (self.units == that.units and |
+ type(self) == type(that) and |
+ self.important == that.important) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromSamePage(cls, values): |
+ """Combines the provided list of values into a single compound value. |
+ |
+ When a page runs multiple times, it may produce multiple values. This |
+ function is given the same-named values across the multiple runs, and has |
+ the responsibility of producing a single result. |
+ |
+ It must return a single Value. If merging does not make sense, the |
+ implementation must pick a representative value from one of the runs. |
+ |
+ For instance, it may be given |
+ [ScalarValue(page, 'a', 1), ScalarValue(page, 'a', 2)] |
+ and it might produce |
+ ListOfScalarValues(page, 'a', [1, 2]) |
+ """ |
+ raise NotImplementedError() |
+ |
+ @classmethod |
+ def MergeLikeValuesFromDifferentPages(cls, values): |
+ """Combines the provided values into a single compound value. |
+ |
+ When a full pageset runs, a single value_name will usually end up getting |
+ collected for multiple pages. For instance, we may end up with |
+ [ScalarValue(page1, 'a', 1), |
+ ScalarValue(page2, 'a', 2)] |
+ |
+ This function takes in the values of the same name, but across multiple |
+ pages, and produces a single summary result value. In this instance, it |
+ could produce a ScalarValue(None, 'a', 1.5) to indicate averaging, or even |
+ ListOfScalarValues(None, 'a', [1, 2]) if concatenated output was desired. |
+ |
+ Some results are so specific to a page that they make no sense when |
+ aggregated across pages. If merging values of this type across pages is |
+ non-sensical, this method may return None. |
+ """ |
+ raise NotImplementedError() |
+ |
+ def _IsImportantGivenOutputIntent(self, output_context): |
+ if output_context == PER_PAGE_RESULT_OUTPUT_CONTEXT: |
+ return False |
+ elif output_context == MERGED_PAGES_RESULT_OUTPUT_CONTEXT: |
+ return self.important |
+ elif output_context == SUMMARY_RESULT_OUTPUT_CONTEXT: |
+ return self.important |
+ |
+ def GetBuildbotDataType(self, output_context): |
+ """Returns the buildbot's equivalent data_type. |
+ |
+ This should be one of the values accepted by perf_tests_results_helper.py. |
+ """ |
+ raise NotImplementedError() |
+ |
+ def GetBuildbotValue(self): |
+ """Returns the buildbot's equivalent value.""" |
+ raise NotImplementedError() |
+ |
+ def GetBuildbotMeasurementAndTraceNameForPerPageResult(self): |
+ measurement, _ = ( |
+ _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name)) |
+ return measurement + '_by_url', self.page.display_name |
+ |
+ def GetBuildbotMeasurementAndTraceNameForMergedPagesResult(self, trace_tag): |
+ measurement, bb_trace_name = ( |
+ _ConvertValueNameToBuildbotMeasurementAndTraceName(self.name)) |
+ if trace_tag: |
+ return measurement, bb_trace_name + trace_tag |
+ else: |
+ return measurement, bb_trace_name |
+ |
+ def GetRepresentativeNumberValue(self): |
+ """Gets a single scalar value that best-represents this value. |
+ |
+ Returns None if not possible. |
+ """ |
+ raise NotImplementedError() |
+ |
+ def GetRepresentativeStringValue(self): |
dtu
2013/11/01 00:16:31
nit: GetRepresentativeString, to avoid confusion w
|
+ """Gets a string value that best-represents this value. |
+ |
+ Returns None if not possible. |
+ """ |
+ raise NotImplementedError() |
+ |
+class ScalarValue(Value): |
+ def __init__(self, page, name, units, value, important=True): |
+ """A single value (float or integer) result from a test. |
+ |
+ A test that counts the number of DOM elements in a page might produce a |
+ scalar value: |
+ ScalarValue(page, 'num_dom_elements', 'count', num_elements) |
+ """ |
+ super(ScalarValue, self).__init__(page, name, units, important) |
+ assert isinstance(value, int) or isinstance(value, float) |
+ self.value = value |
+ |
+ def __repr__(self): |
+ if self.page: |
+ page_name = self.page.url |
+ else: |
+ page_name = None |
+ return 'ScalarValue(%s, %s, %s, %s, important=%s)' % ( |
+ page_name, |
+ self.name, self.units, |
+ self.value, |
+ self.important) |
+ |
+ def GetBuildbotDataType(self, output_context): |
+ if self._IsImportantGivenOutputIntent(output_context): |
+ return 'default' |
+ return 'unimportant' |
+ |
+ def GetBuildbotValue(self): |
+ # Buildbot's print_perf_results method likes to get lists for all values, |
+ # even when they are scalar, so list-ize the return value. |
+ return [self.value] |
+ |
+ def GetRepresentativeNumberValue(self): |
+ return self.value |
+ |
+ def GetRepresentativeStringValue(self): |
+ return str(self.value) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromSamePage(cls, values): |
+ assert len(values) > 0 |
+ v0 = values[0] |
+ return ListOfScalarValues( |
+ v0.page, v0.name, v0.units, |
+ [v.value for v in values], |
+ important=v0.important) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromDifferentPages(cls, values): |
+ assert len(values) > 0 |
+ v0 = values[0] |
+ return ListOfScalarValues( |
+ None, v0.name, v0.units, |
+ [v.value for v in values], |
+ important=v0.important) |
+ |
+ |
+class ListOfScalarValues(Value): |
+ def __init__(self, page, name, units, values, |
+ important=True, same_page_merge_policy=CONCATENATE): |
+ super(ListOfScalarValues, self).__init__(page, name, units, important) |
+ assert len(values) > 0 |
+ assert isinstance(values, list) |
+ for v in values: |
+ assert isinstance(v, int) or isinstance(v, float) |
+ self.values = values |
+ self.same_page_merge_policy = same_page_merge_policy |
+ |
+ def __repr__(self): |
+ if self.page: |
+ page_name = self.page.url |
+ else: |
+ page_name = None |
+ if self.same_page_merge_policy == CONCATENATE: |
+ merge_policy = 'CONCATENATE' |
+ else: |
+ merge_policy = 'PICK_FIRST' |
+ return ('ListOfScalarValues(%s, %s, %s, %s, ' + |
+ 'important=%s, same_page_merge_policy=%s)') % ( |
+ page_name, |
+ self.name, self.units, |
+ repr(self.values), |
+ self.important, |
+ merge_policy) |
+ |
+ def GetBuildbotDataType(self, output_context): |
+ if self._IsImportantGivenOutputIntent(output_context): |
+ return 'default' |
+ return 'unimportant' |
+ |
+ def GetBuildbotValue(self): |
+ return self.values |
+ |
+ def GetRepresentativeNumberValue(self): |
+ return _Mean(self.values) |
+ |
+ def GetRepresentativeStringValue(self): |
+ return repr(self.values) |
+ |
+ def IsMergableWith(self, that): |
+ return (super(ListOfScalarValues, self).IsMergableWith(that) and |
+ self.same_page_merge_policy == that.same_page_merge_policy) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromSamePage(cls, values): |
+ assert len(values) > 0 |
+ v0 = values[0] |
+ |
+ if v0.same_page_merge_policy == PICK_FIRST: |
+ return ListOfScalarValues( |
+ v0.page, v0.name, v0.units, |
+ values[0].values, |
+ important=v0.important, |
+ same_page_merge_policy=v0.same_page_merge_policy) |
+ |
+ assert v0.same_page_merge_policy == CONCATENATE |
+ all_values = [] |
+ for v in values: |
+ all_values.extend(v.values) |
+ return ListOfScalarValues( |
+ v0.page, v0.name, v0.units, |
+ all_values, |
+ important=v0.important, |
+ same_page_merge_policy=v0.same_page_merge_policy) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromDifferentPages(cls, values): |
+ assert len(values) > 0 |
+ v0 = values[0] |
+ all_values = [] |
+ for v in values: |
+ all_values.extend(v.values) |
+ return ListOfScalarValues( |
+ None, v0.name, v0.units, |
+ all_values, |
+ important=v0.important, |
+ same_page_merge_policy=v0.same_page_merge_policy) |
+ |
+ |
+class HistogramValueBucket(object): |
+ def __init__(self, low, high, count=0): |
+ self.low = low |
+ self.high = high |
+ self.count = count |
+ |
+ def ToJSONString(self): |
+ return '{%s}' % ', '.join([ |
+ '"low": %i' % self.low, |
+ '"high": %i' % self.high, |
+ '"count": %i' % self.count]) |
+ |
+class HistogramValue(Value): |
+ def __init__(self, page, name, units, |
+ raw_value=None, raw_value_json=None, important=True): |
+ super(HistogramValue, self).__init__(page, name, units, important) |
+ if raw_value_json: |
+ assert raw_value == None, 'Dont specify both raw_value and raw_value_json' |
+ raw_value = json.loads(raw_value_json) |
+ if raw_value: |
+ assert 'buckets' in raw_value |
+ assert isinstance(raw_value['buckets'], list) |
+ self.buckets = [] |
+ for bucket in raw_value['buckets']: |
+ self.buckets.append(HistogramValueBucket( |
+ low=bucket['low'], |
+ high=bucket['high'], |
+ count=bucket['count'])) |
+ else: |
+ self.buckets = [] |
+ |
+ def __repr__(self): |
+ if self.page: |
+ page_name = self.page.url |
+ else: |
+ page_name = None |
+ return 'HistogramValue(%s, %s, %s, raw_json_string="%s", important=%s)' % ( |
+ page_name, |
+ self.name, self.units, |
+ self.ToJSONString(), |
+ self.important) |
+ |
+ def GetBuildbotDataType(self, output_context): |
+ if self._IsImportantGivenOutputIntent(output_context): |
+ return 'histogram' |
+ return 'unimportant-histogram' |
+ |
+ def GetBuildbotValue(self): |
+ # More buildbot insanity: perf_tests_results_helper requires the histogram |
+ # to be an array of size one. |
+ return [self.ToJSONString()] |
+ |
+ def ToJSONString(self): |
+ # This has to hand-JSONify the histogram to ensure the order of keys |
+ # produced is stable across different systems. |
+ # |
+ # Sigh, buildbot, Y U gotta be that way. |
+ return '{"buckets": [%s]}' % ( |
+ ', '.join([b.ToJSONString() for b in self.buckets])) |
+ |
+ def GetRepresentativeNumberValue(self): |
+ (mean, _) = perf_tests_helper.GeomMeanAndStdDevFromHistogram( |
+ self.ToJSONString()) |
+ return mean |
+ |
+ def GetRepresentativeStringValue(self): |
+ return self.GetBuildbotValue() |
+ |
+ @classmethod |
+ def MergeLikeValuesFromSamePage(cls, values): |
+ assert len(values) > 0 |
+ v0 = values[0] |
+ return HistogramValue( |
+ v0.page, v0.name, v0.units, |
+ raw_value_json=v0.ToJSONString(), |
+ important=v0.important) |
+ |
+ @classmethod |
+ def MergeLikeValuesFromDifferentPages(cls, values): |
+ # Histograms cannot be merged across pages, at least for now. It should be |
+ # theoretically possible, just requires more work. |
+ return None |
+ |
+ |
+def MergeLikeValuesFromSamePage(all_values): |
+ """Merges values that measure the same thing on the same page. |
+ |
+ A page may end up being measured multiple times, meaning that we may end up |
+ with something like this: |
+ ScalarValue(page1, 'x', 1) |
+ ScalarValue(page2, 'x', 4) |
+ ScalarValue(page1, 'x', 2) |
+ ScalarValue(page2, 'x', 5) |
+ |
+ This function will produce: |
+ ListOfScalarValues(page1, 'x', [1, 2]) |
+ ListOfScalarValues(page2, 'x', [4, 5]) |
+ |
+ The workhorse of this code is Value.MergeLikeValuesFromSamePage. |
+ |
+ This requires (but assumes) that the values passed in with the same grouping |
+ key pass the Value.IsMergableWith test. If this is not obeyed, the |
+ results will be undefined. |
+ """ |
+ return _MergeLikeValuesCommon( |
+ all_values, |
+ lambda x: (x.page, x.name), |
+ lambda v0, merge_group: v0.MergeLikeValuesFromSamePage(merge_group)) |
+ |
+ |
+def MergeLikeValuesFromDifferentPages(all_values): |
+ """Merges values that measure the same thing on different pages. |
+ |
+ After using MergeLikeValuesFromSamePage, one still ends up with values from |
+ different pages: |
+ ScalarValue(page1, 'x', 1) |
+ ScalarValue(page1, 'y', 30) |
+ ScalarValue(page2, 'x', 2) |
+ ScalarValue(page2, 'y', 40) |
+ |
+ This function will group the values of the same value_name together: |
+ ListOfScalarValues(None, 'x', [1, 2]) |
+ ListOfScalarValues(None, 'y', [30, 40]) |
+ |
+ The workhorse of this code is Value.MergeLikeValuesFromDifferentPages. |
+ |
+ Not all values that go into this function will come out: not every value can |
+ be merged across pages. Values whose MergeLikeValuesFromDifferentPages returns |
+ None will be omitted from the results. |
+ |
+ This requires (but assumes) that the values passed in with the same name pass |
+ the Value.IsMergableWith test. If this is not obeyed, the results |
+ will be undefined. |
+ """ |
+ return _MergeLikeValuesCommon( |
+ all_values, |
+ lambda x: x.name, |
+ lambda v0, merge_group: v0.MergeLikeValuesFromDifferentPages(merge_group)) |
+ |
+def _MergeLikeValuesCommon(all_values, key_func, merge_func): |
+ """Groups all_values by key_func then applies merge_func to the groups. |
+ |
+ This takes the all_values list and groups each item in that using the key |
+ provided by key_func. This produces groups of values with like keys. Thes are |
+ then handed to the merge_func to produce a new key. If merge_func produces a |
+ non-None return, it is added to the list of returned values. |
+ """ |
+ # When merging, we want to merge values in a consistent order, e.g. so that |
+ # Scalar(1), Scalar(2) predictably produces ListOfScalarValues([1,2]) rather |
+ # than 2,1. |
+ # |
+ # To do this, the values are sorted by key up front. Then, grouping is |
+ # performed using a dictionary, but as new groups are found, the order in |
+ # which they were found is also noted. |
+ # |
+ # Merging is then performed on groups in group-creation-order. This ensures |
+ # that the returned array is in a stable order, group by group. |
+ # |
+ # Within a group, the order is stable because of the original sort. |
+ all_values = list(all_values) |
+ merge_groups = GroupStably(all_values, key_func) |
+ |
+ res = [] |
+ for merge_group in merge_groups: |
+ v0 = merge_group[0] |
+ vM = merge_func(v0, merge_group) |
+ if vM: |
+ res.append(vM) |
+ return res |
+ |
+def GroupStably(all_values, key_func): |
+ """Groups an array by key_func, with the groups returned in a stable order. |
+ |
+ Returns a list of groups. |
+ """ |
+ all_values = list(all_values) |
+ all_values.sort(key=key_func) |
+ |
+ merge_groups = {} |
+ merge_groups_in_creation_order = [] |
+ for value in all_values: |
+ key = key_func(value) |
+ if key not in merge_groups: |
+ merge_groups[key] = [] |
+ merge_groups_in_creation_order.append(merge_groups[key]) |
+ merge_groups[key].append(value) |
+ return merge_groups_in_creation_order |
+ |
+ |
+def ValueNameFromTraceAndChartName(trace_name, chart_name=None): |
+ """Mangles a trace name plus optional chart name into a standard string. |
+ |
+ A value might just be a bareword name, e.g. numPixels. In that case, its |
+ chart may be None. |
+ |
+ But, a value might also be intended for display with other values, in which |
+ case the chart name indicates that grouping. So, you might have |
+ screen.numPixels, screen.resolution, where chartName='screen'. |
+ """ |
+ assert trace_name != 'url', 'The name url cannot be used' |
+ if chart_name: |
+ return '%s.%s' % (chart_name, trace_name) |
+ else: |
+ return trace_name |
+ |
+def _ConvertValueNameToBuildbotMeasurementAndTraceName(value_name): |
+ """Converts a value_name into the buildbot equivalent name pair. |
+ |
+ Buildbot represents values by the measurement name and an optional trace name, |
+ whereas telemetry represents values with a chart_name.trace_name convention, |
+ where chart_name is optional. |
+ |
+ This converts from the telemetry convention to the buildbot convention, |
+ returning a 2-tuple (measurement_name, trace_name). |
+ """ |
+ if '.' in value_name: |
+ return value_name.split('.', 1) |
+ else: |
+ return value_name, value_name |