Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(284)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tracing/tracing/metrics/compare_samples.html

Issue 2089833002: Entry point for bisect sample comparison. (Closed) Base URL: https://github.com/catapult-project/catapult.git@mann
Patch Set: Removed stray line. Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « tracing/bin/compare_samples ('k') | tracing/tracing/metrics/compare_samples.py » ('j') | tracing/tracing/metrics/compare_samples.py » ('J')
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tracing/tracing/metrics/compare_samples.html
diff --git a/tracing/tracing/metrics/compare_samples.html b/tracing/tracing/metrics/compare_samples.html
new file mode 100644
index 0000000000000000000000000000000000000000..0d5f50a90fd20d277d2686f95bfc78d85a56fd97
--- /dev/null
+++ b/tracing/tracing/metrics/compare_samples.html
@@ -0,0 +1,275 @@
+<!DOCTYPE html>
+<!--
+Copyright 2016 The Chromium Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+-->
+
+<link rel="import" href="/tracing/base/iteration_helpers.html">
+<link rel="import" href="/tracing/base/statistics.html">
+<link rel="import" href="/tracing/base/xhr.html">
+<link rel="import" href="/tracing/value/value_set.html">
+
+<script>
+'use strict';
+
+tr.exportTo('tr.metrics', function() {
nednguyen 2016/09/12 17:13:32 You also don't need this export.
RobertoCN 2016/09/19 20:07:43 Done.
nednguyen 2016/09/19 20:14:41 Did you forget to upload your new patch?
+ var escapeChars = s => s.replace(/[\:|=\/#&,]/g, '_');
+
+ function findUnescapedKey(escaped, d) {
+ for (var k of tr.b.dictionaryKeys(d))
+ if (escapeChars(k) === escaped)
+ return k;
+ throw new Error('did not find key ' + escaped + ' in ' +
+ tr.b.dictionaryKeys(d));
+ }
+
+ function geoMeanFromHistogram(h) {
+ if (!h.hasOwnProperty('buckets'))
+ return 0.0;
+ var count = 0;
+ var sumOfLogs = 0;
+ for (var bucket of h.buckets) {
+ if (bucket.hasOwnProperty('high'))
+ bucket.mean = (bucket.low + bucket.high) / 2.0;
+ else
+ bucket.mean = bucket.low;
+
+ if (bucket.mean > 0) {
+ sumOfLogs += Math.log(bucket.mean) * bucket.count;
+ count += bucket.count;
+ }
+ }
+ if (count === 0)
+ return 0.0;
+ return Math.exp(sumOfLogs / count);
+ }
+
+ function splitMetric(metricName) {
+ var parts = metricName.split('/');
+ var interactionName;
+ var traceName = 'summary';
+ var chartName = parts[0];
+ if (parts.length === 3) {
+ // parts[1] is the interactionName
+ if (parts[1])
+ chartName = parts[1] + '@@' + chartName;
+ traceName = parts[2];
+ } else if (parts.length === 2) {
+ if (chartName !== parts[1])
+ traceName = parts[1];
+ } else
+ throw new Error('Could not parse metric name.');
+ return [chartName, traceName];
+ }
+
+ function valuesFromCharts(listOfCharts, metricName) {
+ var all_values = [];
+ var chartAndTrace = splitMetric(metricName);
+ for (var charts of listOfCharts) {
+ var chartName = findUnescapedKey(chartAndTrace[0], charts.charts);
+ if (chartName) {
+ var traceName = findUnescapedKey(
+ chartAndTrace[1], charts.charts[chartName]);
+ if (traceName) {
+ if (charts.charts[chartName][traceName].type ===
+ 'list_of_scalar_values')
+ all_values.push(...charts.charts[chartName][traceName].values);
+ if (charts.charts[chartName][traceName].type === 'histogram')
+ all_values.push(
+ geoMeanFromHistogram(charts.charts[chartName][traceName]));
+ }
+ }
+ }
+ return all_values;
+ }
+
+ function rawValuesByMetricName(valueSet, metricName) {
+ var interactionRecord, valueName, story;
+ var metricNameParts = metricName.split('/');
+ if (metricNameParts[0] === metricNameParts[1])
+ story = 'summary';
+ else
+ story = metricNameParts[1];
+ var chartNameParts = metricNameParts[0].split('-');
+ valueName = chartNameParts[1];
+ if (chartNameParts.length === 2)
+ interactionRecord = chartNameParts[0];
+ var values = valueSet.getValuesWithName(valueName);
+ if (!values || values.length === 0) {
+ // If there was a dash in the chart name, but it wasn't an
+ // interaction record.
+ valueName = metricNameParts[0];
+ values = valueSet.getValuesWithName(valueName);
+ interactionRecord = undefined;
+ if (!values || values.length === 0)
+ throw new Error('No values with name ' + valueName);
+ }
+ var filtered = [];
+ for (var value of values) {
+ if (value.name !== valueName)
+ continue;
+ var ii = tr.v.d.IterationInfo.getFromValue(value);
+ if (interactionRecord) {
+ var IRParts = [];
+ var keys = Object.keys(ii.storyGroupingKeys);
+ keys.sort();
+ for (var key of keys)
+ IRParts.push(ii.storyGroupingKeys[key]);
+ if (interactionRecord === IRParts.join('_') &&
+ escapeChars(ii.storyDisplayName) ===
+ escapeChars(story))
+ filtered.push(value);
+ } else if (escapeChars(ii.storyDisplayName) ===
+ escapeChars(story))
+ filtered.push(value);
+ }
+
+ var rawValues = [];
+ for (var val of filtered) {
+ if (val.numeric instanceof tr.v.Numeric)
+ rawValues = rawValues.concat(val.numeric.sampleValues);
+ else if (val.numeric instanceof tr.v.ScalarNumeric)
+ rawValues.push(val.numeric.value);
+ }
+ return rawValues;
+ }
+
+ function parseFiles(files) {
+ var results = [];
+ for (var path of files) {
+ try {
+ var current = tr.b.getSync('file://' + path);
+ } catch (ex) {
+ var err = new Error('Could not open' + path);
+ err.name = 'File loading error';
+ throw err;
+ }
+ results.push(JSON.parse(current));
+ }
+ return results;
+ }
+
+ var escapeForRegExp = s => s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+
+ var strFromRE = re => re.toString().split('/')[1];
+
+ function valuesFromBuildbotOutput(out, metric) {
+ if (!out)
+ return [];
+ var stringVals = [];
+ var floatVals = [];
+ var chartAndTrace = splitMetric(metric);
+ var metricRE = escapeForRegExp(
+ 'RESULT ' + chartAndTrace[0] + ': ' + chartAndTrace[1] + '=');
+ var singleResultRE = new RegExp(metricRE +
+ strFromRE(/\s*([-]?[\d\.]+)/), 'g');
+ var multiResultsRE = new RegExp(metricRE +
+ strFromRE(/\s*\[\s*([\d\., -]+)\s*\]/), 'g');
+ var meanStdDevRE = new RegExp(metricRE +
+ strFromRE(/\s*\{\s*([-]?\d*(?:\.\d*)?),\s*([-]?\d*(?:\.\d*)?)\}/), 'g');
+ for (var line of out.split(/\r?\n/)) {
+ var singleResultMatch = singleResultRE.exec(line);
+ var multiResultsMatch = multiResultsRE.exec(line);
+ var meanStdDevMatch = meanStdDevRE.exec(line);
+ if (singleResultMatch && singleResultMatch.length > 1)
+ stringVals.push(singleResultMatch[1]);
+ else if (multiResultsMatch && multiResultsMatch.length > 1) {
+ var values = multiResultsMatch[1].split(',');
+ stringVals = stringVals.concat(values);
+ } else if (meanStdDevMatch && meanStdDevMatch.length > 1)
+ stringVals.push(meanStdDevMatch[1]);
+ }
+ for (var val of stringVals) {
+ var f = parseFloat(val);
+ if (!isNaN(f))
+ floatVals.push(f);
+ }
+ return floatVals;
+ }
+
+ function parseMultipleBuildbotStreams(files, metric) {
+ var allValues = [];
+ for (var path of files) {
+ try {
+ var contents = tr.b.getSync('file://' + path);
+ }
+ catch (ex) {
+ var err = new Error('Could not open' + path);
+ err.name = 'File loading error';
+ throw err;
+ }
+ allValues = allValues.concat(valuesFromBuildbotOutput(contents, metric));
+ }
+ return allValues;
+ }
+
+ var BisectComparison = {
dtu 2016/09/15 21:31:56 The word "bisect" isn't used anywhere else. Probab
RobertoCN 2016/09/19 20:07:42 Done.
+ ENOUGH_SAMPLES: 18,
+ SIGNIFICANCE_LEVEL: 0.05,
dtu 2016/09/15 21:31:56 Seems like bisect is currently using 0.01, so I gu
RobertoCN 2016/09/19 20:07:43 Done.
+
+ compareBuildbotOutputs: function(
dtu 2016/09/15 21:31:56 I like these composed functions. Very easy to read
RobertoCN 2016/09/19 20:07:43 Acknowledged.
+ buildbotOutputAPathList, buildbotOutputBPathList, metric) {
+ var aPaths = buildbotOutputAPathList.split(',');
+ var bPaths = buildbotOutputBPathList.split(',');
+ var sampleA = parseMultipleBuildbotStreams(aPaths, metric);
+ var sampleB = parseMultipleBuildbotStreams(bPaths, metric);
+ return this.compareSamples(sampleA, sampleB);
+ },
+
+ compareValuesets: function(valueSetAPathList, valueSetBPathList, metric) {
+ var aPaths = valueSetAPathList.split(',');
+ var bPaths = valueSetBPathList.split(',');
+ var valueSetA = new tr.v.ValueSet();
+ var valueSetB = new tr.v.ValueSet();
+ var dictsA = parseFiles(aPaths);
+ var dictsB = parseFiles(bPaths);
+ for (var d of dictsA)
+ valueSetA.addValuesFromDicts(d);
+ for (var d of dictsB)
+ valueSetB.addValuesFromDicts(d);
+
+ var sampleA = rawValuesByMetricName(valueSetA, metric);
+ var sampleB = rawValuesByMetricName(valueSetB, metric);
+ return this.compareSamples(sampleA, sampleB);
+ },
+
+ compareCharts: function(chartPathListA, chartPathListB, metric) {
+ var aPaths = chartPathListA.split(',');
+ var bPaths = chartPathListB.split(',');
+ var chartsA = parseFiles(aPaths);
+ var chartsB = parseFiles(bPaths);
+ var sampleA = valuesFromCharts(chartsA, metric);
+ var sampleB = valuesFromCharts(chartsB, metric);
+ return this.compareSamples(sampleA, sampleB);
+ },
+
+ compareSamples: function(sampleA, sampleB) {
+ var pValue = tr.b.Statistics.mwu.test(sampleA, sampleB);
+ // Diagnostics
+ var summaryStats = sample => ({
+ std_dev: tr.b.Statistics.stddev(sample),
+ mean: tr.b.Statistics.mean(sample),
dtu 2016/09/15 21:35:41 Question for Ben and Ethan: are you going to be us
benjhayden 2016/09/21 06:33:25 No, I don't think that results2.html will be using
+ debug_values: sample
+ });
+ var result = {
+ sample_a: summaryStats(sampleA),
+ sample_b: summaryStats(sampleB),
+ pValue: pValue.p,
+ UStatistic: pValue.U,
dtu 2016/09/15 21:31:56 Why is the U-statistic useful?
RobertoCN 2016/09/19 20:07:42 Maybe it's not. I decided to surface it because it
+ result: 'needMoreData',
+ };
+ if (pValue.p < this.SIGNIFICANCE_LEVEL)
+ result.result = true; // Reject the null
dtu 2016/09/15 21:31:56 Don't mix types. Make them all string constants, I
RobertoCN 2016/09/19 20:07:43 Done.
+ else if (sampleA.length > this.ENOUGH_SAMPLES &&
+ sampleB.length > this.ENOUGH_SAMPLES)
+ result.result = false; // Fail to reject the null.
+ return result;
+ }
+ };
+
+ return {
+ BisectComparison: BisectComparison
+ };
+});
+</script>
« no previous file with comments | « tracing/bin/compare_samples ('k') | tracing/tracing/metrics/compare_samples.py » ('j') | tracing/tracing/metrics/compare_samples.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698